Merge pull request #3012 from Kriechi/docs-options

docs: add auto-generated options reference

13 files changed, 126 insertions, 65 deletions

diff --git a/.travis.yml b/.travis.yml
index 66a4f61e..efdf6172 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -42,15 +42,15 @@ matrix:
           yarn: true
           directories:
               - web/node_modules
-    - language: python
-      env:
-        - NAME=docs
+    - python: 3.6
+      env: NAME=docs TOXENV=docs
       install:
         - wget https://github.com/gohugoio/hugo/releases/download/v0.37/hugo_0.37_Linux-64bit.deb
         - sudo dpkg -i hugo*.deb
-        - pip install awscli
+        - pip install tox virtualenv setuptools
+        - pyenv global system 3.6
       script:
-        - cd docs && ./ci
+        - tox
       after_success:
         - echo done
 
diff --git a/docs/.gitignore b/docs/.gitignore
index 0eba9c59..64b6fff6 100644
--- a/docs/.gitignore
+++ b/docs/.gitignore
@@ -1,3 +1,4 @@
+generated/
 src/public
 node_modules
 public
diff --git a/docs/build-current b/docs/build-current
index a78acab4..7164de6d 100755
--- a/docs/build-current
+++ b/docs/build-current
@@ -1,5 +1,12 @@
-#!/bin/sh
+#!/usr/bin/env bash
+
 set -e
 
+for script in scripts/* ; do
+  echo "Generating output for $script ..."
+  output="${script##*/}"
+  "$script" > "src/generated/${output%.*}.html"
+done
+
 cd src
 hugo
diff --git a/docs/scripts/filters.py b/docs/scripts/filters.py
index e61733a0..05cc7a0f 100755
--- a/docs/scripts/filters.py
+++ b/docs/scripts/filters.py
@@ -1,4 +1,4 @@
-#!/usr/bin/env python
+#!/usr/bin/env python3
 
 from mitmproxy import flowfilter
 
@@ -6,4 +6,4 @@ from mitmproxy import flowfilter
 print("<table class=\"table filtertable\"><tbody>")
 for i in flowfilter.help:
     print("<tr><th>%s</th><td>%s</td></tr>" % i)
-print("</tbody></table>")
\ No newline at end of file
+print("</tbody></table>")
diff --git a/docs/scripts/options.py b/docs/scripts/options.py
new file mode 100755
index 00000000..ff7d0f7f
--- /dev/null
+++ b/docs/scripts/options.py
@@ -0,0 +1,53 @@
+#!/usr/bin/env python3
+
+from mitmproxy import options, optmanager
+from mitmproxy.tools import dump, console, web
+
+masters = {
+    "mitmproxy": console.master.ConsoleMaster,
+    "mitmdump": dump.DumpMaster,
+    "mitmweb": web.master.WebMaster
+}
+
+unified_options = {}
+
+for tool_name, master in masters.items():
+    opts = options.Options()
+    inst = master(opts)
+    for key, option in optmanager.dump_dicts(opts).items():
+        if key in unified_options:
+            unified_options[key]['tools'].append(tool_name)
+        else:
+            unified_options[key] = option
+            unified_options[key]['tools'] = [tool_name]
+
+print("""
+      <table class=\"table optiontable\">
+        <thead>
+          <tr>
+            <th>Name</th>
+            <th>Type</th>
+            <th>Description</th>
+        </tr>
+        </thead>
+        <tbody>
+      """.strip())
+for key, option in sorted(unified_options.items(), key=lambda t: t[0]):
+    print("""
+          <tr>
+          <th>{}<br/>{}</th>
+          <td>{}</td>
+          <td>{}<br/>
+            Default: {}
+            {}
+          </td>
+          </tr>
+          """.strip().format(
+              key,
+              ' '.join(["<span class='badge'>{}</span>".format(t) for t in option['tools']]),
+              option['type'],
+              option['help'],
+              option['default'],
+              "<br/>Choices: {}".format(', '.join(option['choices'])) if option['choices'] else "",
+    ))
+print("</tbody></table>")
diff --git a/docs/src/content/concepts-filters.md b/docs/src/content/concepts-filters.md
index ada24e32..f0b9a4b2 100644
--- a/docs/src/content/concepts-filters.md
+++ b/docs/src/content/concepts-filters.md
@@ -10,37 +10,7 @@ menu:
 Many commands in the mitmproxy tool make use of filter expressions. Filter
 expressions consist of the following operators:
 
-
-<table class="table filtertable"><tbody>
-<tr><th>~a</th><td>Match asset in response: CSS, Javascript, Flash, images.</td></tr>
-<tr><th>~b regex</th><td>Body</td></tr>
-<tr><th>~bq regex</th><td>Request body</td></tr>
-<tr><th>~bs regex</th><td>Response body</td></tr>
-<tr><th>~c int</th><td>HTTP response code</td></tr>
-<tr><th>~d regex</th><td>Domain</td></tr>
-<tr><th>~dst regex</th><td>Match destination address</td></tr>
-<tr><th>~e</th><td>Match error</td></tr>
-<tr><th>~h regex</th><td>Header</td></tr>
-<tr><th>~hq regex</th><td>Request header</td></tr>
-<tr><th>~hs regex</th><td>Response header</td></tr>
-<tr><th>~http</th><td>Match HTTP flows</td></tr>
-<tr><th>~m regex</th><td>Method</td></tr>
-<tr><th>~marked</th><td>Match marked flows</td></tr>
-<tr><th>~q</th><td>Match request with no response</td></tr>
-<tr><th>~s</th><td>Match response</td></tr>
-<tr><th>~src regex</th><td>Match source address</td></tr>
-<tr><th>~t regex</th><td>Content-type header</td></tr>
-<tr><th>~tcp</th><td>Match TCP flows</td></tr>
-<tr><th>~tq regex</th><td>Request Content-Type header</td></tr>
-<tr><th>~ts regex</th><td>Response Content-Type header</td></tr>
-<tr><th>~u regex</th><td>URL</td></tr>
-<tr><th>~websocket</th><td>Match WebSocket flows</td></tr>
-<tr><th>!</th><td>unary not</td></tr>
-<tr><th>&</th><td>and</td></tr>
-<tr><th>|</th><td>or</td></tr>
-<tr><th>(...)</th><td>grouping</td></tr>
-</tbody></table>
-
+{{< readfile file="/generated/filters.html" >}}
 
 - Regexes are Python-style
 - Regexes can be specified as quoted strings
diff --git a/docs/src/content/concepts-options.md b/docs/src/content/concepts-options.md
index 18776841..b533c175 100644
--- a/docs/src/content/concepts-options.md
+++ b/docs/src/content/concepts-options.md
@@ -34,33 +34,14 @@ interactively have immediate effect in the running instance, and can be made
 persistent by saving the settings out to a YAML configuration file (please see
 the specific tool's interactive help for details on how to do this).
 
-For all tools, options can be set directly by name using the `--set` command-line
-option. Please see the command-line help (`--help`) for usage.
+For all tools, options can be set directly by name using the `--set`
+command-line option. Please see the command-line help (`--help`) for usage.
 
 
-## Example
-
-Here is an excerpt showing the first few lines of the complete `--options`
-output:
-
-{{< highlight yaml  >}}
-
-# Add all certificates of the upstream server to the certificate chain
-# that will be served to the proxy client, as extras. Type bool.
-add_upstream_certs_to_client_chain: false
-
-# Allow remote clients to connect to proxy. If set to false, client will
-# not be able to connect to proxy unless it is on the same network or
-# the proxyauth option is set Type bool.
-allow_remote: false
-
-# Strip out request headers that might cause the server to return
-# 304-not-modified. Type bool.
-anticache: false
-
-# Try to convince servers to send us un-compressed data. Type bool.
-anticomp: false
-
-{{< /highlight >}}
+## Available Options
 
+This list might not reflect what is actually available in your current mitmproxy
+environment. For an up-to-date list please use the `--options` flag for each of
+the mitmproxy tools.
 
+{{< readfile file="/generated/options.html" >}}
diff --git a/docs/src/generated/.gitkeep b/docs/src/generated/.gitkeep
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/docs/src/generated/.gitkeep
diff --git a/docs/src/layouts/shortcodes/readfile.html b/docs/src/layouts/shortcodes/readfile.html
new file mode 100644
index 00000000..6860b0f1
--- /dev/null
+++ b/docs/src/layouts/shortcodes/readfile.html
@@ -0,0 +1,6 @@
+{{$file := .Get "file"}}
+{{- if eq (.Get "markdown") "true" -}}
+{{- $file  | readFile | markdownify -}}
+{{- else -}}
+{{ $file  | readFile | safeHTML }}
+{{- end -}}
diff --git a/docs/src/themes/mitmproxydocs/static/css/style.css b/docs/src/themes/mitmproxydocs/static/css/style.css
index db5a36cf..d42e307e 100644
--- a/docs/src/themes/mitmproxydocs/static/css/style.css
+++ b/docs/src/themes/mitmproxydocs/static/css/style.css
@@ -250,6 +250,21 @@
 .chroma .gu {
   color: #75715e; }
 
+.badge {
+  color: #fff;
+  background-color: #6c757d;
+  display: inline-block;
+  padding: .25em .4em;
+  font-size: 75%;
+  font-weight: 1;
+  line-height: 1;
+  text-align: center;
+  white-space: nowrap;
+  vertical-align: baseline;
+  border-radius: 0.25rem; }
+  .badge:empty {
+    display: none; }
+
 @keyframes spinAround {
   from {
     transform: rotate(0deg); }
diff --git a/docs/style/badge.scss b/docs/style/badge.scss
new file mode 100644
index 00000000..8082f6c7
--- /dev/null
+++ b/docs/style/badge.scss
@@ -0,0 +1,18 @@
+.badge {
+  color: #fff;
+  background-color: #6c757d;
+  display: inline-block;
+  padding: .25em .4em;
+  font-size: 75%;
+  font-weight: 1;
+  line-height: 1;
+  text-align: center;
+  white-space: nowrap;
+  vertical-align: baseline;
+  border-radius: 0.25rem;
+
+  // Empty badges collapse automatically
+  &:empty {
+    display: none;
+  }
+}
diff --git a/docs/style/style.scss b/docs/style/style.scss
index 2b0d2993..b404dc42 100644
--- a/docs/style/style.scss
+++ b/docs/style/style.scss
@@ -1,4 +1,5 @@
 @import "./syntax";
+@import "./badge";
 
 $primary: #C93312;
 $warning-invert: #FFFFFF;
diff --git a/tox.ini b/tox.ini
index 2d0fd047..c88d64a8 100644
--- a/tox.ini
+++ b/tox.ini
@@ -41,3 +41,12 @@ deps =
 commands =
   mitmdump --version
   python ./release/ci.py {posargs}
+
+[testenv:docs]
+passenv = TRAVIS_* AWS_* APPVEYOR_* RTOOL_KEY WHEEL
+deps =
+  -rrequirements.txt
+  awscli
+changedir = docs
+commands =
+  ./ci