Merge pull request #3921 from Kriechi/example-docs

Example docs

19 files changed, 178 insertions, 83 deletions

diff --git a/dev.sh b/dev.sh
index a90b48ab..df7b22d4 100755
--- a/dev.sh
+++ b/dev.sh
@@ -1,6 +1,9 @@
-#!/bin/sh
-set -e
-set -x
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+set -o xtrace
 
 echo "Creating dev environment in ./venv..."
 
@@ -12,4 +15,4 @@ pip3 install -r requirements.txt
 echo ""
 echo "  * Created virtualenv environment in ./venv."
 echo "  * Installed all dependencies into the virtualenv."
-echo "  * You can now activate the $(python3 --version) virtualenv with this command: \`. venv/bin/activate\`"
\ No newline at end of file
+echo "  * You can now activate the $(python3 --version) virtualenv with this command: \`. venv/bin/activate\`"
diff --git a/docs/.gitignore b/docs/.gitignore
index 1fb99949..610ffdf1 100644
--- a/docs/.gitignore
+++ b/docs/.gitignore
@@ -1,5 +1,6 @@
 generated/
-src/public
-node_modules
-public
+src/public/
+node_modules/
+public/
 src/resources/_gen/
+src/content/addons-examples.md
diff --git a/docs/README.md b/docs/README.md
index 5c99fb39..24c24d24 100644
--- a/docs/README.md
+++ b/docs/README.md
@@ -8,7 +8,7 @@ This directory houses the mitmproxy documentation available at <https://docs.mit
  2. Windows users: Depending on your git settings, you may need to manually create a symlink from
  /docs/src/examples to /examples.
  3. Make sure the mitmproxy Python package is installed.
- 4. Run `./build-current` to generate the documentation source files in `./src/generated`.
+ 4. Run `./build.sh` to generate additional documentation source files.
 
 Now you can run `hugo server -D` in ./src.
 
diff --git a/docs/build-archive b/docs/build-archive
deleted file mode 100755
index 004e625a..00000000
--- a/docs/build-archive
+++ /dev/null
@@ -1,4 +0,0 @@
-#!/bin/sh
-set -e
-
-DOCS_ARCHIVE=true ./build-current
diff --git a/docs/build-current b/docs/build-current
deleted file mode 100755
index 7164de6d..00000000
--- a/docs/build-current
+++ /dev/null
@@ -1,12 +0,0 @@
-#!/usr/bin/env bash
-
-set -e
-
-for script in scripts/* ; do
-  echo "Generating output for $script ..."
-  output="${script##*/}"
-  "$script" > "src/generated/${output%.*}.html"
-done
-
-cd src
-hugo
diff --git a/docs/build.sh b/docs/build.sh
new file mode 100755
index 00000000..aaa52a2f
--- /dev/null
+++ b/docs/build.sh
@@ -0,0 +1,23 @@
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+# set -o xtrace
+
+SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )"
+pushd ${SCRIPTPATH}
+
+for script in scripts/* ; do
+  output="${script##*/}"
+  output="src/generated/${output%.*}.html"
+  echo "Generating output for ${script} into ${output} ..."
+  "${script}" > "${output}"
+done
+
+output="src/content/addons-examples.md"
+echo "Generating examples content page into ${output} ..."
+./render_examples.py > "${output}"
+
+cd src
+hugo
diff --git a/docs/ci b/docs/ci.sh
index 95a5218e..159d0b50 100755
--- a/docs/ci
+++ b/docs/ci.sh
@@ -1,9 +1,13 @@
-#!/bin/bash
-set -e
+#!/usr/bin/env bash
 
-# This script gets run from CI to render and upload docs
+set -o errexit
+set -o pipefail
+set -o nounset
+# set -o xtrace
 
-./build-current
+# This script gets run from CI to render and upload docs for the master branch.
+
+./build.sh
 
 # Only upload if we have defined credentials - we only have these defined for
 # trusted commits (i.e. not PRs).
diff --git a/docs/render_examples.py b/docs/render_examples.py
new file mode 100755
index 00000000..9c6dea74
--- /dev/null
+++ b/docs/render_examples.py
@@ -0,0 +1,50 @@
+#!/usr/bin/env python3
+
+import os
+import textwrap
+from pathlib import Path
+
+print("""
+---
+title: "Examples"
+menu:
+    addons:
+        weight: 6
+---
+
+# Examples of Addons and Scripts
+
+The most recent set of examples is also available [on our GitHub project](https://github.com/mitmproxy/mitmproxy/tree/master/examples).
+
+""")
+
+base = os.path.dirname(os.path.realpath(__file__))
+examples_path = os.path.join(base, 'src/examples/')
+pathlist = Path(examples_path).glob('**/*.py')
+
+examples = [os.path.relpath(str(p), examples_path) for p in sorted(pathlist)]
+examples = [p for p in examples if not os.path.basename(p) == '__init__.py']
+examples = [p for p in examples if not os.path.basename(p).startswith('test_')]
+
+current_dir = None
+current_level = 2
+for ex in examples:
+    if os.path.dirname(ex) != current_dir:
+        current_dir = os.path.dirname(ex)
+        sanitized = current_dir.replace('/', '').replace('.', '')
+        print("  * [Examples: {}]({{{{< relref \"addons-examples#{}\">}}}})".format(current_dir, sanitized))
+
+    sanitized = ex.replace('/', '').replace('.', '')
+    print("    * [{}]({{{{< relref \"addons-examples#example-{}\">}}}})".format(os.path.basename(ex), sanitized))
+
+current_dir = None
+current_level = 2
+for ex in examples:
+    if os.path.dirname(ex) != current_dir:
+        current_dir = os.path.dirname(ex)
+        print("#" * current_level, current_dir)
+
+    print(textwrap.dedent("""
+        {} Example: {}
+        {{{{< example src="{}" lang="py" >}}}}
+    """.format("#" * (current_level + 1), ex, "examples/" + ex)))
diff --git a/docs/setup b/docs/setup.sh
index cb63841a..da30a3c9 100755
--- a/docs/setup
+++ b/docs/setup.sh
@@ -1,5 +1,11 @@
-#!/bin/sh
-set -e
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+# set -o xtrace
+
+# This is only needed once to provision a new fresh empty S3 bucket.
 
 aws configure set preview.cloudfront true
 aws --profile mitmproxy \
diff --git a/docs/src/assets/style.scss b/docs/src/assets/style.scss
index 26c22071..33e8863e 100644
--- a/docs/src/assets/style.scss
+++ b/docs/src/assets/style.scss
@@ -47,6 +47,7 @@ body > div {
         width: 100%;
         text-align: right;
     }
+    max-width: 70vw;
     margin-bottom: 1em;
 }
 
diff --git a/docs/src/layouts/shortcodes/example.html b/docs/src/layouts/shortcodes/example.html
index d23cabb6..83a6075d 100644
--- a/docs/src/layouts/shortcodes/example.html
+++ b/docs/src/layouts/shortcodes/example.html
@@ -1,5 +1,4 @@
-
 <div class="example">
 {{ highlight (trim (readFile (.Get "src")) "\n\r") (.Get "lang") "" }}
 <div class="path">{{ (.Get "src" )}}</div>
-</div>
\ No newline at end of file
+</div>
diff --git a/docs/upload-archive b/docs/upload-archive.sh
index 3aaeb9be..e35345e9 100755
--- a/docs/upload-archive
+++ b/docs/upload-archive.sh
@@ -1,5 +1,9 @@
-#!/bin/bash
-set -e
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+# set -o xtrace
 
 if [[ $# -eq 0 ]] ; then
     echo "Please supply a version, e.g. 'v3'"
diff --git a/docs/upload-stable b/docs/upload-stable.sh
index 5aea7479..a2f20f01 100755
--- a/docs/upload-stable
+++ b/docs/upload-stable.sh
@@ -1,5 +1,9 @@
-#!/bin/bash
-set -e
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+# set -o xtrace
 
 aws configure set preview.cloudfront true
 aws --profile mitmproxy \
diff --git a/examples/complex/block_dns_over_https.py b/examples/complex/block_dns_over_https.py
index 479f0baa..5b0b24cf 100644
--- a/examples/complex/block_dns_over_https.py
+++ b/examples/complex/block_dns_over_https.py
@@ -31,36 +31,45 @@ default_blocklist: dict = {
         "dns.google.com"
     ],
     "ips": [
-        "176.103.130.131", "176.103.130.130", "2a00:5a60::ad1:ff", "2a00:5a60::ad2:ff", "176.103.130.134", "176.103.130.132",
-        "2a00:5a60::bad2:ff", "2a00:5a60::bad1:ff", "8.8.4.4", "8.8.8.8", "2001:4860:4860::8888", "2001:4860:4860::8844",
-        "104.16.248.249", "104.16.249.249", "2606:4700::6810:f8f9", "2606:4700::6810:f9f9", "104.16.248.249", "104.16.249.249",
-        "2606:4700::6810:f9f9", "2606:4700::6810:f8f9", "104.18.2.55", "104.18.3.55", "2606:4700::6812:337", "2606:4700::6812:237",
-        "104.18.27.128", "104.18.26.128", "2606:4700::6812:1a80", "2606:4700::6812:1b80", "9.9.9.9", "149.112.112.112", "2620:fe::9",
-        "2620:fe::fe", "9.9.9.9", "149.112.112.9", "2620:fe::fe:9", "2620:fe::9", "9.9.9.10", "149.112.112.10", "2620:fe::10",
-        "2620:fe::fe:10", "9.9.9.11", "149.112.112.11", "2620:fe::fe:11", "2620:fe::11", "146.112.41.2", "2620:119:fc::2",
-        "146.112.41.3", "2620:119:fc::3", "185.228.168.168", "185.228.168.10", "96.113.151.148", "2001:558:fe21:6b:96:113:151:149",
-        "174.68.248.77", "185.43.135.1", "2001:148f:fffe::1", "185.235.81.1", "2a0d:4d00:81::1", "45.90.28.0", "2a07:a8c0::",
-        "104.236.178.232", "2604:a880:1:20::51:f001", "104.28.1.106", "104.28.0.106", "2606:4700:3036::681c:6a",
-        "2606:4700:3034::681c:16a", "136.144.215.158", "2a01:7c8:d002:1ef:5054:ff:fe40:3703", "95.216.212.177",
-        "2a01:4f9:c010:43ce::1", "45.32.55.94", "2001:19f0:7001:3259:5400:2ff:fe71:bc9", "159.69.198.101", "2a01:4f8:1c1c:6b4b::1",
-        "195.30.94.28", "2001:608:a01::3", "104.24.122.53", "104.24.123.53", "2606:4700:3033::6818:7b35", "2606:4700:3035::6818:7a35",
-        "146.185.167.43", "2a03:b0c0:0:1010::e9a:3001", "115.159.131.230", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077",
-        "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077",
-        "139.99.222.72", "45.76.113.31", "104.182.57.196", "168.235.81.167", "2604:180:f3::42", "176.56.236.175", "2a00:d880:5:bf0::7c93",
-        "94.130.106.88", "2a03:4000:38:53c::2", "139.59.48.222", "174.138.29.175", "2400:6180:0:d0::5f73:4001", "104.18.45.204",
-        "104.18.44.204", "2606:4700:3033::6812:2dcc", "2606:4700:3033::6812:2ccc", "104.31.91.138", "104.31.90.138",
-        "2606:4700:3035::681f:5a8a", "2606:4700:3036::681f:5b8a", "185.134.196.54", "46.227.200.55", "46.227.200.54", "185.134.197.54",
-        "2a01:9e00::54", "2a01:9e01::54", "2a01:9e00::55", "2a01:9e01::55", "46.101.66.244", "172.104.93.80",
-        "2400:8902::f03c:91ff:feda:c514", "104.18.44.204", "104.18.45.204", "2606:4700:3033::6812:2ccc", "2606:4700:3033::6812:2dcc",
-        "185.216.27.142", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28",
-        "217.169.20.22", "217.169.20.23", "2001:8b0::2022", "2001:8b0::2023", "172.65.3.223", "2606:4700:60:0:a71e:6467:cef8:2a56",
-        "83.77.85.7", "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", "178.62.214.105", "35.198.2.76", "210.17.9.228",
-        "2001:c50:ffff:1:101:101:101:101", "35.231.247.227", "185.95.218.43", "185.95.218.42", "2a05:fc84::43", "2a05:fc84::42",
-        "116.203.115.192", "116.202.176.26", "2a01:4f8:c2c:52bf::1", "88.198.91.187", "2a01:4f8:1c0c:8233::1", "95.216.181.228",
-        "2a01:4f9:c01f:4::abcd", "45.67.219.208", "2a04:bdc7:100:70::abcd", "185.213.26.187", "2a0d:5600:33:3::abcd", "46.239.223.80",
-        "2001:678:888:69:c45d:2738:c3f2:1878", "149.112.121.10", "149.112.122.10", "2620:10a:80bb::10", "2620:10a:80bc::10",
-        "149.112.121.20", "149.112.122.20", "2620:10a:80bb::20", "2620:10a:80bc::20", "149.112.121.30", "149.112.122.30",
-        "2620:10a:80bc::30", "2620:10a:80bb::30"
+        "104.16.248.249", "104.16.248.249", "104.16.249.249", "104.16.249.249", "104.18.2.55",
+        "104.18.26.128", "104.18.27.128", "104.18.3.55", "104.18.44.204", "104.18.44.204",
+        "104.18.45.204", "104.18.45.204", "104.182.57.196", "104.236.178.232", "104.24.122.53",
+        "104.24.123.53", "104.28.0.106", "104.28.1.106", "104.31.90.138", "104.31.91.138",
+        "115.159.131.230", "116.202.176.26", "116.203.115.192", "136.144.215.158", "139.59.48.222",
+        "139.99.222.72", "146.112.41.2", "146.112.41.3", "146.185.167.43", "149.112.112.10",
+        "149.112.112.11", "149.112.112.112", "149.112.112.9", "149.112.121.10", "149.112.121.20",
+        "149.112.121.30", "149.112.122.10", "149.112.122.20", "149.112.122.30", "159.69.198.101",
+        "168.235.81.167", "172.104.93.80", "172.65.3.223", "174.138.29.175", "174.68.248.77",
+        "176.103.130.130", "176.103.130.131", "176.103.130.132", "176.103.130.134", "176.56.236.175",
+        "178.62.214.105", "185.134.196.54", "185.134.197.54", "185.213.26.187", "185.216.27.142",
+        "185.228.168.10", "185.228.168.168", "185.235.81.1", "185.26.126.37", "185.26.126.37",
+        "185.43.135.1", "185.95.218.42", "185.95.218.43", "195.30.94.28", "2001:148f:fffe::1",
+        "2001:19f0:7001:3259:5400:2ff:fe71:bc9", "2001:19f0:7001:5554:5400:2ff:fe57:3077",
+        "2001:19f0:7001:5554:5400:2ff:fe57:3077", "2001:19f0:7001:5554:5400:2ff:fe57:3077",
+        "2001:4860:4860::8844", "2001:4860:4860::8888",
+        "2001:4b98:dc2:43:216:3eff:fe86:1d28", "2001:558:fe21:6b:96:113:151:149",
+        "2001:608:a01::3", "2001:678:888:69:c45d:2738:c3f2:1878", "2001:8b0::2022", "2001:8b0::2023",
+        "2001:c50:ffff:1:101:101:101:101", "210.17.9.228", "217.169.20.22", "217.169.20.23",
+        "2400:6180:0:d0::5f73:4001", "2400:8902::f03c:91ff:feda:c514", "2604:180:f3::42",
+        "2604:a880:1:20::51:f001", "2606:4700::6810:f8f9", "2606:4700::6810:f9f9", "2606:4700::6812:1a80",
+        "2606:4700::6812:1b80", "2606:4700::6812:237", "2606:4700::6812:337", "2606:4700:3033::6812:2ccc",
+        "2606:4700:3033::6812:2dcc", "2606:4700:3033::6818:7b35", "2606:4700:3034::681c:16a",
+        "2606:4700:3035::6818:7a35", "2606:4700:3035::681f:5a8a", "2606:4700:3036::681c:6a",
+        "2606:4700:3036::681f:5b8a", "2606:4700:60:0:a71e:6467:cef8:2a56", "2620:10a:80bb::10",
+        "2620:10a:80bb::20", "2620:10a:80bb::30" "2620:10a:80bc::10", "2620:10a:80bc::20",
+        "2620:10a:80bc::30", "2620:119:fc::2", "2620:119:fc::3", "2620:fe::10", "2620:fe::11",
+        "2620:fe::9", "2620:fe::fe:10", "2620:fe::fe:11", "2620:fe::fe:9", "2620:fe::fe",
+        "2a00:5a60::ad1:ff", "2a00:5a60::ad2:ff", "2a00:5a60::bad1:ff", "2a00:5a60::bad2:ff",
+        "2a00:d880:5:bf0::7c93", "2a01:4f8:1c0c:8233::1", "2a01:4f8:1c1c:6b4b::1", "2a01:4f8:c2c:52bf::1",
+        "2a01:4f9:c010:43ce::1", "2a01:4f9:c01f:4::abcd", "2a01:7c8:d002:1ef:5054:ff:fe40:3703",
+        "2a01:9e00::54", "2a01:9e00::55", "2a01:9e01::54", "2a01:9e01::55",
+        "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", "2a03:4000:38:53c::2",
+        "2a03:b0c0:0:1010::e9a:3001", "2a04:bdc7:100:70::abcd", "2a05:fc84::42", "2a05:fc84::43",
+        "2a07:a8c0::", "2a0d:4d00:81::1", "2a0d:5600:33:3::abcd", "35.198.2.76", "35.231.247.227",
+        "45.32.55.94", "45.67.219.208", "45.76.113.31", "45.77.180.10", "45.90.28.0",
+        "46.101.66.244", "46.227.200.54", "46.227.200.55", "46.239.223.80", "8.8.4.4",
+        "8.8.8.8", "83.77.85.7", "88.198.91.187", "9.9.9.10", "9.9.9.11", "9.9.9.9",
+        "94.130.106.88", "95.216.181.228", "95.216.212.177", "96.113.151.148",
     ]
 }
 
diff --git a/examples/complex/remote_debug.py b/examples/complex/remote_debug.py
index 4b117bdb..5129c9db 100644
--- a/examples/complex/remote_debug.py
+++ b/examples/complex/remote_debug.py
@@ -4,9 +4,11 @@ For general debugging purposes, it is easier to just debug mitmdump within PyCha
 
 Usage:
     - pip install pydevd on the mitmproxy machine
-    - Open the Run/Debug Configuration dialog box in PyCharm, and select the Python Remote Debug configuration type.
-    - Debugging works in the way that mitmproxy connects to the debug server on startup.
-      Specify host and port that mitmproxy can use to reach your PyCharm instance on startup.
+    - Open the Run/Debug Configuration dialog box in PyCharm, and select the
+      Python Remote Debug configuration type.
+    - Debugging works in the way that mitmproxy connects to the debug server
+      on startup. Specify host and port that mitmproxy can use to reach your
+      PyCharm instance on startup.
     - Adjust this inline script accordingly.
     - Start debug server in PyCharm
     - Set breakpoints
diff --git a/examples/complex/sslstrip.py b/examples/complex/sslstrip.py
index 8b904216..16d9b59a 100644
--- a/examples/complex/sslstrip.py
+++ b/examples/complex/sslstrip.py
@@ -51,9 +51,11 @@ def response(flow: http.HTTPFlow) -> None:
         flow.response.headers['Location'] = location.replace('https://', 'http://', 1)
 
     # strip upgrade-insecure-requests in Content-Security-Policy header
-    if re.search('upgrade-insecure-requests', flow.response.headers.get('Content-Security-Policy', ''), flags=re.IGNORECASE):
+    csp_header = flow.response.headers.get('Content-Security-Policy', '')
+    if re.search('upgrade-insecure-requests', csp_header, flags=re.IGNORECASE):
         csp = flow.response.headers['Content-Security-Policy']
-        flow.response.headers['Content-Security-Policy'] = re.sub(r'upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE)
+        new_header = re.sub(r'upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE)
+        flow.response.headers['Content-Security-Policy'] = new_header
 
     # strip secure flag from 'Set-Cookie' headers
     cookies = flow.response.headers.get_all('Set-Cookie')
diff --git a/release/README.md b/release/README.md
index 8632d644..1de0824f 100644
--- a/release/README.md
+++ b/release/README.md
@@ -37,10 +37,10 @@ These steps assume you are on the correct branch and have a git remote called `o
   `export VERSION=4.0.3 && docker pull mitmproxy/mitmproxy:$VERSION && docker tag mitmproxy/mitmproxy:$VERSION mitmproxy/mitmproxy:latest && docker push mitmproxy/mitmproxy:latest`.
 
 ### Docs
-  - `./build-current`. If everything looks alright, continue with
-  - `./upload-stable`,
-  - `./build-archive`, and
-  - `./upload-archive v4`. Doing this now already saves you from switching back to an old state on the next release.
+  - `./build.sh`. If everything looks alright, continue with
+  - `./upload-stable.sh`,
+  - `DOCS_ARCHIVE=true ./build.sh`, and
+  - `./upload-archive.sh v4`. Doing this now already saves you from switching back to an old state on the next release.
 
 ### Website
  - Update version here:
diff --git a/release/docker/docker-entrypoint.sh b/release/docker/docker-entrypoint.sh
index a4abe4ce..7bb3028a 100755
--- a/release/docker/docker-entrypoint.sh
+++ b/release/docker/docker-entrypoint.sh
@@ -1,13 +1,16 @@
-#!/bin/sh
-set -e
+#!/usr/bin/env bash
+
+set -o errexit
+set -o pipefail
+set -o nounset
+# set -o xtrace
 
 MITMPROXY_PATH="/home/mitmproxy/.mitmproxy"
 
 if [[ "$1" = "mitmdump" || "$1" = "mitmproxy" || "$1" = "mitmweb" ]]; then
-        mkdir -p "$MITMPROXY_PATH"
-        chown -R mitmproxy:mitmproxy "$MITMPROXY_PATH"
-
-        su-exec mitmproxy "$@"
+  mkdir -p "$MITMPROXY_PATH"
+  chown -R mitmproxy:mitmproxy "$MITMPROXY_PATH"
+  su-exec mitmproxy "$@"
 else
-        exec "$@"
+  exec "$@"
 fi
diff --git a/tox.ini b/tox.ini
index 6ae38cbb..353a4d18 100644
--- a/tox.ini
+++ b/tox.ini
@@ -76,4 +76,4 @@ deps =
   awscli
 changedir = docs
 commands =
-  ./ci
+  ./ci.sh