diff options
35 files changed, 320 insertions, 187 deletions
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 8e8080db..fb75993f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -7,7 +7,7 @@ assignees: '' --- -**Is your feature request related to a problem? Please describe.** +#### Is your feature request related to a problem? Please describe. A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] #### Describe the solution you'd like @@ -1,3 +1,26 @@ +13 April 2020: mitmproxy 5.1.1 + * Fixed Docker images not starting due to missing shell + +13 April 2020: mitmproxy 5.1 + + ** Major Changes ** + * Initial Support for TLS 1.3 + + ** Full Changelog ** + * Reduce leaf certificate validity to one year due to upcoming browser changes (@mhils) + * Rename mitmweb's web_iface option to web_host for consistency (@oxr463) + * Sending a SIGTERM now exits mitmproxy without prompt, SIGINT still asks (@ThinkChaos) + * Don't force host header on outgoing requests (@mhils) + * Additional documentation and examples for WebSockets (@Kriechi) + * Gracefully handle hyphens in domain names (@matosconsulting) + * Fix header replacement count (@naivekun) + * Emit serverconnect event only after a connection has been established (@Prinzhorn) + * Fix ValueError in table mode of server replay flow (@ylmrx) + * HTTP/2: send all stream reset types to other connection (@rohfle) + * HTTP/2: fix WINDOW_UPDATE swallowed on closed streams (@Kriechi) + * Fix wrong behavior of --allow-hosts options (@BlownSnail) + * Additional and updated documentation for examples, WebSockets, Getting Started (@Kriechi) + 27 December 2019: mitmproxy 5.0.1 * Fixed precompiled Linux binaries to not crash in table mode @@ -64,10 +87,10 @@ * Fix IPv6 scope suffixes in block addon (#3164) * Fix options update when added (#3157) * Fix "Edit Flow" button in mitmweb (#3136) - + 15 June 2018: mitmproxy 4.0.2 * Skipped! - + 17 May 2018: mitmproxy 4.0.1 @@ -1,6 +1,9 @@ -#!/bin/sh -set -e -set -x +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +set -o xtrace echo "Creating dev environment in ./venv..." @@ -12,4 +15,4 @@ pip3 install -r requirements.txt echo "" echo " * Created virtualenv environment in ./venv." echo " * Installed all dependencies into the virtualenv." -echo " * You can now activate the $(python3 --version) virtualenv with this command: \`. venv/bin/activate\`"
\ No newline at end of file +echo " * You can now activate the $(python3 --version) virtualenv with this command: \`. venv/bin/activate\`" diff --git a/docs/.gitignore b/docs/.gitignore index 1fb99949..610ffdf1 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,5 +1,6 @@ generated/ -src/public -node_modules -public +src/public/ +node_modules/ +public/ src/resources/_gen/ +src/content/addons-examples.md diff --git a/docs/README.md b/docs/README.md index 5c99fb39..24c24d24 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,7 +8,7 @@ This directory houses the mitmproxy documentation available at <https://docs.mit 2. Windows users: Depending on your git settings, you may need to manually create a symlink from /docs/src/examples to /examples. 3. Make sure the mitmproxy Python package is installed. - 4. Run `./build-current` to generate the documentation source files in `./src/generated`. + 4. Run `./build.sh` to generate additional documentation source files. Now you can run `hugo server -D` in ./src. diff --git a/docs/build-archive b/docs/build-archive deleted file mode 100755 index 004e625a..00000000 --- a/docs/build-archive +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -set -e - -DOCS_ARCHIVE=true ./build-current diff --git a/docs/build-current b/docs/build-current deleted file mode 100755 index 7164de6d..00000000 --- a/docs/build-current +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -for script in scripts/* ; do - echo "Generating output for $script ..." - output="${script##*/}" - "$script" > "src/generated/${output%.*}.html" -done - -cd src -hugo diff --git a/docs/build.sh b/docs/build.sh new file mode 100755 index 00000000..aaa52a2f --- /dev/null +++ b/docs/build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace + +SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +pushd ${SCRIPTPATH} + +for script in scripts/* ; do + output="${script##*/}" + output="src/generated/${output%.*}.html" + echo "Generating output for ${script} into ${output} ..." + "${script}" > "${output}" +done + +output="src/content/addons-examples.md" +echo "Generating examples content page into ${output} ..." +./render_examples.py > "${output}" + +cd src +hugo @@ -1,9 +1,13 @@ -#!/bin/bash -set -e +#!/usr/bin/env bash -# This script gets run from CI to render and upload docs +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace -./build-current +# This script gets run from CI to render and upload docs for the master branch. + +./build.sh # Only upload if we have defined credentials - we only have these defined for # trusted commits (i.e. not PRs). diff --git a/docs/render_examples.py b/docs/render_examples.py new file mode 100755 index 00000000..9c6dea74 --- /dev/null +++ b/docs/render_examples.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import os +import textwrap +from pathlib import Path + +print(""" +--- +title: "Examples" +menu: + addons: + weight: 6 +--- + +# Examples of Addons and Scripts + +The most recent set of examples is also available [on our GitHub project](https://github.com/mitmproxy/mitmproxy/tree/master/examples). + +""") + +base = os.path.dirname(os.path.realpath(__file__)) +examples_path = os.path.join(base, 'src/examples/') +pathlist = Path(examples_path).glob('**/*.py') + +examples = [os.path.relpath(str(p), examples_path) for p in sorted(pathlist)] +examples = [p for p in examples if not os.path.basename(p) == '__init__.py'] +examples = [p for p in examples if not os.path.basename(p).startswith('test_')] + +current_dir = None +current_level = 2 +for ex in examples: + if os.path.dirname(ex) != current_dir: + current_dir = os.path.dirname(ex) + sanitized = current_dir.replace('/', '').replace('.', '') + print(" * [Examples: {}]({{{{< relref \"addons-examples#{}\">}}}})".format(current_dir, sanitized)) + + sanitized = ex.replace('/', '').replace('.', '') + print(" * [{}]({{{{< relref \"addons-examples#example-{}\">}}}})".format(os.path.basename(ex), sanitized)) + +current_dir = None +current_level = 2 +for ex in examples: + if os.path.dirname(ex) != current_dir: + current_dir = os.path.dirname(ex) + print("#" * current_level, current_dir) + + print(textwrap.dedent(""" + {} Example: {} + {{{{< example src="{}" lang="py" >}}}} + """.format("#" * (current_level + 1), ex, "examples/" + ex))) diff --git a/docs/setup b/docs/setup.sh index cb63841a..da30a3c9 100755 --- a/docs/setup +++ b/docs/setup.sh @@ -1,5 +1,11 @@ -#!/bin/sh -set -e +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace + +# This is only needed once to provision a new fresh empty S3 bucket. aws configure set preview.cloudfront true aws --profile mitmproxy \ diff --git a/docs/src/assets/style.scss b/docs/src/assets/style.scss index 26c22071..33e8863e 100644 --- a/docs/src/assets/style.scss +++ b/docs/src/assets/style.scss @@ -47,6 +47,7 @@ body > div { width: 100%; text-align: right; } + max-width: 70vw; margin-bottom: 1em; } diff --git a/docs/src/content/concepts-certificates.md b/docs/src/content/concepts-certificates.md index 4e9aa652..20b03dc6 100644 --- a/docs/src/content/concepts-certificates.md +++ b/docs/src/content/concepts-certificates.md @@ -36,12 +36,12 @@ documentation for some common platforms. The mitmproxy CA cert is located in `~/.mitmproxy` after it has been generated at the first start of mitmproxy. - [IOS](http://jasdev.me/intercepting-ios-traffic) - On iOS 10.3 and onwards, you also need to enable full trust for the mitmproxy + On recent iOS versions you also need to enable full trust for the mitmproxy root certificate: 1. Go to Settings > General > About > Certificate Trust Settings. 2. Under "Enable full trust for root certificates", turn on trust for - the mitmproxy certificate. -- [IOS Simulator](https://github.com/ADVTOOLS/ADVTrustStore#how-to-use-advtruststore) + the mitmproxy certificate. +- [iOS Simulator](https://github.com/ADVTOOLS/ADVTrustStore#how-to-use-advtruststore) - [Java](https://docs.oracle.com/cd/E19906-01/820-4916/geygn/index.html) - [Android/Android Simulator](http://wiki.cacert.org/FAQ/ImportRootCert#Android_Phones_.26_Tablets) - [Windows](https://web.archive.org/web/20160612045445/http://windows.microsoft.com/en-ca/windows/import-export-certificates-private-keys#1TC=windows-7) diff --git a/docs/src/content/overview-getting-started.md b/docs/src/content/overview-getting-started.md new file mode 100644 index 00000000..ff018c3b --- /dev/null +++ b/docs/src/content/overview-getting-started.md @@ -0,0 +1,49 @@ +--- +title: "Getting Started" +menu: "overview" +menu: + overview: + weight: 3 +--- + +# Getting Started + +You have already [installed]({{< relref "overview-installation">}}) mitmproxy on +your machine. + +# Launch the tool you need + +You can start any of our three tools from the command line / terminal: + + * [mitmproxy]({{< relref "tools-mitmproxy">}}) -> gives you an interactive TUI + * [mitmdump]({{< relref "tools-mitmdump">}}) -> gives you a plain and simple terminal output + * [mitmweb]({{< relref "tools-mitmweb">}}) -> gives you a browser-based GUI + +When we talk about "mitmproxy" we usually refer to any of the three tools - they +are just different front-ends to the same core proxy. + +# Configure your browser or device + +For the basic setup as [regular proxy]({{< relref +"concepts-modes#regular-proxy">}}), you need to configure your browser or device +to route all web traffic through mitmproxy as HTTP proxy. Browser versions and +configurations options frequently change, so we recommend to simply search the +web on how to configure an HTTP proxy for your system. Some operating system +have a global settings, some browser have their own, other applications use +environment variables, etc. + +You can check that your web traffic is going through mitmproxy by browsing to +http://mitm.it - it should present you with a [simple page]({{< relref +"concepts-certificates/#quick-setup">}}) to install the mitmproxy Certificate +Authority - which is also the next steps. Follow the instructions for your OS / +system and install the CA (and make sure to enable it, some system require +multiple steps!). + +# Verifying everything works + +At this point your running mitmproxy instance should already show the first HTTP +flows from your client. You can test that all TLS-encrypted web traffic is +working as expected by browsing to https://mitmproxy.org - it should show up as +new flow and you can inspect it. + +Done. diff --git a/docs/src/content/overview-installation.md b/docs/src/content/overview-installation.md index 5b94adfc..1cdf62ad 100644 --- a/docs/src/content/overview-installation.md +++ b/docs/src/content/overview-installation.md @@ -34,20 +34,18 @@ the repository maintainers directly for issues with native packages. ## Windows - -All the mitmproxy tools are fully supported under -[WSL (Windows Subsystem for Linux)](https://docs.microsoft.com/en-us/windows/wsl/about). -We recommend to [install WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10), and then -follow the mitmproxy installation instructions for Linux. +All the mitmproxy tools are fully supported under [WSL (Windows Subsystem for +Linux)](https://docs.microsoft.com/en-us/windows/wsl/about). We recommend to +[install WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10), and +then follow the mitmproxy installation instructions for Linux. We also distribute native Windows packages for all tools other than the -mitmproxy console app, which only works under WSL. To install mitmproxy on Windows, -download the installer from [mitmproxy.org](https://mitmproxy.org/). +mitmproxy console app, which only works under WSL. To install mitmproxy on +Windows, download the installer from [mitmproxy.org](https://mitmproxy.org/). After installation, you'll find shortcuts for mitmweb and mitmdump in the start menu. Both executables are added to your PATH and can be invoked from the command line. - # Advanced Installation ## Development Setup @@ -57,7 +55,6 @@ GitHub master branch, please see the our [README](https://github.com/mitmproxy/mitmproxy#installation) on GitHub. - ## Installation from the Python Package Index (PyPI) If your mitmproxy addons require the installation of additional Python packages, @@ -65,10 +62,10 @@ you can install mitmproxy from [PyPI](https://pypi.org/project/mitmproxy/). While there are plenty of options around[^1], we recommend the installation using pipx: -[^1]: If you are familiar with the Python ecosystem, you may know that there are a million ways to install Python - packages. Most of them (pip, virtualenv, pipenv, etc.) should just work, but we don't have the capacity to +[^1]: If you are familiar with the Python ecosystem, you may know that there are a million ways to install Python + packages. Most of them (pip, virtualenv, pipenv, etc.) should just work, but we don't have the capacity to provide support for it. - + 1. Install a recent version of Python (we require at least 3.6). 2. Install [pipx](https://pipxproject.github.io/pipx/). 3. `pipx install mitmproxy` @@ -82,8 +79,10 @@ You can use the official mitmproxy images from ## Security Considerations for Binary Packages -Our pre-compiled binary packages and Docker images include a self-contained Python 3 environment, a recent version of -OpenSSL that support ALPN and HTTP/2, and other dependencies that would otherwise be cumbersome to compile and install. +Our pre-compiled binary packages and Docker images include a self-contained +Python 3 environment, a recent version of OpenSSL that support ALPN and HTTP/2, +and other dependencies that would otherwise be cumbersome to compile and +install. Dependencies in the binary packages are frozen on release, and can't be updated in situ. This means that we necessarily capture any bugs or security issues that @@ -92,4 +91,4 @@ dependencies (though we may do so if we become aware of a really serious issue). If you use our binary packages, please make sure you update regularly to ensure that everything remains current. -As a general principle, mitmproxy does not "phone home" and consequently will not do any update checks.
\ No newline at end of file +As a general principle, mitmproxy does not "phone home" and consequently will not do any update checks. diff --git a/docs/src/layouts/shortcodes/example.html b/docs/src/layouts/shortcodes/example.html index d23cabb6..83a6075d 100644 --- a/docs/src/layouts/shortcodes/example.html +++ b/docs/src/layouts/shortcodes/example.html @@ -1,5 +1,4 @@ - <div class="example"> {{ highlight (trim (readFile (.Get "src")) "\n\r") (.Get "lang") "" }} <div class="path">{{ (.Get "src" )}}</div> -</div>
\ No newline at end of file +</div> diff --git a/docs/upload-archive b/docs/upload-archive.sh index 3aaeb9be..e35345e9 100755 --- a/docs/upload-archive +++ b/docs/upload-archive.sh @@ -1,5 +1,9 @@ -#!/bin/bash -set -e +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace if [[ $# -eq 0 ]] ; then echo "Please supply a version, e.g. 'v3'" diff --git a/docs/upload-stable b/docs/upload-stable.sh index 5aea7479..a2f20f01 100755 --- a/docs/upload-stable +++ b/docs/upload-stable.sh @@ -1,5 +1,9 @@ -#!/bin/bash -set -e +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace aws configure set preview.cloudfront true aws --profile mitmproxy \ diff --git a/examples/complex/block_dns_over_https.py b/examples/complex/block_dns_over_https.py index 479f0baa..5b0b24cf 100644 --- a/examples/complex/block_dns_over_https.py +++ b/examples/complex/block_dns_over_https.py @@ -31,36 +31,45 @@ default_blocklist: dict = { "dns.google.com" ], "ips": [ - "176.103.130.131", "176.103.130.130", "2a00:5a60::ad1:ff", "2a00:5a60::ad2:ff", "176.103.130.134", "176.103.130.132", - "2a00:5a60::bad2:ff", "2a00:5a60::bad1:ff", "8.8.4.4", "8.8.8.8", "2001:4860:4860::8888", "2001:4860:4860::8844", - "104.16.248.249", "104.16.249.249", "2606:4700::6810:f8f9", "2606:4700::6810:f9f9", "104.16.248.249", "104.16.249.249", - "2606:4700::6810:f9f9", "2606:4700::6810:f8f9", "104.18.2.55", "104.18.3.55", "2606:4700::6812:337", "2606:4700::6812:237", - "104.18.27.128", "104.18.26.128", "2606:4700::6812:1a80", "2606:4700::6812:1b80", "9.9.9.9", "149.112.112.112", "2620:fe::9", - "2620:fe::fe", "9.9.9.9", "149.112.112.9", "2620:fe::fe:9", "2620:fe::9", "9.9.9.10", "149.112.112.10", "2620:fe::10", - "2620:fe::fe:10", "9.9.9.11", "149.112.112.11", "2620:fe::fe:11", "2620:fe::11", "146.112.41.2", "2620:119:fc::2", - "146.112.41.3", "2620:119:fc::3", "185.228.168.168", "185.228.168.10", "96.113.151.148", "2001:558:fe21:6b:96:113:151:149", - "174.68.248.77", "185.43.135.1", "2001:148f:fffe::1", "185.235.81.1", "2a0d:4d00:81::1", "45.90.28.0", "2a07:a8c0::", - "104.236.178.232", "2604:a880:1:20::51:f001", "104.28.1.106", "104.28.0.106", "2606:4700:3036::681c:6a", - "2606:4700:3034::681c:16a", "136.144.215.158", "2a01:7c8:d002:1ef:5054:ff:fe40:3703", "95.216.212.177", - "2a01:4f9:c010:43ce::1", "45.32.55.94", "2001:19f0:7001:3259:5400:2ff:fe71:bc9", "159.69.198.101", "2a01:4f8:1c1c:6b4b::1", - "195.30.94.28", "2001:608:a01::3", "104.24.122.53", "104.24.123.53", "2606:4700:3033::6818:7b35", "2606:4700:3035::6818:7a35", - "146.185.167.43", "2a03:b0c0:0:1010::e9a:3001", "115.159.131.230", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", - "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", - "139.99.222.72", "45.76.113.31", "104.182.57.196", "168.235.81.167", "2604:180:f3::42", "176.56.236.175", "2a00:d880:5:bf0::7c93", - "94.130.106.88", "2a03:4000:38:53c::2", "139.59.48.222", "174.138.29.175", "2400:6180:0:d0::5f73:4001", "104.18.45.204", - "104.18.44.204", "2606:4700:3033::6812:2dcc", "2606:4700:3033::6812:2ccc", "104.31.91.138", "104.31.90.138", - "2606:4700:3035::681f:5a8a", "2606:4700:3036::681f:5b8a", "185.134.196.54", "46.227.200.55", "46.227.200.54", "185.134.197.54", - "2a01:9e00::54", "2a01:9e01::54", "2a01:9e00::55", "2a01:9e01::55", "46.101.66.244", "172.104.93.80", - "2400:8902::f03c:91ff:feda:c514", "104.18.44.204", "104.18.45.204", "2606:4700:3033::6812:2ccc", "2606:4700:3033::6812:2dcc", - "185.216.27.142", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", - "217.169.20.22", "217.169.20.23", "2001:8b0::2022", "2001:8b0::2023", "172.65.3.223", "2606:4700:60:0:a71e:6467:cef8:2a56", - "83.77.85.7", "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", "178.62.214.105", "35.198.2.76", "210.17.9.228", - "2001:c50:ffff:1:101:101:101:101", "35.231.247.227", "185.95.218.43", "185.95.218.42", "2a05:fc84::43", "2a05:fc84::42", - "116.203.115.192", "116.202.176.26", "2a01:4f8:c2c:52bf::1", "88.198.91.187", "2a01:4f8:1c0c:8233::1", "95.216.181.228", - "2a01:4f9:c01f:4::abcd", "45.67.219.208", "2a04:bdc7:100:70::abcd", "185.213.26.187", "2a0d:5600:33:3::abcd", "46.239.223.80", - "2001:678:888:69:c45d:2738:c3f2:1878", "149.112.121.10", "149.112.122.10", "2620:10a:80bb::10", "2620:10a:80bc::10", - "149.112.121.20", "149.112.122.20", "2620:10a:80bb::20", "2620:10a:80bc::20", "149.112.121.30", "149.112.122.30", - "2620:10a:80bc::30", "2620:10a:80bb::30" + "104.16.248.249", "104.16.248.249", "104.16.249.249", "104.16.249.249", "104.18.2.55", + "104.18.26.128", "104.18.27.128", "104.18.3.55", "104.18.44.204", "104.18.44.204", + "104.18.45.204", "104.18.45.204", "104.182.57.196", "104.236.178.232", "104.24.122.53", + "104.24.123.53", "104.28.0.106", "104.28.1.106", "104.31.90.138", "104.31.91.138", + "115.159.131.230", "116.202.176.26", "116.203.115.192", "136.144.215.158", "139.59.48.222", + "139.99.222.72", "146.112.41.2", "146.112.41.3", "146.185.167.43", "149.112.112.10", + "149.112.112.11", "149.112.112.112", "149.112.112.9", "149.112.121.10", "149.112.121.20", + "149.112.121.30", "149.112.122.10", "149.112.122.20", "149.112.122.30", "159.69.198.101", + "168.235.81.167", "172.104.93.80", "172.65.3.223", "174.138.29.175", "174.68.248.77", + "176.103.130.130", "176.103.130.131", "176.103.130.132", "176.103.130.134", "176.56.236.175", + "178.62.214.105", "185.134.196.54", "185.134.197.54", "185.213.26.187", "185.216.27.142", + "185.228.168.10", "185.228.168.168", "185.235.81.1", "185.26.126.37", "185.26.126.37", + "185.43.135.1", "185.95.218.42", "185.95.218.43", "195.30.94.28", "2001:148f:fffe::1", + "2001:19f0:7001:3259:5400:2ff:fe71:bc9", "2001:19f0:7001:5554:5400:2ff:fe57:3077", + "2001:19f0:7001:5554:5400:2ff:fe57:3077", "2001:19f0:7001:5554:5400:2ff:fe57:3077", + "2001:4860:4860::8844", "2001:4860:4860::8888", + "2001:4b98:dc2:43:216:3eff:fe86:1d28", "2001:558:fe21:6b:96:113:151:149", + "2001:608:a01::3", "2001:678:888:69:c45d:2738:c3f2:1878", "2001:8b0::2022", "2001:8b0::2023", + "2001:c50:ffff:1:101:101:101:101", "210.17.9.228", "217.169.20.22", "217.169.20.23", + "2400:6180:0:d0::5f73:4001", "2400:8902::f03c:91ff:feda:c514", "2604:180:f3::42", + "2604:a880:1:20::51:f001", "2606:4700::6810:f8f9", "2606:4700::6810:f9f9", "2606:4700::6812:1a80", + "2606:4700::6812:1b80", "2606:4700::6812:237", "2606:4700::6812:337", "2606:4700:3033::6812:2ccc", + "2606:4700:3033::6812:2dcc", "2606:4700:3033::6818:7b35", "2606:4700:3034::681c:16a", + "2606:4700:3035::6818:7a35", "2606:4700:3035::681f:5a8a", "2606:4700:3036::681c:6a", + "2606:4700:3036::681f:5b8a", "2606:4700:60:0:a71e:6467:cef8:2a56", "2620:10a:80bb::10", + "2620:10a:80bb::20", "2620:10a:80bb::30" "2620:10a:80bc::10", "2620:10a:80bc::20", + "2620:10a:80bc::30", "2620:119:fc::2", "2620:119:fc::3", "2620:fe::10", "2620:fe::11", + "2620:fe::9", "2620:fe::fe:10", "2620:fe::fe:11", "2620:fe::fe:9", "2620:fe::fe", + "2a00:5a60::ad1:ff", "2a00:5a60::ad2:ff", "2a00:5a60::bad1:ff", "2a00:5a60::bad2:ff", + "2a00:d880:5:bf0::7c93", "2a01:4f8:1c0c:8233::1", "2a01:4f8:1c1c:6b4b::1", "2a01:4f8:c2c:52bf::1", + "2a01:4f9:c010:43ce::1", "2a01:4f9:c01f:4::abcd", "2a01:7c8:d002:1ef:5054:ff:fe40:3703", + "2a01:9e00::54", "2a01:9e00::55", "2a01:9e01::54", "2a01:9e01::55", + "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", "2a03:4000:38:53c::2", + "2a03:b0c0:0:1010::e9a:3001", "2a04:bdc7:100:70::abcd", "2a05:fc84::42", "2a05:fc84::43", + "2a07:a8c0::", "2a0d:4d00:81::1", "2a0d:5600:33:3::abcd", "35.198.2.76", "35.231.247.227", + "45.32.55.94", "45.67.219.208", "45.76.113.31", "45.77.180.10", "45.90.28.0", + "46.101.66.244", "46.227.200.54", "46.227.200.55", "46.239.223.80", "8.8.4.4", + "8.8.8.8", "83.77.85.7", "88.198.91.187", "9.9.9.10", "9.9.9.11", "9.9.9.9", + "94.130.106.88", "95.216.181.228", "95.216.212.177", "96.113.151.148", ] } diff --git a/examples/complex/remote_debug.py b/examples/complex/remote_debug.py index 4b117bdb..5129c9db 100644 --- a/examples/complex/remote_debug.py +++ b/examples/complex/remote_debug.py @@ -4,9 +4,11 @@ For general debugging purposes, it is easier to just debug mitmdump within PyCha Usage: - pip install pydevd on the mitmproxy machine - - Open the Run/Debug Configuration dialog box in PyCharm, and select the Python Remote Debug configuration type. - - Debugging works in the way that mitmproxy connects to the debug server on startup. - Specify host and port that mitmproxy can use to reach your PyCharm instance on startup. + - Open the Run/Debug Configuration dialog box in PyCharm, and select the + Python Remote Debug configuration type. + - Debugging works in the way that mitmproxy connects to the debug server + on startup. Specify host and port that mitmproxy can use to reach your + PyCharm instance on startup. - Adjust this inline script accordingly. - Start debug server in PyCharm - Set breakpoints diff --git a/examples/complex/sslstrip.py b/examples/complex/sslstrip.py index 8b904216..16d9b59a 100644 --- a/examples/complex/sslstrip.py +++ b/examples/complex/sslstrip.py @@ -51,9 +51,11 @@ def response(flow: http.HTTPFlow) -> None: flow.response.headers['Location'] = location.replace('https://', 'http://', 1) # strip upgrade-insecure-requests in Content-Security-Policy header - if re.search('upgrade-insecure-requests', flow.response.headers.get('Content-Security-Policy', ''), flags=re.IGNORECASE): + csp_header = flow.response.headers.get('Content-Security-Policy', '') + if re.search('upgrade-insecure-requests', csp_header, flags=re.IGNORECASE): csp = flow.response.headers['Content-Security-Policy'] - flow.response.headers['Content-Security-Policy'] = re.sub(r'upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) + new_header = re.sub(r'upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) + flow.response.headers['Content-Security-Policy'] = new_header # strip secure flag from 'Set-Cookie' headers cookies = flow.response.headers.get_all('Set-Cookie') diff --git a/mitmproxy/addons/onboardingapp/templates/index.html b/mitmproxy/addons/onboardingapp/templates/index.html index aee6858c..822e2856 100644 --- a/mitmproxy/addons/onboardingapp/templates/index.html +++ b/mitmproxy/addons/onboardingapp/templates/index.html @@ -20,6 +20,14 @@ function changeTo(device) { </ul> </div> <div class="col-md-4"> + <h3 class="text-center">How to install on iOS 13+</h3> + <ul> + <li>Install and active the new Profile</li> + <li>Goto Settings -> General -> About -> Certificate Trust Settings</li> + <li>Toggle mitmproxy to ON</li> + <li>Done!</li> + </div> + <div class="col-md-4"> <h3 class="text-center">How to install on browsers</h3> <ul> <li>Safari on macOS uses the macOS keychain. So installing our CA in the system is enough.</li> @@ -27,15 +35,6 @@ function changeTo(device) { <li>Firefox on macOS has its own CA store and needs to be installed with Firefox-specific instructions that can be found <a href="https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox">HERE</a> .</li> </ul> </div> - <div class="col-md-4"> - <h3 class="text-center">How to install on iOS v10.3</h3> - <ul> - <li>After certificate installation, open Settings</li> - <li>Navigate to General and then About</li> - <li>Select Certificate Trust Settings</li> - <li>Each root that has been installed via a profile will be listed below the heading Enable Full Trust For Root Certificates. Toggle mitmproxy on</li> - <li>Done!</li> - </div> </div> </div>`; } diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index d8e943d3..4c0f1d6b 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -297,7 +297,7 @@ def create_client_context( if cert: try: context.use_privatekey_file(cert) - context.use_certificate_file(cert) + context.use_certificate_chain_file(cert) except SSL.Error as v: raise exceptions.TlsException("SSL client certificate error: %s" % str(v)) return context diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py index c1dd6179..23eb39f0 100644 --- a/mitmproxy/tools/_main.py +++ b/mitmproxy/tools/_main.py @@ -110,6 +110,8 @@ def run( master.commands.dump() sys.exit(0) if extra: + if(args.filter_args): + master.log.info(f"Only processing flows that match \"{' & '.join(args.filter_args)}\"") opts.update(**extra(args)) loop = asyncio.get_event_loop() diff --git a/mitmproxy/tools/console/defaultkeys.py b/mitmproxy/tools/console/defaultkeys.py index a0f27625..a27fd85c 100644 --- a/mitmproxy/tools/console/defaultkeys.py +++ b/mitmproxy/tools/console/defaultkeys.py @@ -1,14 +1,14 @@ def map(km): - km.add(":", "console.command ", ["global"], "Command prompt") + km.add(":", "console.command ", ["commonkey", "global"], "Command prompt") km.add("?", "console.view.help", ["global"], "View help") km.add("B", "browser.start", ["global"], "Start an attached browser") km.add("C", "console.view.commands", ["global"], "View commands") km.add("K", "console.view.keybindings", ["global"], "View key bindings") - km.add("O", "console.view.options", ["global"], "View options") - km.add("E", "console.view.eventlog", ["global"], "View event log") + km.add("O", "console.view.options", ["commonkey", "global"], "View options") + km.add("E", "console.view.eventlog", ["commonkey", "global"], "View event log") km.add("Q", "console.exit", ["global"], "Exit immediately") - km.add("q", "console.view.pop", ["global"], "Exit the current view") + km.add("q", "console.view.pop", ["commonkey", "global"], "Exit the current view") km.add("-", "console.layout.cycle", ["global"], "Cycle to next layout") km.add("shift tab", "console.panes.next", ["global"], "Focus next layout pane") km.add("ctrl right", "console.panes.next", ["global"], "Focus next layout pane") @@ -20,8 +20,8 @@ def map(km): km.add("j", "console.nav.down", ["global"], "Down") km.add("l", "console.nav.right", ["global"], "Right") km.add("h", "console.nav.left", ["global"], "Left") - km.add("tab", "console.nav.next", ["global"], "Next") - km.add("enter", "console.nav.select", ["global"], "Select") + km.add("tab", "console.nav.next", ["commonkey", "global"], "Next") + km.add("enter", "console.nav.select", ["commonkey", "global"], "Select") km.add("space", "console.nav.pagedown", ["global"], "Page down") km.add("ctrl f", "console.nav.pagedown", ["global"], "Page down") km.add("ctrl b", "console.nav.pageup", ["global"], "Page up") diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index c4dafee6..3fef70ce 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -170,7 +170,7 @@ class FlowDetails(tabs.Tabs): self.master.log.debug(error) # Give hint that you have to tab for the response. if description == "No content" and isinstance(message, http.HTTPRequest): - description = "No request content (press tab to view response)" + description = "No request content" # If the users has a wide terminal, he gets fewer lines; this should not be an issue. chars_per_line = 80 diff --git a/mitmproxy/tools/console/help.py b/mitmproxy/tools/console/help.py index fb4e0051..5a7bbb9a 100644 --- a/mitmproxy/tools/console/help.py +++ b/mitmproxy/tools/console/help.py @@ -51,10 +51,22 @@ class HelpView(tabs.Tabs, layoutwidget.LayoutWidget): text = [ urwid.Text( [ - ("title", "Keybindings for this view") + ("title", "Common Keybindings") ] ) + ] + + text.extend(self.format_keys(self.master.keymap.list("commonkey"))) + + text.append( + urwid.Text( + [ + "\n", + ("title", "Keybindings for this view") + ] + ) + ) if self.helpctx: text.extend(self.format_keys(self.master.keymap.list(self.helpctx))) diff --git a/mitmproxy/tools/console/keymap.py b/mitmproxy/tools/console/keymap.py index a42225cf..0a1c64ef 100644 --- a/mitmproxy/tools/console/keymap.py +++ b/mitmproxy/tools/console/keymap.py @@ -18,6 +18,7 @@ class KeyBindingError(Exception): Contexts = { "chooser", "commands", + "commonkey", "dataviewer", "eventlog", "flowlist", diff --git a/release/README.md b/release/README.md index 8632d644..fb245e23 100644 --- a/release/README.md +++ b/release/README.md @@ -27,20 +27,18 @@ These steps assume you are on the correct branch and have a git remote called `o - The Homebrew maintainers are typically very fast and detect our new relese within a day. - If you feel the need, you can run this from a macOS machine: - `brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/v<version number here>` + `brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/v<version number here>.tar.gz mitmproxy` ### Docker - The docker image is built by our CI workers and pushed to Docker Hub automatically. - Please verify that https://hub.docker.com/r/mitmproxy/mitmproxy/tags/ has the latest version. -- The latest and latest-ARMv7 tags should auto-update. @mhils introduced this after the 5.0.0 release. - Please verify that this is the case and remove this notice. For reference, this is how to do it manually: - `export VERSION=4.0.3 && docker pull mitmproxy/mitmproxy:$VERSION && docker tag mitmproxy/mitmproxy:$VERSION mitmproxy/mitmproxy:latest && docker push mitmproxy/mitmproxy:latest`. +- Please verify that the latest tag points to the most recent image (same digest / hash). ### Docs - - `./build-current`. If everything looks alright, continue with - - `./upload-stable`, - - `./build-archive`, and - - `./upload-archive v4`. Doing this now already saves you from switching back to an old state on the next release. + - `./build.sh`. If everything looks alright, continue with + - `./upload-stable.sh`, + - `DOCS_ARCHIVE=true ./build.sh`, and + - `./upload-archive.sh v4`. Doing this now already saves you from switching back to an old state on the next release. ### Website - Update version here: diff --git a/release/cibuild.py b/release/cibuild.py index d070a4b9..b00bdb5c 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -356,15 +356,17 @@ def build_docker_image(be: BuildEnviron): # pragma: no cover "--file", "release/docker/Dockerfile", "." ]) - subprocess.check_call([ + # smoke-test the newly built docker image + r = subprocess.run([ "docker", - "build", - "--tag", be.docker_tag + "-ARMv7", - "--build-arg", "WHEEL_MITMPROXY={}".format(whl), - "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(os.path.basename(whl)), - "--file", "release/docker/DockerfileARMv7", - "." - ]) + "run", + "--rm", + be.docker_tag, + "mitmdump", + "--version", + ], check=True, capture_output=True) + print(r.stdout.decode()) + assert "Mitmproxy: " in r.stdout.decode() def build_pyinstaller(be: BuildEnviron): # pragma: no cover @@ -569,11 +571,10 @@ def upload(): # pragma: no cover "-u", be.docker_username, "-p", be.docker_password, ]) - for variant in ["", "-ARMv7"]: - subprocess.check_call(["docker", "push", be.docker_tag + variant]) - if be.is_prod_release: - subprocess.check_call(["docker", "tag", be.docker_tag + variant, "mitmproxy/mitmproxy:latest" + variant]) - subprocess.check_call(["docker", "push", "mitmproxy/mitmproxy:latest" + variant]) + subprocess.check_call(["docker", "push", be.docker_tag]) + if be.is_prod_release: + subprocess.check_call(["docker", "tag", be.docker_tag, "mitmproxy/mitmproxy:latest"]) + subprocess.check_call(["docker", "push", "mitmproxy/mitmproxy:latest"]) if __name__ == "__main__": # pragma: no cover diff --git a/release/docker/Dockerfile b/release/docker/Dockerfile index 258bccf5..5f496e9f 100644 --- a/release/docker/Dockerfile +++ b/release/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.8 +FROM alpine:3.11 ENV LANG=en_US.UTF-8 diff --git a/release/docker/DockerfileARMv7 b/release/docker/DockerfileARMv7 deleted file mode 100644 index 40f10ede..00000000 --- a/release/docker/DockerfileARMv7 +++ /dev/null @@ -1,46 +0,0 @@ -FROM resin/raspberrypi3-alpine:3.7 - -ENV LANG=en_US.UTF-8 - -ARG WHEEL_MITMPROXY -ARG WHEEL_BASENAME_MITMPROXY - -COPY $WHEEL_MITMPROXY /home/mitmproxy/ - -RUN [ "cross-build-start" ] - -# Add our user first to make sure the ID get assigned consistently, -# regardless of whatever dependencies get added. -RUN addgroup -S mitmproxy && adduser -S -G mitmproxy mitmproxy \ - && apk add --no-cache \ - su-exec \ - git \ - g++ \ - libffi \ - libffi-dev \ - libstdc++ \ - openssl \ - openssl-dev \ - python3 \ - python3-dev \ - && python3 -m ensurepip --upgrade \ - && pip3 install -U pip \ - && LDFLAGS=-L/lib pip3 install -U /home/mitmproxy/${WHEEL_BASENAME_MITMPROXY} \ - && apk del --purge \ - git \ - g++ \ - libffi-dev \ - openssl-dev \ - python3-dev \ - && rm -rf ~/.cache/pip /home/mitmproxy/${WHEEL_BASENAME_MITMPROXY} - -RUN [ "cross-build-end" ] - -VOLUME /home/mitmproxy/.mitmproxy - -COPY release/docker/docker-entrypoint.sh /usr/local/bin/ -ENTRYPOINT ["docker-entrypoint.sh"] - -EXPOSE 8080 8081 - -CMD ["mitmproxy"] diff --git a/release/docker/README.md b/release/docker/README.md index 2fa93949..df9834b8 100644 --- a/release/docker/README.md +++ b/release/docker/README.md @@ -40,8 +40,7 @@ The available release tags can be seen * `master` always tracks the git-master branch and represents the unstable development tree. * `latest` always points to the same image as the most recent stable release, including bugfix releases (e.g., `4.0.0` and `4.0.1`). -* `X.Y.Z` tags contain the mitmproxy release with this version number. -* `*-ARMv7` are images built for Raspbian / Raspberry Pi systems. +* `X.Y.Z` tags contain the mitmproxy release with this version number. # Security Notice diff --git a/release/docker/docker-entrypoint.sh b/release/docker/docker-entrypoint.sh index a4abe4ce..84ea81e6 100755 --- a/release/docker/docker-entrypoint.sh +++ b/release/docker/docker-entrypoint.sh @@ -1,13 +1,17 @@ #!/bin/sh -set -e +# WARNING: do not change the shebang - the Docker base image might not have what you want! + +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace MITMPROXY_PATH="/home/mitmproxy/.mitmproxy" if [[ "$1" = "mitmdump" || "$1" = "mitmproxy" || "$1" = "mitmweb" ]]; then - mkdir -p "$MITMPROXY_PATH" - chown -R mitmproxy:mitmproxy "$MITMPROXY_PATH" - - su-exec mitmproxy "$@" + mkdir -p "$MITMPROXY_PATH" + chown -R mitmproxy:mitmproxy "$MITMPROXY_PATH" + su-exec mitmproxy "$@" else - exec "$@" + exec "$@" fi @@ -76,4 +76,4 @@ deps = awscli changedir = docs commands = - ./ci + ./ci.sh |