diff options
45 files changed, 957 insertions, 509 deletions
diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md index 8e8080db..fb75993f 100644 --- a/.github/ISSUE_TEMPLATE/feature_request.md +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -7,7 +7,7 @@ assignees: '' --- -**Is your feature request related to a problem? Please describe.** +#### Is your feature request related to a problem? Please describe. A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] #### Describe the solution you'd like @@ -1,3 +1,26 @@ +13 April 2020: mitmproxy 5.1.1 + * Fixed Docker images not starting due to missing shell + +13 April 2020: mitmproxy 5.1 + + ** Major Changes ** + * Initial Support for TLS 1.3 + + ** Full Changelog ** + * Reduce leaf certificate validity to one year due to upcoming browser changes (@mhils) + * Rename mitmweb's web_iface option to web_host for consistency (@oxr463) + * Sending a SIGTERM now exits mitmproxy without prompt, SIGINT still asks (@ThinkChaos) + * Don't force host header on outgoing requests (@mhils) + * Additional documentation and examples for WebSockets (@Kriechi) + * Gracefully handle hyphens in domain names (@matosconsulting) + * Fix header replacement count (@naivekun) + * Emit serverconnect event only after a connection has been established (@Prinzhorn) + * Fix ValueError in table mode of server replay flow (@ylmrx) + * HTTP/2: send all stream reset types to other connection (@rohfle) + * HTTP/2: fix WINDOW_UPDATE swallowed on closed streams (@Kriechi) + * Fix wrong behavior of --allow-hosts options (@BlownSnail) + * Additional and updated documentation for examples, WebSockets, Getting Started (@Kriechi) + 27 December 2019: mitmproxy 5.0.1 * Fixed precompiled Linux binaries to not crash in table mode @@ -64,10 +87,10 @@ * Fix IPv6 scope suffixes in block addon (#3164) * Fix options update when added (#3157) * Fix "Edit Flow" button in mitmweb (#3136) - + 15 June 2018: mitmproxy 4.0.2 * Skipped! - + 17 May 2018: mitmproxy 4.0.1 @@ -1,6 +1,9 @@ -#!/bin/sh -set -e -set -x +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +set -o xtrace echo "Creating dev environment in ./venv..." @@ -12,4 +15,4 @@ pip3 install -r requirements.txt echo "" echo " * Created virtualenv environment in ./venv." echo " * Installed all dependencies into the virtualenv." -echo " * You can now activate the $(python3 --version) virtualenv with this command: \`. venv/bin/activate\`"
\ No newline at end of file +echo " * You can now activate the $(python3 --version) virtualenv with this command: \`. venv/bin/activate\`" diff --git a/docs/.gitignore b/docs/.gitignore index 1fb99949..610ffdf1 100644 --- a/docs/.gitignore +++ b/docs/.gitignore @@ -1,5 +1,6 @@ generated/ -src/public -node_modules -public +src/public/ +node_modules/ +public/ src/resources/_gen/ +src/content/addons-examples.md diff --git a/docs/README.md b/docs/README.md index 5c99fb39..24c24d24 100644 --- a/docs/README.md +++ b/docs/README.md @@ -8,7 +8,7 @@ This directory houses the mitmproxy documentation available at <https://docs.mit 2. Windows users: Depending on your git settings, you may need to manually create a symlink from /docs/src/examples to /examples. 3. Make sure the mitmproxy Python package is installed. - 4. Run `./build-current` to generate the documentation source files in `./src/generated`. + 4. Run `./build.sh` to generate additional documentation source files. Now you can run `hugo server -D` in ./src. diff --git a/docs/build-archive b/docs/build-archive deleted file mode 100755 index 004e625a..00000000 --- a/docs/build-archive +++ /dev/null @@ -1,4 +0,0 @@ -#!/bin/sh -set -e - -DOCS_ARCHIVE=true ./build-current diff --git a/docs/build-current b/docs/build-current deleted file mode 100755 index 7164de6d..00000000 --- a/docs/build-current +++ /dev/null @@ -1,12 +0,0 @@ -#!/usr/bin/env bash - -set -e - -for script in scripts/* ; do - echo "Generating output for $script ..." - output="${script##*/}" - "$script" > "src/generated/${output%.*}.html" -done - -cd src -hugo diff --git a/docs/build.sh b/docs/build.sh new file mode 100755 index 00000000..aaa52a2f --- /dev/null +++ b/docs/build.sh @@ -0,0 +1,23 @@ +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace + +SCRIPTPATH="$( cd "$(dirname "$0")" >/dev/null 2>&1 ; pwd -P )" +pushd ${SCRIPTPATH} + +for script in scripts/* ; do + output="${script##*/}" + output="src/generated/${output%.*}.html" + echo "Generating output for ${script} into ${output} ..." + "${script}" > "${output}" +done + +output="src/content/addons-examples.md" +echo "Generating examples content page into ${output} ..." +./render_examples.py > "${output}" + +cd src +hugo @@ -1,9 +1,13 @@ -#!/bin/bash -set -e +#!/usr/bin/env bash -# This script gets run from CI to render and upload docs +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace -./build-current +# This script gets run from CI to render and upload docs for the master branch. + +./build.sh # Only upload if we have defined credentials - we only have these defined for # trusted commits (i.e. not PRs). diff --git a/docs/render_examples.py b/docs/render_examples.py new file mode 100755 index 00000000..9c6dea74 --- /dev/null +++ b/docs/render_examples.py @@ -0,0 +1,50 @@ +#!/usr/bin/env python3 + +import os +import textwrap +from pathlib import Path + +print(""" +--- +title: "Examples" +menu: + addons: + weight: 6 +--- + +# Examples of Addons and Scripts + +The most recent set of examples is also available [on our GitHub project](https://github.com/mitmproxy/mitmproxy/tree/master/examples). + +""") + +base = os.path.dirname(os.path.realpath(__file__)) +examples_path = os.path.join(base, 'src/examples/') +pathlist = Path(examples_path).glob('**/*.py') + +examples = [os.path.relpath(str(p), examples_path) for p in sorted(pathlist)] +examples = [p for p in examples if not os.path.basename(p) == '__init__.py'] +examples = [p for p in examples if not os.path.basename(p).startswith('test_')] + +current_dir = None +current_level = 2 +for ex in examples: + if os.path.dirname(ex) != current_dir: + current_dir = os.path.dirname(ex) + sanitized = current_dir.replace('/', '').replace('.', '') + print(" * [Examples: {}]({{{{< relref \"addons-examples#{}\">}}}})".format(current_dir, sanitized)) + + sanitized = ex.replace('/', '').replace('.', '') + print(" * [{}]({{{{< relref \"addons-examples#example-{}\">}}}})".format(os.path.basename(ex), sanitized)) + +current_dir = None +current_level = 2 +for ex in examples: + if os.path.dirname(ex) != current_dir: + current_dir = os.path.dirname(ex) + print("#" * current_level, current_dir) + + print(textwrap.dedent(""" + {} Example: {} + {{{{< example src="{}" lang="py" >}}}} + """.format("#" * (current_level + 1), ex, "examples/" + ex))) diff --git a/docs/setup b/docs/setup.sh index cb63841a..da30a3c9 100755 --- a/docs/setup +++ b/docs/setup.sh @@ -1,5 +1,11 @@ -#!/bin/sh -set -e +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace + +# This is only needed once to provision a new fresh empty S3 bucket. aws configure set preview.cloudfront true aws --profile mitmproxy \ diff --git a/docs/src/assets/style.scss b/docs/src/assets/style.scss index 26c22071..33e8863e 100644 --- a/docs/src/assets/style.scss +++ b/docs/src/assets/style.scss @@ -47,6 +47,7 @@ body > div { width: 100%; text-align: right; } + max-width: 70vw; margin-bottom: 1em; } diff --git a/docs/src/content/concepts-certificates.md b/docs/src/content/concepts-certificates.md index 4e9aa652..20b03dc6 100644 --- a/docs/src/content/concepts-certificates.md +++ b/docs/src/content/concepts-certificates.md @@ -36,12 +36,12 @@ documentation for some common platforms. The mitmproxy CA cert is located in `~/.mitmproxy` after it has been generated at the first start of mitmproxy. - [IOS](http://jasdev.me/intercepting-ios-traffic) - On iOS 10.3 and onwards, you also need to enable full trust for the mitmproxy + On recent iOS versions you also need to enable full trust for the mitmproxy root certificate: 1. Go to Settings > General > About > Certificate Trust Settings. 2. Under "Enable full trust for root certificates", turn on trust for - the mitmproxy certificate. -- [IOS Simulator](https://github.com/ADVTOOLS/ADVTrustStore#how-to-use-advtruststore) + the mitmproxy certificate. +- [iOS Simulator](https://github.com/ADVTOOLS/ADVTrustStore#how-to-use-advtruststore) - [Java](https://docs.oracle.com/cd/E19906-01/820-4916/geygn/index.html) - [Android/Android Simulator](http://wiki.cacert.org/FAQ/ImportRootCert#Android_Phones_.26_Tablets) - [Windows](https://web.archive.org/web/20160612045445/http://windows.microsoft.com/en-ca/windows/import-export-certificates-private-keys#1TC=windows-7) diff --git a/docs/src/content/overview-getting-started.md b/docs/src/content/overview-getting-started.md new file mode 100644 index 00000000..ff018c3b --- /dev/null +++ b/docs/src/content/overview-getting-started.md @@ -0,0 +1,49 @@ +--- +title: "Getting Started" +menu: "overview" +menu: + overview: + weight: 3 +--- + +# Getting Started + +You have already [installed]({{< relref "overview-installation">}}) mitmproxy on +your machine. + +# Launch the tool you need + +You can start any of our three tools from the command line / terminal: + + * [mitmproxy]({{< relref "tools-mitmproxy">}}) -> gives you an interactive TUI + * [mitmdump]({{< relref "tools-mitmdump">}}) -> gives you a plain and simple terminal output + * [mitmweb]({{< relref "tools-mitmweb">}}) -> gives you a browser-based GUI + +When we talk about "mitmproxy" we usually refer to any of the three tools - they +are just different front-ends to the same core proxy. + +# Configure your browser or device + +For the basic setup as [regular proxy]({{< relref +"concepts-modes#regular-proxy">}}), you need to configure your browser or device +to route all web traffic through mitmproxy as HTTP proxy. Browser versions and +configurations options frequently change, so we recommend to simply search the +web on how to configure an HTTP proxy for your system. Some operating system +have a global settings, some browser have their own, other applications use +environment variables, etc. + +You can check that your web traffic is going through mitmproxy by browsing to +http://mitm.it - it should present you with a [simple page]({{< relref +"concepts-certificates/#quick-setup">}}) to install the mitmproxy Certificate +Authority - which is also the next steps. Follow the instructions for your OS / +system and install the CA (and make sure to enable it, some system require +multiple steps!). + +# Verifying everything works + +At this point your running mitmproxy instance should already show the first HTTP +flows from your client. You can test that all TLS-encrypted web traffic is +working as expected by browsing to https://mitmproxy.org - it should show up as +new flow and you can inspect it. + +Done. diff --git a/docs/src/content/overview-installation.md b/docs/src/content/overview-installation.md index 5b94adfc..1cdf62ad 100644 --- a/docs/src/content/overview-installation.md +++ b/docs/src/content/overview-installation.md @@ -34,20 +34,18 @@ the repository maintainers directly for issues with native packages. ## Windows - -All the mitmproxy tools are fully supported under -[WSL (Windows Subsystem for Linux)](https://docs.microsoft.com/en-us/windows/wsl/about). -We recommend to [install WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10), and then -follow the mitmproxy installation instructions for Linux. +All the mitmproxy tools are fully supported under [WSL (Windows Subsystem for +Linux)](https://docs.microsoft.com/en-us/windows/wsl/about). We recommend to +[install WSL](https://docs.microsoft.com/en-us/windows/wsl/install-win10), and +then follow the mitmproxy installation instructions for Linux. We also distribute native Windows packages for all tools other than the -mitmproxy console app, which only works under WSL. To install mitmproxy on Windows, -download the installer from [mitmproxy.org](https://mitmproxy.org/). +mitmproxy console app, which only works under WSL. To install mitmproxy on +Windows, download the installer from [mitmproxy.org](https://mitmproxy.org/). After installation, you'll find shortcuts for mitmweb and mitmdump in the start menu. Both executables are added to your PATH and can be invoked from the command line. - # Advanced Installation ## Development Setup @@ -57,7 +55,6 @@ GitHub master branch, please see the our [README](https://github.com/mitmproxy/mitmproxy#installation) on GitHub. - ## Installation from the Python Package Index (PyPI) If your mitmproxy addons require the installation of additional Python packages, @@ -65,10 +62,10 @@ you can install mitmproxy from [PyPI](https://pypi.org/project/mitmproxy/). While there are plenty of options around[^1], we recommend the installation using pipx: -[^1]: If you are familiar with the Python ecosystem, you may know that there are a million ways to install Python - packages. Most of them (pip, virtualenv, pipenv, etc.) should just work, but we don't have the capacity to +[^1]: If you are familiar with the Python ecosystem, you may know that there are a million ways to install Python + packages. Most of them (pip, virtualenv, pipenv, etc.) should just work, but we don't have the capacity to provide support for it. - + 1. Install a recent version of Python (we require at least 3.6). 2. Install [pipx](https://pipxproject.github.io/pipx/). 3. `pipx install mitmproxy` @@ -82,8 +79,10 @@ You can use the official mitmproxy images from ## Security Considerations for Binary Packages -Our pre-compiled binary packages and Docker images include a self-contained Python 3 environment, a recent version of -OpenSSL that support ALPN and HTTP/2, and other dependencies that would otherwise be cumbersome to compile and install. +Our pre-compiled binary packages and Docker images include a self-contained +Python 3 environment, a recent version of OpenSSL that support ALPN and HTTP/2, +and other dependencies that would otherwise be cumbersome to compile and +install. Dependencies in the binary packages are frozen on release, and can't be updated in situ. This means that we necessarily capture any bugs or security issues that @@ -92,4 +91,4 @@ dependencies (though we may do so if we become aware of a really serious issue). If you use our binary packages, please make sure you update regularly to ensure that everything remains current. -As a general principle, mitmproxy does not "phone home" and consequently will not do any update checks.
\ No newline at end of file +As a general principle, mitmproxy does not "phone home" and consequently will not do any update checks. diff --git a/docs/src/layouts/shortcodes/example.html b/docs/src/layouts/shortcodes/example.html index d23cabb6..83a6075d 100644 --- a/docs/src/layouts/shortcodes/example.html +++ b/docs/src/layouts/shortcodes/example.html @@ -1,5 +1,4 @@ - <div class="example"> {{ highlight (trim (readFile (.Get "src")) "\n\r") (.Get "lang") "" }} <div class="path">{{ (.Get "src" )}}</div> -</div>
\ No newline at end of file +</div> diff --git a/docs/upload-archive b/docs/upload-archive.sh index 3aaeb9be..e35345e9 100755 --- a/docs/upload-archive +++ b/docs/upload-archive.sh @@ -1,5 +1,9 @@ -#!/bin/bash -set -e +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace if [[ $# -eq 0 ]] ; then echo "Please supply a version, e.g. 'v3'" diff --git a/docs/upload-stable b/docs/upload-stable.sh index 5aea7479..a2f20f01 100755 --- a/docs/upload-stable +++ b/docs/upload-stable.sh @@ -1,5 +1,9 @@ -#!/bin/bash -set -e +#!/usr/bin/env bash + +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace aws configure set preview.cloudfront true aws --profile mitmproxy \ diff --git a/examples/complex/block_dns_over_https.py b/examples/complex/block_dns_over_https.py index 479f0baa..5b0b24cf 100644 --- a/examples/complex/block_dns_over_https.py +++ b/examples/complex/block_dns_over_https.py @@ -31,36 +31,45 @@ default_blocklist: dict = { "dns.google.com" ], "ips": [ - "176.103.130.131", "176.103.130.130", "2a00:5a60::ad1:ff", "2a00:5a60::ad2:ff", "176.103.130.134", "176.103.130.132", - "2a00:5a60::bad2:ff", "2a00:5a60::bad1:ff", "8.8.4.4", "8.8.8.8", "2001:4860:4860::8888", "2001:4860:4860::8844", - "104.16.248.249", "104.16.249.249", "2606:4700::6810:f8f9", "2606:4700::6810:f9f9", "104.16.248.249", "104.16.249.249", - "2606:4700::6810:f9f9", "2606:4700::6810:f8f9", "104.18.2.55", "104.18.3.55", "2606:4700::6812:337", "2606:4700::6812:237", - "104.18.27.128", "104.18.26.128", "2606:4700::6812:1a80", "2606:4700::6812:1b80", "9.9.9.9", "149.112.112.112", "2620:fe::9", - "2620:fe::fe", "9.9.9.9", "149.112.112.9", "2620:fe::fe:9", "2620:fe::9", "9.9.9.10", "149.112.112.10", "2620:fe::10", - "2620:fe::fe:10", "9.9.9.11", "149.112.112.11", "2620:fe::fe:11", "2620:fe::11", "146.112.41.2", "2620:119:fc::2", - "146.112.41.3", "2620:119:fc::3", "185.228.168.168", "185.228.168.10", "96.113.151.148", "2001:558:fe21:6b:96:113:151:149", - "174.68.248.77", "185.43.135.1", "2001:148f:fffe::1", "185.235.81.1", "2a0d:4d00:81::1", "45.90.28.0", "2a07:a8c0::", - "104.236.178.232", "2604:a880:1:20::51:f001", "104.28.1.106", "104.28.0.106", "2606:4700:3036::681c:6a", - "2606:4700:3034::681c:16a", "136.144.215.158", "2a01:7c8:d002:1ef:5054:ff:fe40:3703", "95.216.212.177", - "2a01:4f9:c010:43ce::1", "45.32.55.94", "2001:19f0:7001:3259:5400:2ff:fe71:bc9", "159.69.198.101", "2a01:4f8:1c1c:6b4b::1", - "195.30.94.28", "2001:608:a01::3", "104.24.122.53", "104.24.123.53", "2606:4700:3033::6818:7b35", "2606:4700:3035::6818:7a35", - "146.185.167.43", "2a03:b0c0:0:1010::e9a:3001", "115.159.131.230", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", - "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", "45.77.180.10", "2001:19f0:7001:5554:5400:2ff:fe57:3077", - "139.99.222.72", "45.76.113.31", "104.182.57.196", "168.235.81.167", "2604:180:f3::42", "176.56.236.175", "2a00:d880:5:bf0::7c93", - "94.130.106.88", "2a03:4000:38:53c::2", "139.59.48.222", "174.138.29.175", "2400:6180:0:d0::5f73:4001", "104.18.45.204", - "104.18.44.204", "2606:4700:3033::6812:2dcc", "2606:4700:3033::6812:2ccc", "104.31.91.138", "104.31.90.138", - "2606:4700:3035::681f:5a8a", "2606:4700:3036::681f:5b8a", "185.134.196.54", "46.227.200.55", "46.227.200.54", "185.134.197.54", - "2a01:9e00::54", "2a01:9e01::54", "2a01:9e00::55", "2a01:9e01::55", "46.101.66.244", "172.104.93.80", - "2400:8902::f03c:91ff:feda:c514", "104.18.44.204", "104.18.45.204", "2606:4700:3033::6812:2ccc", "2606:4700:3033::6812:2dcc", - "185.216.27.142", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", "185.26.126.37", "2001:4b98:dc2:43:216:3eff:fe86:1d28", - "217.169.20.22", "217.169.20.23", "2001:8b0::2022", "2001:8b0::2023", "172.65.3.223", "2606:4700:60:0:a71e:6467:cef8:2a56", - "83.77.85.7", "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", "178.62.214.105", "35.198.2.76", "210.17.9.228", - "2001:c50:ffff:1:101:101:101:101", "35.231.247.227", "185.95.218.43", "185.95.218.42", "2a05:fc84::43", "2a05:fc84::42", - "116.203.115.192", "116.202.176.26", "2a01:4f8:c2c:52bf::1", "88.198.91.187", "2a01:4f8:1c0c:8233::1", "95.216.181.228", - "2a01:4f9:c01f:4::abcd", "45.67.219.208", "2a04:bdc7:100:70::abcd", "185.213.26.187", "2a0d:5600:33:3::abcd", "46.239.223.80", - "2001:678:888:69:c45d:2738:c3f2:1878", "149.112.121.10", "149.112.122.10", "2620:10a:80bb::10", "2620:10a:80bc::10", - "149.112.121.20", "149.112.122.20", "2620:10a:80bb::20", "2620:10a:80bc::20", "149.112.121.30", "149.112.122.30", - "2620:10a:80bc::30", "2620:10a:80bb::30" + "104.16.248.249", "104.16.248.249", "104.16.249.249", "104.16.249.249", "104.18.2.55", + "104.18.26.128", "104.18.27.128", "104.18.3.55", "104.18.44.204", "104.18.44.204", + "104.18.45.204", "104.18.45.204", "104.182.57.196", "104.236.178.232", "104.24.122.53", + "104.24.123.53", "104.28.0.106", "104.28.1.106", "104.31.90.138", "104.31.91.138", + "115.159.131.230", "116.202.176.26", "116.203.115.192", "136.144.215.158", "139.59.48.222", + "139.99.222.72", "146.112.41.2", "146.112.41.3", "146.185.167.43", "149.112.112.10", + "149.112.112.11", "149.112.112.112", "149.112.112.9", "149.112.121.10", "149.112.121.20", + "149.112.121.30", "149.112.122.10", "149.112.122.20", "149.112.122.30", "159.69.198.101", + "168.235.81.167", "172.104.93.80", "172.65.3.223", "174.138.29.175", "174.68.248.77", + "176.103.130.130", "176.103.130.131", "176.103.130.132", "176.103.130.134", "176.56.236.175", + "178.62.214.105", "185.134.196.54", "185.134.197.54", "185.213.26.187", "185.216.27.142", + "185.228.168.10", "185.228.168.168", "185.235.81.1", "185.26.126.37", "185.26.126.37", + "185.43.135.1", "185.95.218.42", "185.95.218.43", "195.30.94.28", "2001:148f:fffe::1", + "2001:19f0:7001:3259:5400:2ff:fe71:bc9", "2001:19f0:7001:5554:5400:2ff:fe57:3077", + "2001:19f0:7001:5554:5400:2ff:fe57:3077", "2001:19f0:7001:5554:5400:2ff:fe57:3077", + "2001:4860:4860::8844", "2001:4860:4860::8888", + "2001:4b98:dc2:43:216:3eff:fe86:1d28", "2001:558:fe21:6b:96:113:151:149", + "2001:608:a01::3", "2001:678:888:69:c45d:2738:c3f2:1878", "2001:8b0::2022", "2001:8b0::2023", + "2001:c50:ffff:1:101:101:101:101", "210.17.9.228", "217.169.20.22", "217.169.20.23", + "2400:6180:0:d0::5f73:4001", "2400:8902::f03c:91ff:feda:c514", "2604:180:f3::42", + "2604:a880:1:20::51:f001", "2606:4700::6810:f8f9", "2606:4700::6810:f9f9", "2606:4700::6812:1a80", + "2606:4700::6812:1b80", "2606:4700::6812:237", "2606:4700::6812:337", "2606:4700:3033::6812:2ccc", + "2606:4700:3033::6812:2dcc", "2606:4700:3033::6818:7b35", "2606:4700:3034::681c:16a", + "2606:4700:3035::6818:7a35", "2606:4700:3035::681f:5a8a", "2606:4700:3036::681c:6a", + "2606:4700:3036::681f:5b8a", "2606:4700:60:0:a71e:6467:cef8:2a56", "2620:10a:80bb::10", + "2620:10a:80bb::20", "2620:10a:80bb::30" "2620:10a:80bc::10", "2620:10a:80bc::20", + "2620:10a:80bc::30", "2620:119:fc::2", "2620:119:fc::3", "2620:fe::10", "2620:fe::11", + "2620:fe::9", "2620:fe::fe:10", "2620:fe::fe:11", "2620:fe::fe:9", "2620:fe::fe", + "2a00:5a60::ad1:ff", "2a00:5a60::ad2:ff", "2a00:5a60::bad1:ff", "2a00:5a60::bad2:ff", + "2a00:d880:5:bf0::7c93", "2a01:4f8:1c0c:8233::1", "2a01:4f8:1c1c:6b4b::1", "2a01:4f8:c2c:52bf::1", + "2a01:4f9:c010:43ce::1", "2a01:4f9:c01f:4::abcd", "2a01:7c8:d002:1ef:5054:ff:fe40:3703", + "2a01:9e00::54", "2a01:9e00::55", "2a01:9e01::54", "2a01:9e01::55", + "2a02:1205:34d5:5070:b26e:bfff:fe1d:e19b", "2a03:4000:38:53c::2", + "2a03:b0c0:0:1010::e9a:3001", "2a04:bdc7:100:70::abcd", "2a05:fc84::42", "2a05:fc84::43", + "2a07:a8c0::", "2a0d:4d00:81::1", "2a0d:5600:33:3::abcd", "35.198.2.76", "35.231.247.227", + "45.32.55.94", "45.67.219.208", "45.76.113.31", "45.77.180.10", "45.90.28.0", + "46.101.66.244", "46.227.200.54", "46.227.200.55", "46.239.223.80", "8.8.4.4", + "8.8.8.8", "83.77.85.7", "88.198.91.187", "9.9.9.10", "9.9.9.11", "9.9.9.9", + "94.130.106.88", "95.216.181.228", "95.216.212.177", "96.113.151.148", ] } diff --git a/examples/complex/remote_debug.py b/examples/complex/remote_debug.py index 4b117bdb..5129c9db 100644 --- a/examples/complex/remote_debug.py +++ b/examples/complex/remote_debug.py @@ -4,9 +4,11 @@ For general debugging purposes, it is easier to just debug mitmdump within PyCha Usage: - pip install pydevd on the mitmproxy machine - - Open the Run/Debug Configuration dialog box in PyCharm, and select the Python Remote Debug configuration type. - - Debugging works in the way that mitmproxy connects to the debug server on startup. - Specify host and port that mitmproxy can use to reach your PyCharm instance on startup. + - Open the Run/Debug Configuration dialog box in PyCharm, and select the + Python Remote Debug configuration type. + - Debugging works in the way that mitmproxy connects to the debug server + on startup. Specify host and port that mitmproxy can use to reach your + PyCharm instance on startup. - Adjust this inline script accordingly. - Start debug server in PyCharm - Set breakpoints diff --git a/examples/complex/sslstrip.py b/examples/complex/sslstrip.py index 8b904216..16d9b59a 100644 --- a/examples/complex/sslstrip.py +++ b/examples/complex/sslstrip.py @@ -51,9 +51,11 @@ def response(flow: http.HTTPFlow) -> None: flow.response.headers['Location'] = location.replace('https://', 'http://', 1) # strip upgrade-insecure-requests in Content-Security-Policy header - if re.search('upgrade-insecure-requests', flow.response.headers.get('Content-Security-Policy', ''), flags=re.IGNORECASE): + csp_header = flow.response.headers.get('Content-Security-Policy', '') + if re.search('upgrade-insecure-requests', csp_header, flags=re.IGNORECASE): csp = flow.response.headers['Content-Security-Policy'] - flow.response.headers['Content-Security-Policy'] = re.sub(r'upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) + new_header = re.sub(r'upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) + flow.response.headers['Content-Security-Policy'] = new_header # strip secure flag from 'Set-Cookie' headers cookies = flow.response.headers.get_all('Set-Cookie') diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 7adefd7a..6a3cc5fb 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -127,15 +127,18 @@ class ClientPlayback: self.q = queue.Queue() self.thread: RequestReplayThread = None - def check(self, f: http.HTTPFlow): + def check(self, f: flow.Flow): if f.live: return "Can't replay live flow." if f.intercepted: return "Can't replay intercepted flow." - if not f.request: - return "Can't replay flow with missing request." - if f.request.raw_content is None: - return "Can't replay flow with missing content." + if isinstance(f, http.HTTPFlow): + if not f.request: + return "Can't replay flow with missing request." + if f.request.raw_content is None: + return "Can't replay flow with missing content." + else: + return "Can only replay HTTP flows." def load(self, loader): loader.add_option( diff --git a/mitmproxy/addons/onboardingapp/templates/index.html b/mitmproxy/addons/onboardingapp/templates/index.html index aee6858c..822e2856 100644 --- a/mitmproxy/addons/onboardingapp/templates/index.html +++ b/mitmproxy/addons/onboardingapp/templates/index.html @@ -20,6 +20,14 @@ function changeTo(device) { </ul> </div> <div class="col-md-4"> + <h3 class="text-center">How to install on iOS 13+</h3> + <ul> + <li>Install and active the new Profile</li> + <li>Goto Settings -> General -> About -> Certificate Trust Settings</li> + <li>Toggle mitmproxy to ON</li> + <li>Done!</li> + </div> + <div class="col-md-4"> <h3 class="text-center">How to install on browsers</h3> <ul> <li>Safari on macOS uses the macOS keychain. So installing our CA in the system is enough.</li> @@ -27,15 +35,6 @@ function changeTo(device) { <li>Firefox on macOS has its own CA store and needs to be installed with Firefox-specific instructions that can be found <a href="https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox">HERE</a> .</li> </ul> </div> - <div class="col-md-4"> - <h3 class="text-center">How to install on iOS v10.3</h3> - <ul> - <li>After certificate installation, open Settings</li> - <li>Navigate to General and then About</li> - <li>Select Certificate Trust Settings</li> - <li>Each root that has been installed via a profile will be listed below the heading Enable Full Trust For Root Certificates. Toggle mitmproxy on</li> - <li>Done!</li> - </div> </div> </div>`; } diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 1d57d781..4d0a7ef9 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -21,7 +21,10 @@ from mitmproxy import command from mitmproxy import connections from mitmproxy import ctx from mitmproxy import io -from mitmproxy import http # noqa +from mitmproxy import http +from mitmproxy import tcp +from mitmproxy.utils import human + # The underlying sorted list implementation expects the sort key to be stable # for the lifetime of the object. However, if we sort by size, for instance, @@ -38,7 +41,7 @@ class _OrderKey: def __init__(self, view): self.view = view - def generate(self, f: http.HTTPFlow) -> typing.Any: # pragma: no cover + def generate(self, f: mitmproxy.flow.Flow) -> typing.Any: # pragma: no cover pass def refresh(self, f): @@ -68,32 +71,49 @@ class _OrderKey: class OrderRequestStart(_OrderKey): - def generate(self, f: http.HTTPFlow) -> int: - return f.request.timestamp_start or 0 + def generate(self, f: mitmproxy.flow.Flow) -> float: + return f.timestamp_start class OrderRequestMethod(_OrderKey): - def generate(self, f: http.HTTPFlow) -> str: - return f.request.method + def generate(self, f: mitmproxy.flow.Flow) -> str: + if isinstance(f, http.HTTPFlow): + return f.request.method + elif isinstance(f, tcp.TCPFlow): + return "TCP" + else: + raise NotImplementedError() class OrderRequestURL(_OrderKey): - def generate(self, f: http.HTTPFlow) -> str: - return f.request.url + def generate(self, f: mitmproxy.flow.Flow) -> str: + if isinstance(f, http.HTTPFlow): + return f.request.url + elif isinstance(f, tcp.TCPFlow): + return human.format_address(f.server_conn.address) + else: + raise NotImplementedError() class OrderKeySize(_OrderKey): - def generate(self, f: http.HTTPFlow) -> int: - s = 0 - if f.request.raw_content: - s += len(f.request.raw_content) - if f.response and f.response.raw_content: - s += len(f.response.raw_content) - return s - + def generate(self, f: mitmproxy.flow.Flow) -> int: + if isinstance(f, http.HTTPFlow): + size = 0 + if f.request.raw_content: + size += len(f.request.raw_content) + if f.response and f.response.raw_content: + size += len(f.response.raw_content) + return size + elif isinstance(f, tcp.TCPFlow): + size = 0 + for message in f.messages: + size += len(message.content) + return size + else: + raise NotImplementedError() -matchall = flowfilter.parse(".") +matchall = flowfilter.parse("~http | ~tcp") orders = [ ("t", "time"), @@ -555,6 +575,18 @@ class View(collections.abc.Sequence): def kill(self, f): self.update([f]) + def tcp_start(self, f): + self.add([f]) + + def tcp_message(self, f): + self.update([f]) + + def tcp_error(self, f): + self.update([f]) + + def tcp_end(self, f): + self.update([f]) + def update(self, flows: typing.Sequence[mitmproxy.flow.Flow]) -> None: """ Updates a list of flows. If flow is not in the state, it's ignored. diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index 35d1a688..450667a6 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -180,3 +180,8 @@ class Flow(stateobject.StateObject): if self.reply.state == "taken": self.reply.ack() self.reply.commit() + + @property + def timestamp_start(self) -> float: + """Start time of the flow.""" + return self.client_conn.timestamp_start diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 6b527e75..e9902224 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -173,6 +173,10 @@ class HTTPFlow(flow.Flow): s += ">" return s.format(flow=self) + @property + def timestamp_start(self) -> float: + return self.request.timestamp_start + def copy(self): f = super().copy() if self.request: diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index d8e943d3..4c0f1d6b 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -297,7 +297,7 @@ def create_client_context( if cert: try: context.use_privatekey_file(cert) - context.use_certificate_file(cert) + context.use_certificate_chain_file(cert) except SSL.Error as v: raise exceptions.TlsException("SSL client certificate error: %s" % str(v)) return context diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py index c1dd6179..23eb39f0 100644 --- a/mitmproxy/tools/_main.py +++ b/mitmproxy/tools/_main.py @@ -110,6 +110,8 @@ def run( master.commands.dump() sys.exit(0) if extra: + if(args.filter_args): + master.log.info(f"Only processing flows that match \"{' & '.join(args.filter_args)}\"") opts.update(**extra(args)) loop = asyncio.get_event_loop() diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 3dce8363..cba3a355 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -1,7 +1,6 @@ +import enum import platform import typing -import datetime -import time import math from functools import lru_cache from publicsuffix2 import get_sld, get_tld @@ -9,7 +8,10 @@ from publicsuffix2 import get_sld, get_tld import urwid import urwid.util +from mitmproxy import flow +from mitmproxy.http import HTTPFlow from mitmproxy.utils import human +from mitmproxy.tcp import TCPFlow # Detect Windows Subsystem for Linux IS_WSL = "Microsoft" in platform.platform() @@ -82,7 +84,7 @@ def format_keyvals( return ret -def fcol(s, attr): +def fcol(s: str, attr: str) -> typing.Tuple[str, int, urwid.Text]: s = str(s) return ( "fixed", @@ -105,20 +107,48 @@ if urwid.util.detected_encoding: else: SYMBOL_REPLAY = u"[r]" SYMBOL_RETURN = u"<-" - SYMBOL_MARK = "[m]" + SYMBOL_MARK = "#" SYMBOL_UP = "^" SYMBOL_DOWN = " " SYMBOL_ELLIPSIS = "~" - -def fixlen(s, maxlen): +SCHEME_STYLES = { + 'http': 'scheme_http', + 'https': 'scheme_https', + 'tcp': 'scheme_tcp', +} +HTTP_REQUEST_METHOD_STYLES = { + 'GET': 'method_get', + 'POST': 'method_post', + 'DELETE': 'method_delete', + 'HEAD': 'method_head', + 'PUT': 'method_put' +} +HTTP_RESPONSE_CODE_STYLE = { + 2: "code_200", + 3: "code_300", + 4: "code_400", + 5: "code_500", +} + + +class RenderMode(enum.Enum): + TABLE = 1 + """The flow list in table format, i.e. one row per flow.""" + LIST = 2 + """The flow list in list format, i.e. potentially multiple rows per flow.""" + DETAILVIEW = 3 + """The top lines in the detail view.""" + + +def fixlen(s: str, maxlen: int) -> str: if len(s) <= maxlen: return s.ljust(maxlen) else: return s[0:maxlen - len(SYMBOL_ELLIPSIS)] + SYMBOL_ELLIPSIS -def fixlen_r(s, maxlen): +def fixlen_r(s: str, maxlen: int) -> str: if len(s) <= maxlen: return s.rjust(maxlen) else: @@ -233,8 +263,8 @@ def colorize_req(s): for i in range(len(s)): c = s[i] if ((i < i_query and c == '/') or - (i < i_query and i > i_last_slash and c == '.') or - (i == i_query)): + (i < i_query and i > i_last_slash and c == '.') or + (i == i_query)): a = 'url_punctuation' elif i > i_query: if in_val: @@ -268,294 +298,435 @@ def colorize_url(url): 'https:': 'scheme_https', } return [ - (schemes.get(parts[0], "scheme_other"), len(parts[0]) - 1), - ('url_punctuation', 3), # :// - ] + colorize_host(parts[2]) + colorize_req('/' + parts[3]) + (schemes.get(parts[0], "scheme_other"), len(parts[0]) - 1), + ('url_punctuation', 3), # :// + ] + colorize_host(parts[2]) + colorize_req('/' + parts[3]) + + +def format_http_content_type(content_type: str) -> typing.Tuple[str, str]: + content_type = content_type.split(";")[0] + if content_type.endswith('/javascript'): + style = 'content_script' + elif content_type.startswith('text/'): + style = 'content_text' + elif (content_type.startswith('image/') or + content_type.startswith('video/') or + content_type.startswith('font/') or + "/x-font-" in content_type): + style = 'content_media' + elif content_type.endswith('/json') or content_type.endswith('/xml'): + style = 'content_data' + elif content_type.startswith('application/'): + style = 'content_raw' + else: + style = 'content_other' + return content_type, style + + +def format_duration(duration: float) -> typing.Tuple[str, str]: + pretty_duration = human.pretty_duration(duration) + style = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + 1000 * duration) / 12, 0.99)) + return pretty_duration, style + + +def format_size(num_bytes: int) -> typing.Tuple[str, str]: + pretty_size = human.pretty_size(num_bytes) + style = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + num_bytes) / 20, 0.99)) + return pretty_size, style + + +def format_left_indicators( + *, + focused: bool, + intercepted: bool, + timestamp: float +): + indicators: typing.List[typing.Union[str, typing.Tuple[str, str]]] = [] + if focused: + indicators.append(("focus", ">>")) + else: + indicators.append(" ") + pretty_timestamp = human.format_timestamp(timestamp)[-8:] + if intercepted: + indicators.append(("intercept", pretty_timestamp)) + else: + indicators.append(("text", pretty_timestamp)) + return "fixed", 10, urwid.Text(indicators) + + +def format_right_indicators( + *, + replay: bool, + marked: bool +): + indicators: typing.List[typing.Union[str, typing.Tuple[str, str]]] = [] + if replay: + indicators.append(("replay", SYMBOL_REPLAY)) + else: + indicators.append(" ") + if marked: + indicators.append(("mark", SYMBOL_MARK)) + else: + indicators.append(" ") + return "fixed", 2, urwid.Text(indicators) @lru_cache(maxsize=800) -def raw_format_list(f): - f = dict(f) - pile = [] +def format_http_flow_list( + *, + render_mode: RenderMode, + focused: bool, + marked: bool, + request_method: str, + request_scheme: str, + request_host: str, + request_path: str, + request_url: str, + request_http_version: str, + request_timestamp: float, + request_is_push_promise: bool, + request_is_replay: bool, + intercepted: bool, + response_code: typing.Optional[int], + response_reason: typing.Optional[str], + response_content_length: typing.Optional[int], + response_content_type: typing.Optional[str], + response_is_replay: bool, + duration: typing.Optional[float], + error_message: typing.Optional[str], +) -> urwid.Widget: req = [] - if f["extended"]: + + if render_mode is RenderMode.DETAILVIEW: + req.append(fcol(human.format_timestamp(request_timestamp), "highlight")) + else: + if focused: + req.append(fcol(">>", "focus")) + else: + req.append(fcol(" ", "focus")) + + method_style = HTTP_REQUEST_METHOD_STYLES.get(request_method, "method_other") + req.append(fcol(request_method, method_style)) + + if request_is_push_promise: + req.append(fcol('PUSH_PROMISE', 'method_http2_push')) + + preamble_len = sum(x[1] for x in req) + len(req) - 1 + + if request_http_version not in ("HTTP/1.0", "HTTP/1.1"): + request_url += " " + request_http_version + if intercepted and not response_code: + url_style = "intercept" + elif response_code or error_message: + url_style = "text" + else: + url_style = "title" + + if render_mode is RenderMode.DETAILVIEW: req.append( - fcol( - human.format_timestamp(f["req_timestamp"]), - "highlight" - ) + urwid.Text([(url_style, request_url)]) ) else: - req.append(fcol(">>" if f["focus"] else " ", "focus")) + req.append(truncated_plain(request_url, url_style)) - if f["marked"]: - req.append(fcol(SYMBOL_MARK, "mark")) + req.append(format_right_indicators(replay=request_is_replay or response_is_replay, marked=marked)) - if f["req_is_replay"]: - req.append(fcol(SYMBOL_REPLAY, "replay")) + resp = [ + ("fixed", preamble_len, urwid.Text("")) + ] + if response_code: + if intercepted: + style = "intercept" + else: + style = "" - req.append(fcol(f["req_method"], "method")) + status_style = style or HTTP_RESPONSE_CODE_STYLE.get(response_code // 100, "code_other") + resp.append(fcol(SYMBOL_RETURN, status_style)) + if response_is_replay: + resp.append(fcol(SYMBOL_REPLAY, "replay")) + resp.append(fcol(str(response_code), status_style)) + if response_reason and render_mode is RenderMode.DETAILVIEW: + resp.append(fcol(response_reason, status_style)) + + if response_content_type: + ct, ct_style = format_http_content_type(response_content_type) + resp.append(fcol(ct, style or ct_style)) + + if response_content_length: + size, size_style = format_size(response_content_length) + elif response_content_length == 0: + size = "[no content]" + size_style = "text" + else: + size = "[content missing]" + size_style = "text" + resp.append(fcol(size, style or size_style)) + + if duration: + dur, dur_style = format_duration(duration) + resp.append(fcol(dur, style or dur_style)) + elif error_message: + resp.append(fcol(SYMBOL_RETURN, "error")) + resp.append(urwid.Text([("error", error_message)])) - preamble = sum(i[1] for i in req) + len(req) - 1 + return urwid.Pile([ + urwid.Columns(req, dividechars=1), + urwid.Columns(resp, dividechars=1) + ]) - if f["intercepted"] and not f["acked"]: - uc = "intercept" - elif "resp_code" in f or "err_msg" in f: - uc = "text" - else: - uc = "title" - url = f["req_url"] +@lru_cache(maxsize=800) +def format_http_flow_table( + *, + render_mode: RenderMode, + focused: bool, + marked: bool, + request_method: str, + request_scheme: str, + request_host: str, + request_path: str, + request_url: str, + request_http_version: str, + request_timestamp: float, + request_is_push_promise: bool, + request_is_replay: bool, + intercepted: bool, + response_code: typing.Optional[int], + response_reason: typing.Optional[str], + response_content_length: typing.Optional[int], + response_content_type: typing.Optional[str], + response_is_replay: bool, + duration: typing.Optional[float], + error_message: typing.Optional[str], +) -> urwid.Widget: + items = [ + format_left_indicators( + focused=focused, + intercepted=intercepted, + timestamp=request_timestamp + ) + ] - if f["cols"] and len(url) > f["cols"]: - url = url[:f["cols"]] + "…" + if intercepted and not response_code: + request_style = "intercept" + else: + request_style = "" - if f["req_http_version"] not in ("HTTP/1.0", "HTTP/1.1"): - url += " " + f["req_http_version"] - req.append( - urwid.Text([(uc, url)]) - ) + scheme_style = request_style or SCHEME_STYLES.get(request_scheme, "scheme_other") + items.append(fcol(fixlen(request_scheme.upper(), 5), scheme_style)) - pile.append(urwid.Columns(req, dividechars=1)) + if request_is_push_promise: + method_style = 'method_http2_push' + else: + method_style = request_style or HTTP_REQUEST_METHOD_STYLES.get(request_method, "method_other") + items.append(fcol(fixlen(request_method, 4), method_style)) - resp = [] - resp.append( - ("fixed", preamble, urwid.Text("")) - ) + items.append(('weight', 0.25, TruncatedText(request_host, colorize_host(request_host), 'right'))) + items.append(('weight', 1.0, TruncatedText(request_path, colorize_req(request_path), 'left'))) - if "resp_code" in f: - codes = { - 2: "code_200", - 3: "code_300", - 4: "code_400", - 5: "code_500", - } - ccol = codes.get(f["resp_code"] // 100, "code_other") - resp.append(fcol(SYMBOL_RETURN, ccol)) - if f["resp_is_replay"]: - resp.append(fcol(SYMBOL_REPLAY, "replay")) - resp.append(fcol(f["resp_code"], ccol)) - if f["extended"]: - resp.append(fcol(f["resp_reason"], ccol)) - if f["intercepted"] and f["resp_code"] and not f["acked"]: - rc = "intercept" + if intercepted and response_code: + response_style = "intercept" + else: + response_style = "" + + if response_code: + + status = str(response_code) + status_style = response_style or HTTP_RESPONSE_CODE_STYLE.get(response_code // 100, "code_other") + + if response_content_length and response_content_type: + content, content_style = format_http_content_type(response_content_type) + content_style = response_style or content_style + elif response_content_length: + content = '' + content_style = 'content_none' + elif response_content_length == 0: + content = "[no content]" + content_style = 'content_none' else: - rc = "text" + content = "[content missing]" + content_style = 'content_none' - if f["resp_ctype"]: - resp.append(fcol(f["resp_ctype"], rc)) - resp.append(fcol(f["resp_clen"], rc)) - pretty_duration = human.pretty_duration(f["duration"]) - resp.append(fcol(pretty_duration, rc)) + elif error_message: + status = 'err' + status_style = 'error' + content = error_message + content_style = 'error' - elif f["err_msg"]: - resp.append(fcol(SYMBOL_RETURN, "error")) - resp.append( - urwid.Text([ - ( - "error", - f["err_msg"] - ) - ]) - ) - pile.append(urwid.Columns(resp, dividechars=1)) - return urwid.Pile(pile) + else: + status = '' + status_style = 'text' + content = '' + content_style = '' + items.append(fcol(fixlen(status, 3), status_style)) + items.append(('weight', 0.15, truncated_plain(content, content_style, 'right'))) -@lru_cache(maxsize=800) -def raw_format_table(f): - f = dict(f) - pile = [] - req = [] + if response_content_length: + size, size_style = format_size(response_content_length) + items.append(fcol(fixlen_r(size, 5), response_style or size_style)) + else: + items.append(("fixed", 5, urwid.Text(""))) - cursor = [' ', 'focus'] - if f['focus']: - cursor[0] = '>' - req.append(fcol(*cursor)) + if duration: + duration_pretty, duration_style = format_duration(duration) + items.append(fcol(fixlen_r(duration_pretty, 5), response_style or duration_style)) + else: + items.append(("fixed", 5, urwid.Text(""))) - if f.get('resp_is_replay', False) or f.get('req_is_replay', False): - req.append(fcol(SYMBOL_REPLAY, 'replay')) - if f['marked']: - req.append(fcol(SYMBOL_MARK, 'mark')) + items.append(format_right_indicators( + replay=request_is_replay or response_is_replay, + marked=marked + )) + return urwid.Columns(items, dividechars=1, min_width=15) - if f["two_line"]: - req.append(TruncatedText(f["req_url"], colorize_url(f["req_url"]), 'left')) - pile.append(urwid.Columns(req, dividechars=1)) - req = [] - req.append(fcol(' ', 'text')) +@lru_cache(maxsize=800) +def format_tcp_flow( + *, + render_mode: RenderMode, + focused: bool, + timestamp_start: float, + marked: bool, + client_address, + server_address, + total_size: int, + duration: typing.Optional[float], + error_message: typing.Optional[str], +): + conn = f"{human.format_address(client_address)} <-> {human.format_address(server_address)}" + + items = [] + + if render_mode in (RenderMode.TABLE, RenderMode.DETAILVIEW): + items.append( + format_left_indicators(focused=focused, intercepted=False, timestamp=timestamp_start) + ) + else: + if focused: + items.append(fcol(">>", "focus")) + else: + items.append(fcol(" ", "focus")) - if f["intercepted"] and not f["acked"]: - uc = "intercept" - elif "resp_code" in f or f["err_msg"] is not None: - uc = "highlight" + if render_mode is RenderMode.TABLE: + items.append(fcol("TCP ", SCHEME_STYLES["tcp"])) else: - uc = "title" + items.append(fcol("TCP", SCHEME_STYLES["tcp"])) - if f["extended"]: - s = human.format_timestamp(f["req_timestamp"]) + items.append(('weight', 1.0, truncated_plain(conn, "text", 'left'))) + if error_message: + items.append(('weight', 1.0, truncated_plain(error_message, "error", 'left'))) + + if total_size: + size, size_style = format_size(total_size) + items.append(fcol(fixlen_r(size, 5), size_style)) else: - s = datetime.datetime.fromtimestamp(time.mktime(time.localtime(f["req_timestamp"]))).strftime("%H:%M:%S") - req.append(fcol(s, uc)) - - methods = { - 'GET': 'method_get', - 'POST': 'method_post', - 'DELETE': 'method_delete', - 'HEAD': 'method_head', - 'PUT': 'method_put' - } - uc = methods.get(f["req_method"], "method_other") - if f['extended']: - req.append(fcol(f["req_method"], uc)) - if f["req_promise"]: - req.append(fcol('PUSH_PROMISE', 'method_http2_push')) + items.append(("fixed", 5, urwid.Text(""))) + + if duration: + duration_pretty, duration_style = format_duration(duration) + items.append(fcol(fixlen_r(duration_pretty, 5), duration_style)) else: - if f["req_promise"]: - uc = 'method_http2_push' - req.append(("fixed", 4, truncated_plain(f["req_method"], uc))) + items.append(("fixed", 5, urwid.Text(""))) + + items.append(format_right_indicators(replay=False, marked=marked)) + + return urwid.Pile([ + urwid.Columns(items, dividechars=1, min_width=15) + ]) + - if f["two_line"]: - req.append(fcol(f["req_http_version"], 'text')) +def format_flow( + f: flow.Flow, + *, + render_mode: RenderMode, + hostheader: bool = False, # pass options directly if we need more stuff from them + focused: bool = True, +) -> urwid.Widget: + """ + This functions calls the proper renderer depending on the flow type. + We also want to cache the renderer output, so we extract all attributes + relevant for display and call the render with only that. This assures that rows + are updated if the flow is changed. + """ + duration: typing.Optional[float] + error_message: typing.Optional[str] + if f.error: + error_message = f.error.msg else: - schemes = { - 'http': 'scheme_http', - 'https': 'scheme_https', - } - req.append(fcol(fixlen(f["req_scheme"].upper(), 5), schemes.get(f["req_scheme"], "scheme_other"))) - - req.append(('weight', 0.25, TruncatedText(f["req_host"], colorize_host(f["req_host"]), 'right'))) - req.append(('weight', 1.0, TruncatedText(f["req_path"], colorize_req(f["req_path"]), 'left'))) - - ret = (' ' * len(SYMBOL_RETURN), 'text') - status = ('', 'text') - content = ('', 'text') - size = ('', 'text') - duration = ('', 'text') - - if "resp_code" in f: - codes = { - 2: "code_200", - 3: "code_300", - 4: "code_400", - 5: "code_500", - } - ccol = codes.get(f["resp_code"] // 100, "code_other") - ret = (SYMBOL_RETURN, ccol) - status = (str(f["resp_code"]), ccol) - - if f["resp_len"] < 0: - if f["intercepted"] and f["resp_code"] and not f["acked"]: - rc = "intercept" + error_message = None + + if isinstance(f, TCPFlow): + total_size = 0 + for message in f.messages: + total_size += len(message.content) + if f.messages: + duration = f.messages[-1].timestamp - f.timestamp_start + else: + duration = None + return format_tcp_flow( + render_mode=render_mode, + focused=focused, + timestamp_start=f.timestamp_start, + marked=f.marked, + client_address=f.client_conn.address, + server_address=f.server_conn.address, + total_size=total_size, + duration=duration, + error_message=error_message, + ) + elif isinstance(f, HTTPFlow): + intercepted = ( + f.intercepted and not (f.reply and f.reply.state == "committed") + ) + response_content_length: typing.Optional[int] + if f.response: + if f.response.raw_content is not None: + response_content_length = len(f.response.raw_content) else: - rc = "content_none" - - if f["resp_len"] == -1: - contentdesc = "[content missing]" + response_content_length = None + response_code = f.response.status_code + response_reason = f.response.reason + response_content_type = f.response.headers.get("content-type") + response_is_replay = f.response.is_replay + if f.response.timestamp_end: + duration = max([f.response.timestamp_end - f.request.timestamp_start, 0]) else: - contentdesc = "[no content]" - content = (contentdesc, rc) + duration = None else: - if f["resp_ctype"]: - ctype = f["resp_ctype"].split(";")[0] - if ctype.endswith('/javascript'): - rc = 'content_script' - elif ctype.startswith('text/'): - rc = 'content_text' - elif (ctype.startswith('image/') or - ctype.startswith('video/') or - ctype.startswith('font/') or - "/x-font-" in ctype): - rc = 'content_media' - elif ctype.endswith('/json') or ctype.endswith('/xml'): - rc = 'content_data' - elif ctype.startswith('application/'): - rc = 'content_raw' - else: - rc = 'content_other' - content = (ctype, rc) - - rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + f["resp_len"]) / 20, 0.99)) - - size_str = human.pretty_size(f["resp_len"]) - if not f['extended']: - # shorten to 5 chars max - if len(size_str) > 5: - size_str = size_str[0:4].rstrip('.') + size_str[-1:] - size = (size_str, rc) - - if f['duration'] is not None: - rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + 1000 * f['duration']) / 12, 0.99)) - duration = (human.pretty_duration(f['duration']), rc) - - elif f["err_msg"]: - status = ('Err', 'error') - content = f["err_msg"], 'error' - - if f["two_line"]: - req.append(fcol(*ret)) - req.append(fcol(fixlen(status[0], 3), status[1])) - req.append(('weight', 0.15, truncated_plain(content[0], content[1], 'right'))) - if f['extended']: - req.append(fcol(*size)) - else: - req.append(fcol(fixlen_r(size[0], 5), size[1])) - req.append(fcol(fixlen_r(duration[0], 5), duration[1])) - - pile.append(urwid.Columns(req, dividechars=1, min_width=15)) - - return urwid.Pile(pile) - - -def format_flow(f, focus, extended=False, hostheader=False, cols=False, layout='default'): - acked = False - if f.reply and f.reply.state == "committed": - acked = True - d = dict( - focus=focus, - extended=extended, - two_line=extended or cols < 100, - cols=cols, - intercepted=f.intercepted, - acked=acked, - req_timestamp=f.request.timestamp_start, - req_is_replay=f.request.is_replay, - req_method=f.request.method, - req_promise='h2-pushed-stream' in f.metadata, - req_url=f.request.pretty_url if hostheader else f.request.url, - req_scheme=f.request.scheme, - req_host=f.request.pretty_host if hostheader else f.request.host, - req_path=f.request.path, - req_http_version=f.request.http_version, - err_msg=f.error.msg if f.error else None, - marked=f.marked, - ) - if f.response: - if f.response.raw_content: - content_len = len(f.response.raw_content) - contentdesc = human.pretty_size(len(f.response.raw_content)) - elif f.response.raw_content is None: - content_len = -1 - contentdesc = "[content missing]" + response_content_length = None + response_code = None + response_reason = None + response_content_type = None + response_is_replay = False + duration = None + + if render_mode in (RenderMode.LIST, RenderMode.DETAILVIEW): + render_func = format_http_flow_list else: - content_len = -2 - contentdesc = "[no content]" - - duration = None - if f.response.timestamp_end and f.request.timestamp_start: - duration = max([f.response.timestamp_end - f.request.timestamp_start, 0]) - - d.update(dict( - resp_code=f.response.status_code, - resp_reason=f.response.reason, - resp_is_replay=f.response.is_replay, - resp_len=content_len, - resp_ctype=f.response.headers.get("content-type"), - resp_clen=contentdesc, + render_func = format_http_flow_table + return render_func( + render_mode=render_mode, + focused=focused, + marked=f.marked, + request_method=f.request.method, + request_scheme=f.request.scheme, + request_host=f.request.pretty_host if hostheader else f.request.host, + request_path=f.request.path, + request_url=f.request.pretty_url if hostheader else f.request.url, + request_http_version=f.request.http_version, + request_timestamp=f.request.timestamp_start, + request_is_push_promise='h2-pushed-stream' in f.metadata, + request_is_replay=f.request.is_replay, + intercepted=intercepted, + response_code=response_code, + response_reason=response_reason, + response_content_length=response_content_length, + response_content_type=response_content_type, + response_is_replay=response_is_replay, duration=duration, - )) + error_message=error_message, + ) - if ((layout == 'default' and cols < 100) or layout == "list"): - return raw_format_list(tuple(sorted(d.items()))) else: - return raw_format_table(tuple(sorted(d.items()))) + raise NotImplementedError() diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index 905653e7..12448945 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -9,6 +9,7 @@ from mitmproxy import exceptions from mitmproxy import flow from mitmproxy import http from mitmproxy import log +from mitmproxy import tcp from mitmproxy.tools.console import keymap from mitmproxy.tools.console import overlay from mitmproxy.tools.console import signals @@ -112,7 +113,7 @@ class ConsoleAddon: choices=sorted(console_palettes), ) loader.add_option( - "console_palette_transparent", bool, False, + "console_palette_transparent", bool, True, "Set transparent background for palette." ) loader.add_option( @@ -334,9 +335,10 @@ class ConsoleAddon: @command.command("console.view.flow") def view_flow(self, flow: flow.Flow) -> None: """View a flow.""" - if hasattr(flow, "request"): - # FIME: Also set focus? + if isinstance(flow, (http.HTTPFlow, tcp.TCPFlow)): self.master.switch_view("flowview") + else: + ctx.log.warn(f"No detail view for {type(flow).__name__}.") @command.command("console.exit") def exit(self) -> None: diff --git a/mitmproxy/tools/console/flowdetailview.py b/mitmproxy/tools/console/flowdetailview.py index 443ca526..fb2494e8 100644 --- a/mitmproxy/tools/console/flowdetailview.py +++ b/mitmproxy/tools/console/flowdetailview.py @@ -1,5 +1,7 @@ +import typing import urwid +import mitmproxy.flow from mitmproxy import http from mitmproxy.tools.console import common, searchable from mitmproxy.utils import human @@ -13,13 +15,19 @@ def maybe_timestamp(base, attr): return "active" -def flowdetails(state, flow: http.HTTPFlow): +def flowdetails(state, flow: mitmproxy.flow.Flow): text = [] sc = flow.server_conn cc = flow.client_conn - req = flow.request - resp = flow.response + req: typing.Optional[http.HTTPRequest] + resp: typing.Optional[http.HTTPResponse] + if isinstance(flow, http.HTTPFlow): + req = flow.request + resp = flow.response + else: + req = None + resp = None metadata = flow.metadata if metadata is not None and len(metadata) > 0: @@ -126,6 +134,12 @@ def flowdetails(state, flow: http.HTTPFlow): maybe_timestamp(cc, "timestamp_tls_setup") ) ) + parts.append( + ( + "Client conn. closed", + maybe_timestamp(cc, "timestamp_end") + ) + ) if sc is not None and sc.timestamp_start: parts.append( @@ -147,6 +161,12 @@ def flowdetails(state, flow: http.HTTPFlow): maybe_timestamp(sc, "timestamp_tls_setup") ) ) + parts.append( + ( + "Server conn. closed", + maybe_timestamp(sc, "timestamp_end") + ) + ) if req is not None and req.timestamp_start: parts.append( diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index 9650c0d3..b21a16b3 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -14,12 +14,17 @@ class FlowItem(urwid.WidgetWrap): def get_text(self): cols, _ = self.master.ui.get_cols_rows() + layout = self.master.options.console_flowlist_layout + if layout == "list" or (layout == 'default' and cols < 100): + render_mode = common.RenderMode.LIST + else: + render_mode = common.RenderMode.TABLE + return common.format_flow( self.flow, - self.flow is self.master.view.focus.flow, + render_mode=render_mode, + focused=self.flow is self.master.view.focus.flow, hostheader=self.master.options.showhost, - cols=cols, - layout=self.master.options.console_flowlist_layout ) def selectable(self): @@ -27,9 +32,8 @@ class FlowItem(urwid.WidgetWrap): def mouse_event(self, size, event, button, col, row, focus): if event == "mouse press" and button == 1: - if self.flow.request: - self.master.commands.execute("console.view.flow @focus") - return True + self.master.commands.execute("console.view.flow @focus") + return True def keypress(self, size, key): return key diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index fd41da0d..3fef70ce 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -5,9 +5,11 @@ from typing import Optional, Union # noqa import urwid +import mitmproxy.flow from mitmproxy import contentviews from mitmproxy import ctx from mitmproxy import http +from mitmproxy import tcp from mitmproxy.tools.console import common from mitmproxy.tools.console import layoutwidget from mitmproxy.tools.console import flowdetailview @@ -24,8 +26,8 @@ class SearchError(Exception): class FlowViewHeader(urwid.WidgetWrap): def __init__( - self, - master: "mitmproxy.tools.console.master.ConsoleMaster", + self, + master: "mitmproxy.tools.console.master.ConsoleMaster", ) -> None: self.master = master self.focus_changed() @@ -35,11 +37,8 @@ class FlowViewHeader(urwid.WidgetWrap): if self.master.view.focus.flow: self._w = common.format_flow( self.master.view.focus.flow, - False, - extended=True, + render_mode=common.RenderMode.DETAILVIEW, hostheader=self.master.options.showhost, - cols=cols, - layout=self.master.options.console_flowlist_layout ) else: self._w = urwid.Pile([]) @@ -52,45 +51,90 @@ class FlowDetails(tabs.Tabs): self.show() self.last_displayed_body = None - def focus_changed(self): - if self.master.view.focus.flow: - self.tabs = [ - (self.tab_request, self.view_request), - (self.tab_response, self.view_response), - (self.tab_details, self.view_details), - ] - self.show() - else: - self.master.window.pop() - @property def view(self): return self.master.view @property - def flow(self): + def flow(self) -> mitmproxy.flow.Flow: return self.master.view.focus.flow - def tab_request(self): - if self.flow.intercepted and not self.flow.response: + def focus_changed(self): + if self.flow: + if isinstance(self.flow, http.HTTPFlow): + self.tabs = [ + (self.tab_http_request, self.view_request), + (self.tab_http_response, self.view_response), + (self.tab_details, self.view_details), + ] + elif isinstance(self.flow, tcp.TCPFlow): + self.tabs = [ + (self.tab_tcp_stream, self.view_tcp_stream), + (self.tab_details, self.view_details), + ] + self.show() + else: + self.master.window.pop() + + def tab_http_request(self): + flow = self.flow + assert isinstance(flow, http.HTTPFlow) + if self.flow.intercepted and not flow.response: return "Request intercepted" else: return "Request" - def tab_response(self): - if self.flow.intercepted and self.flow.response: + def tab_http_response(self): + flow = self.flow + assert isinstance(flow, http.HTTPFlow) + if self.flow.intercepted and flow.response: return "Response intercepted" else: return "Response" + def tab_tcp_stream(self): + return "TCP Stream" + def tab_details(self): return "Detail" def view_request(self): - return self.conn_text(self.flow.request) + flow = self.flow + assert isinstance(flow, http.HTTPFlow) + return self.conn_text(flow.request) def view_response(self): - return self.conn_text(self.flow.response) + flow = self.flow + assert isinstance(flow, http.HTTPFlow) + return self.conn_text(flow.response) + + def view_tcp_stream(self) -> urwid.Widget: + flow = self.flow + assert isinstance(flow, tcp.TCPFlow) + + if not flow.messages: + return searchable.Searchable([urwid.Text(("highlight", "No messages."))]) + + from_client = None + messages = [] + for message in flow.messages: + if message.from_client is not from_client: + messages.append(message.content) + from_client = message.from_client + else: + messages[-1] += message.content + + from_client = flow.messages[0].from_client + parts = [] + for message in messages: + parts.append( + ( + "head" if from_client else "key", + message + ) + ) + from_client = not from_client + return searchable.Searchable([urwid.Text(parts)]) def view_details(self): return flowdetailview.flowdetails(self.view, self.flow) @@ -229,7 +273,7 @@ class FlowView(urwid.Frame, layoutwidget.LayoutWidget): def __init__(self, master): super().__init__( FlowDetails(master), - header = FlowViewHeader(master), + header=FlowViewHeader(master), ) self.master = master diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 683afa42..a680a5a7 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -22,9 +22,8 @@ class Palette: 'option_selected_key', # List and Connections - 'method', 'method_get', 'method_post', 'method_delete', 'method_other', 'method_head', 'method_put', 'method_http2_push', - 'scheme_http', 'scheme_https', 'scheme_other', + 'scheme_http', 'scheme_https', 'scheme_tcp', 'scheme_other', 'url_punctuation', 'url_domain', 'url_filename', 'url_extension', 'url_query_key', 'url_query_value', 'content_none', 'content_text', 'content_script', 'content_media', 'content_data', 'content_raw', 'content_other', 'focus', @@ -124,7 +123,6 @@ class LowDark(Palette): option_active_selected = ('light red', 'light gray'), # List and Connections - method = ('dark cyan', 'default'), method_get = ('light green', 'default'), method_post = ('brown', 'default'), method_delete = ('light red', 'default'), @@ -135,6 +133,7 @@ class LowDark(Palette): scheme_http = ('dark cyan', 'default'), scheme_https = ('dark green', 'default'), + scheme_tcp=('dark magenta', 'default'), scheme_other = ('dark magenta', 'default'), url_punctuation = ('light gray', 'default'), @@ -229,7 +228,6 @@ class LowLight(Palette): option_active_selected = ('light red', 'light gray'), # List and Connections - method = ('dark cyan', 'default'), method_get = ('dark green', 'default'), method_post = ('brown', 'default'), method_head = ('dark cyan', 'default'), @@ -240,6 +238,7 @@ class LowLight(Palette): scheme_http = ('dark cyan', 'default'), scheme_https = ('light green', 'default'), + scheme_tcp=('light magenta', 'default'), scheme_other = ('light magenta', 'default'), url_punctuation = ('dark gray', 'default'), @@ -353,7 +352,6 @@ class SolarizedLight(LowLight): # List and Connections - method = ('dark cyan', 'default'), method_get = (sol_green, 'default'), method_post = (sol_orange, 'default'), method_head = (sol_cyan, 'default'), @@ -364,6 +362,7 @@ class SolarizedLight(LowLight): scheme_http = (sol_cyan, 'default'), scheme_https = ('light green', 'default'), + scheme_tcp=('light magenta', 'default'), scheme_other = ('light magenta', 'default'), url_punctuation = ('dark gray', 'default'), @@ -434,7 +433,6 @@ class SolarizedDark(LowDark): # List and Connections focus = (sol_base1, 'default'), - method = (sol_cyan, 'default'), method_get = (sol_green, 'default'), method_post = (sol_orange, 'default'), method_delete = (sol_red, 'default'), diff --git a/release/README.md b/release/README.md index 8632d644..fb245e23 100644 --- a/release/README.md +++ b/release/README.md @@ -27,20 +27,18 @@ These steps assume you are on the correct branch and have a git remote called `o - The Homebrew maintainers are typically very fast and detect our new relese within a day. - If you feel the need, you can run this from a macOS machine: - `brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/v<version number here>` + `brew bump-formula-pr --url https://github.com/mitmproxy/mitmproxy/archive/v<version number here>.tar.gz mitmproxy` ### Docker - The docker image is built by our CI workers and pushed to Docker Hub automatically. - Please verify that https://hub.docker.com/r/mitmproxy/mitmproxy/tags/ has the latest version. -- The latest and latest-ARMv7 tags should auto-update. @mhils introduced this after the 5.0.0 release. - Please verify that this is the case and remove this notice. For reference, this is how to do it manually: - `export VERSION=4.0.3 && docker pull mitmproxy/mitmproxy:$VERSION && docker tag mitmproxy/mitmproxy:$VERSION mitmproxy/mitmproxy:latest && docker push mitmproxy/mitmproxy:latest`. +- Please verify that the latest tag points to the most recent image (same digest / hash). ### Docs - - `./build-current`. If everything looks alright, continue with - - `./upload-stable`, - - `./build-archive`, and - - `./upload-archive v4`. Doing this now already saves you from switching back to an old state on the next release. + - `./build.sh`. If everything looks alright, continue with + - `./upload-stable.sh`, + - `DOCS_ARCHIVE=true ./build.sh`, and + - `./upload-archive.sh v4`. Doing this now already saves you from switching back to an old state on the next release. ### Website - Update version here: diff --git a/release/cibuild.py b/release/cibuild.py index d070a4b9..b00bdb5c 100755 --- a/release/cibuild.py +++ b/release/cibuild.py @@ -356,15 +356,17 @@ def build_docker_image(be: BuildEnviron): # pragma: no cover "--file", "release/docker/Dockerfile", "." ]) - subprocess.check_call([ + # smoke-test the newly built docker image + r = subprocess.run([ "docker", - "build", - "--tag", be.docker_tag + "-ARMv7", - "--build-arg", "WHEEL_MITMPROXY={}".format(whl), - "--build-arg", "WHEEL_BASENAME_MITMPROXY={}".format(os.path.basename(whl)), - "--file", "release/docker/DockerfileARMv7", - "." - ]) + "run", + "--rm", + be.docker_tag, + "mitmdump", + "--version", + ], check=True, capture_output=True) + print(r.stdout.decode()) + assert "Mitmproxy: " in r.stdout.decode() def build_pyinstaller(be: BuildEnviron): # pragma: no cover @@ -569,11 +571,10 @@ def upload(): # pragma: no cover "-u", be.docker_username, "-p", be.docker_password, ]) - for variant in ["", "-ARMv7"]: - subprocess.check_call(["docker", "push", be.docker_tag + variant]) - if be.is_prod_release: - subprocess.check_call(["docker", "tag", be.docker_tag + variant, "mitmproxy/mitmproxy:latest" + variant]) - subprocess.check_call(["docker", "push", "mitmproxy/mitmproxy:latest" + variant]) + subprocess.check_call(["docker", "push", be.docker_tag]) + if be.is_prod_release: + subprocess.check_call(["docker", "tag", be.docker_tag, "mitmproxy/mitmproxy:latest"]) + subprocess.check_call(["docker", "push", "mitmproxy/mitmproxy:latest"]) if __name__ == "__main__": # pragma: no cover diff --git a/release/docker/Dockerfile b/release/docker/Dockerfile index 258bccf5..5f496e9f 100644 --- a/release/docker/Dockerfile +++ b/release/docker/Dockerfile @@ -1,4 +1,4 @@ -FROM alpine:3.8 +FROM alpine:3.11 ENV LANG=en_US.UTF-8 diff --git a/release/docker/DockerfileARMv7 b/release/docker/DockerfileARMv7 deleted file mode 100644 index 40f10ede..00000000 --- a/release/docker/DockerfileARMv7 +++ /dev/null @@ -1,46 +0,0 @@ -FROM resin/raspberrypi3-alpine:3.7 - -ENV LANG=en_US.UTF-8 - -ARG WHEEL_MITMPROXY -ARG WHEEL_BASENAME_MITMPROXY - -COPY $WHEEL_MITMPROXY /home/mitmproxy/ - -RUN [ "cross-build-start" ] - -# Add our user first to make sure the ID get assigned consistently, -# regardless of whatever dependencies get added. -RUN addgroup -S mitmproxy && adduser -S -G mitmproxy mitmproxy \ - && apk add --no-cache \ - su-exec \ - git \ - g++ \ - libffi \ - libffi-dev \ - libstdc++ \ - openssl \ - openssl-dev \ - python3 \ - python3-dev \ - && python3 -m ensurepip --upgrade \ - && pip3 install -U pip \ - && LDFLAGS=-L/lib pip3 install -U /home/mitmproxy/${WHEEL_BASENAME_MITMPROXY} \ - && apk del --purge \ - git \ - g++ \ - libffi-dev \ - openssl-dev \ - python3-dev \ - && rm -rf ~/.cache/pip /home/mitmproxy/${WHEEL_BASENAME_MITMPROXY} - -RUN [ "cross-build-end" ] - -VOLUME /home/mitmproxy/.mitmproxy - -COPY release/docker/docker-entrypoint.sh /usr/local/bin/ -ENTRYPOINT ["docker-entrypoint.sh"] - -EXPOSE 8080 8081 - -CMD ["mitmproxy"] diff --git a/release/docker/README.md b/release/docker/README.md index 2fa93949..df9834b8 100644 --- a/release/docker/README.md +++ b/release/docker/README.md @@ -40,8 +40,7 @@ The available release tags can be seen * `master` always tracks the git-master branch and represents the unstable development tree. * `latest` always points to the same image as the most recent stable release, including bugfix releases (e.g., `4.0.0` and `4.0.1`). -* `X.Y.Z` tags contain the mitmproxy release with this version number. -* `*-ARMv7` are images built for Raspbian / Raspberry Pi systems. +* `X.Y.Z` tags contain the mitmproxy release with this version number. # Security Notice diff --git a/release/docker/docker-entrypoint.sh b/release/docker/docker-entrypoint.sh index a4abe4ce..84ea81e6 100755 --- a/release/docker/docker-entrypoint.sh +++ b/release/docker/docker-entrypoint.sh @@ -1,13 +1,17 @@ #!/bin/sh -set -e +# WARNING: do not change the shebang - the Docker base image might not have what you want! + +set -o errexit +set -o pipefail +set -o nounset +# set -o xtrace MITMPROXY_PATH="/home/mitmproxy/.mitmproxy" if [[ "$1" = "mitmdump" || "$1" = "mitmproxy" || "$1" = "mitmweb" ]]; then - mkdir -p "$MITMPROXY_PATH" - chown -R mitmproxy:mitmproxy "$MITMPROXY_PATH" - - su-exec mitmproxy "$@" + mkdir -p "$MITMPROXY_PATH" + chown -R mitmproxy:mitmproxy "$MITMPROXY_PATH" + su-exec mitmproxy "$@" else - exec "$@" + exec "$@" fi diff --git a/test/mitmproxy/addons/test_clientplayback.py b/test/mitmproxy/addons/test_clientplayback.py index 1929ee3d..ecab29df 100644 --- a/test/mitmproxy/addons/test_clientplayback.py +++ b/test/mitmproxy/addons/test_clientplayback.py @@ -144,6 +144,9 @@ class TestClientPlayback: f.request.raw_content = None assert "missing content" in cp.check(f) + f = tflow.ttcpflow() + assert "Can only replay HTTP" in cp.check(f) + @pytest.mark.asyncio async def test_playback(self): cp = clientplayback.ClientPlayback() diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index f5088a68..506924e4 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -36,7 +36,7 @@ def test_order_refresh(): assert sargs -def test_order_generators(): +def test_order_generators_http(): v = view.View() tf = tflow.tflow(resp=True) @@ -53,6 +53,23 @@ def test_order_generators(): assert sz.generate(tf) == len(tf.request.raw_content) + len(tf.response.raw_content) +def test_order_generators_tcp(): + v = view.View() + tf = tflow.ttcpflow() + + rs = view.OrderRequestStart(v) + assert rs.generate(tf) == 946681200 + + rm = view.OrderRequestMethod(v) + assert rm.generate(tf) == "TCP" + + ru = view.OrderRequestURL(v) + assert ru.generate(tf) == "address:22" + + sz = view.OrderKeySize(v) + assert sz.generate(tf) == sum(len(m.content) for m in tf.messages) + + def test_simple(): v = view.View() f = tft(start=1) @@ -105,6 +122,21 @@ def test_simple(): assert len(v._store) == 0 +def test_simple_tcp(): + v = view.View() + f = tflow.ttcpflow() + assert v.store_count() == 0 + v.tcp_start(f) + assert list(v) == [f] + + # These all just call update + v.tcp_start(f) + v.tcp_message(f) + v.tcp_error(f) + v.tcp_end(f) + assert list(v) == [f] + + def test_filter(): v = view.View() v.request(tft(method="get")) diff --git a/test/mitmproxy/test_http.py b/test/mitmproxy/test_http.py index 8a299d8e..6526b56a 100644 --- a/test/mitmproxy/test_http.py +++ b/test/mitmproxy/test_http.py @@ -254,6 +254,10 @@ class TestHTTPFlow: f.response.decode() assert f.response.raw_content == b"abarb" + def test_timestamp_start(self): + f = tflow.tflow() + assert f.timestamp_start == f.request.timestamp_start + def test_make_error_response(): resp = http.make_error_response(543, 'foobar', Headers()) diff --git a/test/mitmproxy/tools/console/test_common.py b/test/mitmproxy/tools/console/test_common.py index 72438c49..1f59ac4e 100644 --- a/test/mitmproxy/tools/console/test_common.py +++ b/test/mitmproxy/tools/console/test_common.py @@ -5,10 +5,16 @@ from mitmproxy.tools.console import common def test_format_flow(): - f = tflow.tflow(resp=True) - assert common.format_flow(f, True) - assert common.format_flow(f, True, hostheader=True) - assert common.format_flow(f, True, extended=True) + flows = [ + tflow.tflow(resp=True), + tflow.tflow(err=True), + tflow.ttcpflow(), + tflow.ttcpflow(err=True), + ] + for f in flows: + for render_mode in common.RenderMode: + assert common.format_flow(f, render_mode=render_mode) + assert common.format_flow(f, render_mode=render_mode, hostheader=True, focused=False) def test_format_keyvals(): @@ -26,7 +32,7 @@ def test_format_keyvals(): ) ), 1 ) - assert wrapped.render((30, )) + assert wrapped.render((30,)) assert common.format_keyvals( [ ("aa", wrapped) @@ -76,4 +76,4 @@ deps = awscli changedir = docs commands = - ./ci + ./ci.sh |