diff options
127 files changed, 1500 insertions, 698 deletions
diff --git a/.github/ISSUE_TEMPLATE/bug_report.md b/.github/ISSUE_TEMPLATE/bug_report.md new file mode 100644 index 00000000..01b6fb85 --- /dev/null +++ b/.github/ISSUE_TEMPLATE/bug_report.md @@ -0,0 +1,19 @@ +--- +name: Bug report +about: Create a report to help us improve +title: '' +labels: kind/triage +assignees: '' + +--- + +#### Problem Description +A clear and concise description of what the bug is. + +#### Steps to reproduce the behavior: +1. +2. +3. + +#### System Information +Paste the output of "mitmproxy --version" here. diff --git a/.github/ISSUE_TEMPLATE/feature_request.md b/.github/ISSUE_TEMPLATE/feature_request.md new file mode 100644 index 00000000..8e8080db --- /dev/null +++ b/.github/ISSUE_TEMPLATE/feature_request.md @@ -0,0 +1,20 @@ +--- +name: Feature request +about: Suggest an idea for this project +title: '' +labels: kind/feature +assignees: '' + +--- + +**Is your feature request related to a problem? Please describe.** +A clear and concise description of what the problem is. Ex. I'm always frustrated when [...] + +#### Describe the solution you'd like +A clear and concise description of what you want to happen. + +#### Describe alternatives you've considered +A clear and concise description of any alternative solutions or features you've considered. + +#### Additional context +Add any other context or screenshots about the feature request here. @@ -9,6 +9,7 @@ MANIFEST *.egg-info/ .coverage* .idea +.vscode .cache/ .tox*/ build/ diff --git a/.travis.yml b/.travis.yml index dca567bf..035efb79 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -sudo: false language: python branches: @@ -34,7 +33,6 @@ matrix: - python: 3.7 env: TOXENV=py37 dist: xenial - sudo: true # required workaround for https://github.com/travis-ci/travis-ci/issues/9815 - language: node_js node_js: "node" before_install: @@ -53,8 +51,7 @@ matrix: install: - wget https://github.com/gohugoio/hugo/releases/download/v0.41/hugo_0.41_Linux-64bit.deb - sudo dpkg -i hugo*.deb - - pip install tox virtualenv setuptools - - pyenv global system 3.6 + - pip install -U tox virtualenv setuptools script: - tox after_success: @@ -67,11 +64,11 @@ install: brew update || brew update brew outdated pyenv || brew upgrade pyenv eval "$(pyenv init -)" - env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.5 - pyenv global 3.6.5 - pyenv shell 3.6.5 + env PYTHON_CONFIGURE_OPTS="--enable-framework" pyenv install --skip-existing 3.6.9 + pyenv global 3.6.9 + pyenv shell 3.6.9 fi - - pip install tox virtualenv setuptools + - pip install -U tox virtualenv setuptools script: # All these steps MUST succeed for the job to be successful! @@ -5,8 +5,8 @@ mitmproxy This repository contains the **mitmproxy** and **pathod** projects. -``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console -interface. +``mitmproxy`` is an interactive, SSL/TLS-capable intercepting proxy with a console +interface for HTTP/1, HTTP/2, and WebSockets. ``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP. @@ -21,21 +21,18 @@ Documentation & Help -------------------- -General information, tutorials, and precompiled binaries can be found on the mitmproxy -and pathod websites. +General information, tutorials, and precompiled binaries can be found on the mitmproxy website. |mitmproxy_site| The documentation for mitmproxy is available on our website: -|mitmproxy_docs_stable| |mitmproxy_docs_master| +|mitmproxy_docs_stable| |mitmproxy_docs_master| +If you have questions on how to use mitmproxy, please +ask them on StackOverflow! -Join our discussion forum on Discourse to ask questions, help -each other solve problems, and come up with new ideas for the project. - -|mitmproxy_discourse| - +|mitmproxy_stackoverflow| Join our developer chat on Slack if you would like to contribute to mitmproxy itself. @@ -54,7 +51,7 @@ Contributing As an open source project, mitmproxy welcomes contributions of all forms. If you would like to bring the project forward, please consider contributing in the following areas: -- **Maintenance:** We are *incredibly* thankful for individuals who are stepping up and helping with maintenance. This includes (but is not limited to) triaging issues, reviewing pull requests and picking up stale ones, helping out other users in our forums_, creating minimal, complete and verifiable examples or test cases for existing bug reports, updating documentation, or fixing minor bugs that have recently been reported. +- **Maintenance:** We are *incredibly* thankful for individuals who are stepping up and helping with maintenance. This includes (but is not limited to) triaging issues, reviewing pull requests and picking up stale ones, helping out other users on StackOverflow_, creating minimal, complete and verifiable examples or test cases for existing bug reports, updating documentation, or fixing minor bugs that have recently been reported. - **Code Contributions:** We actively mark issues that we consider are `good first contributions`_. If you intend to work on a larger contribution to the project, please come talk to us first. Development Setup @@ -106,7 +103,7 @@ For speedier testing, we recommend you run `pytest`_ directly on individual test .. code-block:: bash cd test/mitmproxy/addons - pytest --cov mitmproxy.addons.anticache --looponfail test_anticache.py + pytest --cov mitmproxy.addons.anticache --cov-report term-missing --looponfail test_anticache.py As pytest does not check the code style, you probably want to run ``tox -e lint`` before committing your changes. @@ -146,21 +143,21 @@ with the following command: tox -e lint -.. |mitmproxy_site| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-mitmproxy.org-blue.svg +.. |mitmproxy_site| image:: https://shields.mitmproxy.org/badge/https%3A%2F%2F-mitmproxy.org-blue.svg :target: https://mitmproxy.org/ :alt: mitmproxy.org -.. |mitmproxy_docs_stable| image:: https://shields.mitmproxy.org/api/docs-stable-brightgreen.svg +.. |mitmproxy_docs_stable| image:: https://shields.mitmproxy.org/badge/docs-stable-brightgreen.svg :target: https://docs.mitmproxy.org/stable/ :alt: mitmproxy documentation stable - -.. |mitmproxy_docs_master| image:: https://shields.mitmproxy.org/api/docs-master-brightgreen.svg + +.. |mitmproxy_docs_master| image:: https://shields.mitmproxy.org/badge/docs-master-brightgreen.svg :target: https://docs.mitmproxy.org/master/ :alt: mitmproxy documentation master -.. |mitmproxy_discourse| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-discourse.mitmproxy.org-orange.svg - :target: https://discourse.mitmproxy.org - :alt: Discourse: mitmproxy +.. |mitmproxy_stackoverflow| image:: https://shields.mitmproxy.org/stackexchange/stackoverflow/t/mitmproxy?color=orange&label=stackoverflow%20questions + :target: https://stackoverflow.com/questions/tagged/mitmproxy + :alt: StackOverflow: mitmproxy .. |slack| image:: http://slack.mitmproxy.org/badge.svg :target: http://slack.mitmproxy.org/ @@ -195,5 +192,5 @@ with the following command: .. _yarn: https://yarnpkg.com/en/ .. _PEP8: https://www.python.org/dev/peps/pep-0008 .. _`Google Style Guide`: https://google.github.io/styleguide/pyguide.html -.. _forums: https://discourse.mitmproxy.org/ +.. _StackOverflow: https://stackoverflow.com/questions/tagged/mitmproxy .. _`good first contributions`: https://github.com/mitmproxy/mitmproxy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 diff --git a/docs/README.md b/docs/README.md index cc06f081..a9ee1113 100644 --- a/docs/README.md +++ b/docs/README.md @@ -7,7 +7,8 @@ This directory houses the mitmproxy documentation available at <https://docs.mit 1. Install [hugo](https://gohugo.io/). 2. Windows users: Depending on your git settings, you may need to manually create a symlink from /docs/src/examples to /examples. - + 3. Make sure the mitmproxy Python package is installed. + 4. Run `./build-current` to generate the documentation source files in `./src/generated`. Now you can run `hugo server -D` in ./src. diff --git a/docs/src/content/_index.md b/docs/src/content/_index.md index cd368df0..6283343d 100644 --- a/docs/src/content/_index.md +++ b/docs/src/content/_index.md @@ -11,8 +11,7 @@ menu: The mitmproxy project's tools are a set of front-ends that expose common underlying functionality. -**mitmproxy** is an interactive man-in-the-middle proxy for HTTP and HTTPS -with a console interface. +**mitmproxy** is an interactive, SSL/TLS-capable intercepting proxy with a console interface for HTTP/1, HTTP/2, and WebSockets. **mitmdump** is the command-line version of mitmproxy. Think tcpdump for HTTP. @@ -21,6 +20,9 @@ with a console interface. Documentation, tutorials and distribution packages can be found on the [mitmproxy website](https://mitmproxy.org). +Development information and our source code can be found in our +[GitHub repository](https://github.com/mitmproxy/mitmproxy). + ## Features @@ -29,8 +31,7 @@ Documentation, tutorials and distribution packages can be found on the - Replay the client-side of an HTTP conversations - Replay HTTP responses of a previously recorded server - Reverse proxy mode to forward traffic to a specified server -- Transparent proxy mode on OSX and Linux +- Transparent proxy mode on macOS and Linux - Make scripted changes to HTTP traffic using Python - SSL/TLS certificates for interception are generated on the fly - And much, much more... - diff --git a/docs/src/content/addons-scripting.md b/docs/src/content/addons-scripting.md index 4e9916ca..6a18eaf4 100644 --- a/docs/src/content/addons-scripting.md +++ b/docs/src/content/addons-scripting.md @@ -27,6 +27,6 @@ You can look at the [http][] module, or the [Request][], and [Response][] classes for other attributes that you can use when scripting. -[http][]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/http.py +[http]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/http.py [Request]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/net/http/request.py [Response]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/net/http/response.py diff --git a/docs/src/content/concepts-certificates.md b/docs/src/content/concepts-certificates.md index 88482047..4e2ae47a 100644 --- a/docs/src/content/concepts-certificates.md +++ b/docs/src/content/concepts-certificates.md @@ -24,6 +24,9 @@ something like this: Click on the relevant icon, follow the setup instructions for the platform you're on and you are good to go. +Note: If you are using an iOS device, you should be using the Safari browser +so that it opens the proper prompts for installing the certificate. + ## Installing the mitmproxy CA certificate manually Sometimes using the quick install app is not an option - Java or the iOS diff --git a/docs/src/content/concepts-protocols.md b/docs/src/content/concepts-protocols.md index fc056545..c79274bf 100644 --- a/docs/src/content/concepts-protocols.md +++ b/docs/src/content/concepts-protocols.md @@ -36,7 +36,7 @@ mitmproxy currently does not support HTTP/2 Cleartext (h2c) since none of the major browser vendors have implemented it. Some websites are still having problems with correct HTTP/2 support in their -webservers and can cause errors, dropped connectiones, or simply no response at +webservers and can cause errors, dropped connections, or simply no response at all. We are trying to be as tolerant and forgiving as possible with the types of data we send and receive, but [some](https://github.com/mitmproxy/mitmproxy/issues/1745) diff --git a/docs/src/content/howto-ignoredomains.md b/docs/src/content/howto-ignoredomains.md index 902a17be..9a337eba 100644 --- a/docs/src/content/howto-ignoredomains.md +++ b/docs/src/content/howto-ignoredomains.md @@ -10,7 +10,7 @@ menu: There are two main reasons why you may want to exempt some traffic from mitmproxy's interception mechanism: -- **Certificate pinning:** Some traffic is is protected using [Certificate +- **Certificate pinning:** Some traffic is protected using [Certificate Pinning](https://security.stackexchange.com/questions/29988/what-is-certificate-pinning) and mitmproxy's interception leads to errors. For example, the Twitter app, Windows Update or the Apple App Store fail to work if mitmproxy is active. diff --git a/docs/src/content/howto-install-system-trusted-ca-android.md b/docs/src/content/howto-install-system-trusted-ca-android.md new file mode 100644 index 00000000..2ef67f30 --- /dev/null +++ b/docs/src/content/howto-install-system-trusted-ca-android.md @@ -0,0 +1,86 @@ +--- +title: "Install System CA on Android" +menu: + howto: + weight: 4 +--- + +# Install System CA Certificate on Android Emulator + +[Since Android 7, apps ignore user certificates](https://android-developers.googleblog.com/2016/07/changes-to-trusted-certificate.html), unless they are configured to use them. +As most applications do not explicitly opt in to use user certificates, we need to place our mitmproxy CA certificate in the system certificate store, +in order to avid having to patch each application, which we want to monitor. + +Please note, that apps can decide to ignore the system certificate store and maintain their own CA certificates. In this case you have to patch the application. + +## 1. Prerequisites + + - Emulator from Android SDK with proxy settings pointing to mitmproxy + + - Mitmproxy CA certificate + - Usually located in `~/.mitmproxy/mitmproxy-ca-cert.cer` + - If the folder is empty or does not exist, run `mitmproxy` in order to generate the certificates + +## 2. Rename certificate +Enter your certificate folder +{{< highlight bash >}} +cd ~/.mitmproxy/ +{{< / highlight >}} + + - CA Certificates in Android are stored by the name of their hash, with a '0' as extension + - Now generate the hash of your certificate + +{{< highlight bash >}} +openssl x509 -inform PEM -subject_hash_old -in mitmproxy-ca-cert.cer | head -1 +{{< / highlight >}} +Lets assume, the output is `c8450d0d` + +We can now copy `mitmproxy-ca-cert.cer` to `c8450d0d.0` and our system certificate is ready to use +{{< highlight bash >}} +cp mitmproxy-ca-cert.cer c8450d0d.0 +{{< / highlight >}} + +## 3. Insert certificate into system certificate store + +Note, that Android 9 (API LEVEL 28) was used to test the following steps and that the `emulator` executable is located in the Android SDK + + - Start your android emulator. + - Get a list of your AVDs with `emulator -list-avds` + - Make sure to use the `-writable-system` option. Otherwise it will not be possible to write to `/system` + - Keep in mind, that the **emulator will load a clean system image when starting without `-writable-system` option**. + - This means you always have to start the emulator with `-writable-system` option in order to use your certificate + +{{< highlight bash >}} +emulator -avd <avd_name_here> -writable-system +{{< / highlight >}} + + - Restart adb as root + +{{< highlight bash >}} +adb root +{{< / highlight >}} + + - Get write access to `/system` on the device + - In earlier versions (API LEVEL < 28) of Android you have to use `adb shell "mount -o rw,remount /system"` + +{{< highlight bash >}} +adb shell "mount -o rw,remount /" +{{< / highlight >}} + + - Push your certificate to the system certificate store and set file permissions + +{{< highlight bash >}} +adb push c8450d0d.0 /system/etc/security/cacerts +adb shell "chmod 664 /system/etc/security/cacerts/c8450d0d.0" +{{< / highlight >}} + +## 4. Reboot device and enjoy decrypted TLS traffic + + - Reboot your device. + - You CA certificate should now be system trusted + +{{< highlight bash >}} +adb reboot +{{< / highlight >}} + +**Remember**: You **always** have to start the emulator using the `-writable-system` option in order to use your certificate
\ No newline at end of file diff --git a/docs/src/content/howto-transparent-vms.md b/docs/src/content/howto-transparent-vms.md index 1446ede7..f251bc44 100644 --- a/docs/src/content/howto-transparent-vms.md +++ b/docs/src/content/howto-transparent-vms.md @@ -14,9 +14,13 @@ Internal Network* setup can be applied to other setups. ## 1. Configure Proxy VM -On the proxy machine, **eth0** is connected to the internet. **eth1** is -connected to the internal network that will be proxified and configured -to use a static ip (192.168.3.1). +First, we have to find out under which name Ubuntu has mapped our network interfaces. You can find this information with: + +{{< highlight bash >}} +ip link +{{< / highlight >}} + +Usually with Ubuntu and Virtualbox, **eth0** or **enp0s3** (Ubuntu 15.10 and newer) is connected to the internet and **eth1** or **enp0s8** (Ubuntu 15.10 and newer) is connected to the internal network that will be proxified and configured to use a static ip (192.168.3.1). If the names differ, use the ones you got from the *ip link* command. ### VirtualBox configuration @@ -65,6 +69,7 @@ Replace **/etc/dnsmasq.conf** with the following configuration: {{< highlight none >}} # Listen for DNS requests on the internal network interface=eth1 +bind-interfaces # Act as a DHCP server, assign IP addresses to clients dhcp-range=192.168.3.10,192.168.3.100,96h # Broadcast gateway and dns server information @@ -93,10 +98,11 @@ IP address via DHCP: ## 3. Redirect traffic to mitmproxy -To redirect traffic to mitmproxy, we need to add two iptables +To redirect traffic to mitmproxy, we need to enable IP forwarding and add two iptables rules: {{< highlight bash >}} +sudo sysctl -w net.ipv4.ip_forward=1 sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 80 -j REDIRECT --to-port 8080 sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-port 8080 {{< / highlight >}} diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index ae36f579..d448bd82 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -50,7 +50,7 @@ a newly created `/etc/sysctl.d/mitmproxy.conf` (see [here](https://superuser.com sysctl -w net.ipv4.conf.all.send_redirects=0 {{< / highlight >}} -If your test device is on the same physical network, your machine shouldn't inform the device that +If your test device is on the same physical network, your machine shouldn't inform the device that there's a shorter route available by skipping the proxy. If you want to persist this across reboots, see above. @@ -83,9 +83,34 @@ The `--mode transparent` option turns on transparent mode, and the `--showhost` ### 5. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the default gateway and +Set the test device up to use the host on which mitmproxy is running as the default gateway and [install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). +### Work-around to redirect traffic originating from the machine itself + +Follow steps **1, 2** as above, but *instead* of the commands in step **3**, run the following + +Create a user to run the mitmproxy + +{{< highlight bash >}} +sudo useradd --create-home mitmproxyuser +sudo -u mitmproxyuser bash -c 'cd ~ && pip install --user mitmproxy' +{{< / highlight >}} + +Then, configure the iptables rules to redirect all traffic from our local machine to mitmproxy. **Note**, as soon as you run these, you won't be able to perform successful network calls *until* you start mitmproxy. If you run into issues, `iptables -t nat -F` is a heavy handed way to flush (clear) *all* the rules from the iptables `nat` table (which includes any other rules you had configured). + +{{< highlight bash >}} +iptables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 80 -j REDIRECT --to-port 8080 +iptables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 443 -j REDIRECT --to-port 8080 +ip6tables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 80 -j REDIRECT --to-port 8080 +ip6tables -t nat -A OUTPUT -p tcp -m owner ! --uid-owner mitmproxyuser --dport 443 -j REDIRECT --to-port 8080 +{{< / highlight >}} + +This will redirect the packets from all users other than `mitmproxyuser` on the machine to mitmproxy. To avoid circularity, run mitmproxy as the user `mitmproxyuser`. Hence step **4** should look like: +{{< highlight bash >}} +sudo -u mitmproxyuser bash -c '$HOME/.local/bin/mitmproxy --mode transparent --showhost --set block_global=false' +{{< / highlight >}} + ## OpenBSD @@ -124,7 +149,7 @@ doas pfctl -e You probably want a command like this: {{< highlight bash >}} -mitmproxy --mode transparent --showhost +mitmproxy --mode transparent --listen-host 127.0.0.1 --showhost {{< / highlight >}} The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells @@ -132,7 +157,7 @@ mitmproxy to use the value of the Host header for URL display. ### 6. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the default gateway and +Set the test device up to use the host on which mitmproxy is running as the default gateway and [install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). @@ -213,7 +238,7 @@ mitmproxy to use the value of the Host header for URL display. ### 7. Finally, configure your test device. -Set the test device up to use the host on which mitmproxy is running as the default gateway and +Set the test device up to use the host on which mitmproxy is running as the default gateway and [install the mitmproxy certificate authority on the test device]({{< relref "concepts-certificates" >}}). {{% note %}} @@ -229,7 +254,7 @@ for more. ### Work-around to redirect traffic originating from the machine itself -Follow the steps **1, 2** as above. In step **3** change the contents of the file **pf.conf** to +Follow steps **1, 2** as above, but in step **2** change the contents of the file **pf.conf** to {{< highlight none >}} #The ports to redirect to proxy @@ -246,18 +271,12 @@ tproxy_user = "nobody" #This cannot involve the user which runs the #transparent proxy as that would cause an infinite loop. # -#Here we redirect for all users which don't run transparent proxy. -redir_users = "{ !=" $tproxy_user "}" - -#If you only wish to redirect traffic for particular users -#you may also do: -#redir_users = "{= john, = jane}" rdr pass proto tcp from any to any port $redir_ports -> $tproxy -pass out route-to (lo0 127.0.0.1) proto tcp from any to any port $redir_ports user $redir_users +pass out route-to (lo0 127.0.0.1) proto tcp from any to any port $redir_ports user { != $tproxy_user } {{< / highlight >}} -Follow steps **4-6** above. This will redirect the packets from all users other than `nobody` on the machine to mitmproxy. To avoid circularity, run mitmproxy as the user `nobody`. Hence step **7** should look like: +Follow steps **3-5** above. This will redirect the packets from all users other than `nobody` on the machine to mitmproxy. To avoid circularity, run mitmproxy as the user `nobody`. Hence step **6** should look like: {{< highlight bash >}} sudo -u nobody mitmproxy --mode transparent --showhost diff --git a/docs/src/content/howto-wireshark-tls.md b/docs/src/content/howto-wireshark-tls.md index 588223ac..a55d177b 100644 --- a/docs/src/content/howto-wireshark-tls.md +++ b/docs/src/content/howto-wireshark-tls.md @@ -7,7 +7,7 @@ menu: # Wireshark and SSL/TLS Master Secrets -The SSL/SSL master keys can be logged by mitmproxy so that external programs can +The SSL/TLS master keys can be logged by mitmproxy so that external programs can decrypt SSL/TLS connections both from and to the proxy. Recent versions of Wireshark can use these log files to decrypt packets. See the [Wireshark wiki](https://wiki.wireshark.org/SSL#Using_the_.28Pre.29-Master-Secret) for more information. diff --git a/docs/src/content/tute-highscores.md b/docs/src/content/tute-highscores.md index f5cbd7bc..2d03076d 100644 --- a/docs/src/content/tute-highscores.md +++ b/docs/src/content/tute-highscores.md @@ -67,7 +67,7 @@ timestamp. Looks pretty simple to mess with. Lets edit the score submission. First, select it in mitmproxy, then press <span data-role="kbd">enter</span> to view it. Make sure you're -viewing the request, not the response -you can use +viewing the request, not the response - you can use <span data-role="kbd">tab</span> to flick between the two. Now press <span data-role="kbd">e</span> for edit. You'll be prompted for the part of the request you want to change - press <span data-role="kbd">r</span> diff --git a/test/mitmproxy/addons/onboardingapp/__init__.py b/examples/__init__.py index e69de29b..e69de29b 100644 --- a/test/mitmproxy/addons/onboardingapp/__init__.py +++ b/examples/__init__.py diff --git a/examples/addons/commands-paths.py b/examples/addons/commands-paths.py index f37a0fbc..4d9535b9 100644 --- a/examples/addons/commands-paths.py +++ b/examples/addons/commands-paths.py @@ -20,9 +20,9 @@ class MyAddon: for f in flows: totals[f.request.host] = totals.setdefault(f.request.host, 0) + 1 - fp = open(path, "w+") - for cnt, dom in sorted([(v, k) for (k, v) in totals.items()]): - fp.write("%s: %s\n" % (cnt, dom)) + with open(path, "w+") as fp: + for cnt, dom in sorted([(v, k) for (k, v) in totals.items()]): + fp.write("%s: %s\n" % (cnt, dom)) ctx.log.alert("done") diff --git a/examples/complex/__init__.py b/examples/complex/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/examples/complex/__init__.py diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index 33a2f79f..e3cea9fd 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -87,7 +87,10 @@ def response(flow): } # HAR timings are integers in ms, so we re-encode the raw timings to that format. - timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()]) + timings = { + k: int(1000 * v) if v != -1 else -1 + for k, v in timings_raw.items() + } # full_time is the sum of all timings. # Timings set to -1 will be ignored as per spec. diff --git a/examples/complex/sslstrip.py b/examples/complex/sslstrip.py index c862536f..69b9ea9e 100644 --- a/examples/complex/sslstrip.py +++ b/examples/complex/sslstrip.py @@ -38,7 +38,7 @@ def response(flow: http.HTTPFlow) -> None: flow.response.content = flow.response.content.replace(b'https://', b'http://') # strip meta tag upgrade-insecure-requests in response body - csp_meta_tag_pattern = b'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>' + csp_meta_tag_pattern = br'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>' flow.response.content = re.sub(csp_meta_tag_pattern, b'', flow.response.content, flags=re.IGNORECASE) # strip links in 'Location' header @@ -52,7 +52,7 @@ def response(flow: http.HTTPFlow) -> None: # strip upgrade-insecure-requests in Content-Security-Policy header if re.search('upgrade-insecure-requests', flow.response.headers.get('Content-Security-Policy', ''), flags=re.IGNORECASE): csp = flow.response.headers['Content-Security-Policy'] - flow.response.headers['Content-Security-Policy'] = re.sub('upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) + flow.response.headers['Content-Security-Policy'] = re.sub(r'upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) # strip secure flag from 'Set-Cookie' headers cookies = flow.response.headers.get_all('Set-Cookie') diff --git a/examples/complex/xss_scanner.py b/examples/complex/xss_scanner.py index cdaaf478..d5f4aaab 100755 --- a/examples/complex/xss_scanner.py +++ b/examples/complex/xss_scanner.py @@ -1,4 +1,4 @@ -""" +r""" __ __ _____ _____ _____ \ \ / // ____/ ____| / ____| @@ -86,7 +86,7 @@ def get_cookies(flow: http.HTTPFlow) -> Cookies: return {name: value for name, value in flow.request.cookies.fields} -def find_unclaimed_URLs(body: str, requestUrl: bytes) -> None: +def find_unclaimed_URLs(body, requestUrl): """ Look for unclaimed URLs in script tags and log them if found""" def getValue(attrs: List[Tuple[str, str]], attrName: str) -> Optional[str]: for name, value in attrs: @@ -111,7 +111,7 @@ def find_unclaimed_URLs(body: str, requestUrl: bytes) -> None: try: socket.gethostbyname(domain) except socket.gaierror: - ctx.log.error("XSS found in %s due to unclaimed URL \"%s\"." % (requestUrl, url)) + ctx.log.error(f"XSS found in {requestUrl} due to unclaimed URL \"{url}\".") def test_end_of_URL_injection(original_body: str, request_URL: str, cookies: Cookies) -> VulnData: diff --git a/issue_template.md b/issue_template.md index 2ea213a5..3dbac2ac 100644 --- a/issue_template.md +++ b/issue_template.md @@ -14,4 +14,4 @@ <!-- Paste the output of "mitmproxy --version" here. --> -<!-- Please use the mitmproxy forums (https://discourse.mitmproxy.org/) for support/how-to questions. Thanks! :) --> +<!-- Please use StackOverflow (https://stackoverflow.com/questions/tagged/mitmproxy) for support/how-to questions. Thanks! :) --> diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py index 4214d6ea..8a565a73 100644 --- a/mitmproxy/addonmanager.py +++ b/mitmproxy/addonmanager.py @@ -184,7 +184,7 @@ class AddonManager: raise exceptions.AddonManagerError("No such addon: %s" % n) self.chain = [i for i in self.chain if i is not a] del self.lookup[_get_name(a)] - self.invoke_addon(a, "done") + self.invoke_addon(addon, "done") def __len__(self): return len(self.chain) diff --git a/mitmproxy/addons/block.py b/mitmproxy/addons/block.py index 91f9f709..4ccde0e1 100644 --- a/mitmproxy/addons/block.py +++ b/mitmproxy/addons/block.py @@ -36,4 +36,4 @@ class Block: layer.reply.kill() if ctx.options.block_global and address.is_global: ctx.log.warn("Client connection from %s killed by block_global" % astr) - layer.reply.kill()
\ No newline at end of file + layer.reply.kill() diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index c56c0e74..7bdaeb33 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -203,8 +203,9 @@ class ClientPlayback: # https://github.com/mitmproxy/mitmproxy/issues/2197 if hf.request.http_version == "HTTP/2.0": hf.request.http_version = "HTTP/1.1" - host = hf.request.headers.pop(":authority") - hf.request.headers.insert(0, "host", host) + host = hf.request.headers.pop(":authority", None) + if host is not None: + hf.request.headers.insert(0, "host", host) self.q.put(hf) ctx.master.addons.trigger("update", lst) diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index a908dbb3..5c9bbcd0 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -289,7 +289,7 @@ class Core: """ The possible values for an encoding specification. """ - return ["gzip", "deflate", "br"] + return ["gzip", "deflate", "br", "zstd"] @command.command("options.load") def options_load(self, path: mitmproxy.types.Path) -> None: diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py index 6bb52e84..9aff2878 100644 --- a/mitmproxy/addons/cut.py +++ b/mitmproxy/addons/cut.py @@ -126,20 +126,18 @@ class Cut: format is UTF-8 encoded CSV. If there is exactly one row and one column, the data is written to file as-is, with raw bytes preserved. """ + v: typing.Union[str, bytes] fp = io.StringIO(newline="") if len(cuts) == 1 and len(flows) == 1: v = extract(cuts[0], flows[0]) - if isinstance(v, bytes): - fp.write(strutils.always_str(v)) - else: - fp.write(v) + fp.write(strutils.always_str(v)) # type: ignore ctx.log.alert("Clipped single cut.") else: writer = csv.writer(fp) for f in flows: vals = [extract(c, f) for c in cuts] writer.writerow( - [strutils.always_str(v) or "" for v in vals] # type: ignore + [strutils.always_str(v) for v in vals] ) ctx.log.alert("Clipped %s cuts as CSV." % len(cuts)) try: diff --git a/mitmproxy/addons/eventstore.py b/mitmproxy/addons/eventstore.py index 50fea7ab..188a3b39 100644 --- a/mitmproxy/addons/eventstore.py +++ b/mitmproxy/addons/eventstore.py @@ -14,7 +14,7 @@ class EventStore: self.sig_refresh = blinker.Signal() @property - def size(self) -> int: + def size(self) -> typing.Optional[int]: return self.data.maxlen def log(self, entry: LogEntry) -> None: diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 90e95d3e..2776118a 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -11,17 +11,23 @@ import mitmproxy.types import pyperclip -def raise_if_missing_request(f: flow.Flow) -> None: +def cleanup_request(f: flow.Flow): if not hasattr(f, "request"): raise exceptions.CommandError("Can't export flow with no request.") + request = f.request.copy() # type: ignore + request.decode(strict=False) + # a bit of clean-up + if request.method == 'GET' and request.headers.get("content-length", None) == "0": + request.headers.pop('content-length') + request.headers.pop(':authority', None) + return request def curl_command(f: flow.Flow) -> str: - raise_if_missing_request(f) data = "curl " - request = f.request.copy() # type: ignore - request.decode(strict=False) + request = cleanup_request(f) for k, v in request.headers.items(multi=True): + data += "--compressed " if k == 'accept-encoding' else "" data += "-H '%s:%s' " % (k, v) if request.method != "GET": data += "-X %s " % request.method @@ -35,11 +41,8 @@ def curl_command(f: flow.Flow) -> str: def httpie_command(f: flow.Flow) -> str: - raise_if_missing_request(f) - request = f.request.copy() # type: ignore - data = "http %s " % request.method - request.decode(strict=False) - data += "%s" % request.url + request = cleanup_request(f) + data = "http %s %s" % (request.method, request.url) for k, v in request.headers.items(multi=True): data += " '%s:%s'" % (k, v) if request.content: @@ -51,8 +54,7 @@ def httpie_command(f: flow.Flow) -> str: def raw(f: flow.Flow) -> bytes: - raise_if_missing_request(f) - return assemble.assemble_request(f.request) # type: ignore + return assemble.assemble_request(cleanup_request(f)) # type: ignore formats = dict( diff --git a/mitmproxy/addons/onboarding.py b/mitmproxy/addons/onboarding.py index 900acb08..94ca7c49 100644 --- a/mitmproxy/addons/onboarding.py +++ b/mitmproxy/addons/onboarding.py @@ -10,7 +10,7 @@ class Onboarding(wsgiapp.WSGIApp): name = "onboarding" def __init__(self): - super().__init__(app.Adapter(app.application), None, None) + super().__init__(app, None, None) def load(self, loader): loader.add_option( @@ -32,6 +32,7 @@ class Onboarding(wsgiapp.WSGIApp): def configure(self, updated): self.host = ctx.options.onboarding_host self.port = ctx.options.onboarding_port + app.config["CONFDIR"] = ctx.options.confdir def request(self, f): if ctx.options.onboarding: diff --git a/mitmproxy/addons/onboardingapp/__init__.py b/mitmproxy/addons/onboardingapp/__init__.py index e69de29b..722fed03 100644 --- a/mitmproxy/addons/onboardingapp/__init__.py +++ b/mitmproxy/addons/onboardingapp/__init__.py @@ -0,0 +1,37 @@ +import os + +from flask import Flask, render_template + +from mitmproxy.options import CONF_BASENAME, CONF_DIR + +app = Flask(__name__) +# will be overridden in the addon, setting this here so that the Flask app can be run standalone. +app.config["CONFDIR"] = CONF_DIR + + +@app.route('/') +def index(): + return render_template("index.html") + + +@app.route('/cert/pem') +def pem(): + return read_cert("pem", "application/x-x509-ca-cert") + + +@app.route('/cert/p12') +def p12(): + return read_cert("p12", "application/x-pkcs12") + + +def read_cert(ext, content_type): + filename = CONF_BASENAME + f"-ca-cert.{ext}" + p = os.path.join(app.config["CONFDIR"], filename) + p = os.path.expanduser(p) + with open(p, "rb") as f: + cert = f.read() + + return cert, { + "Content-Type": content_type, + "Content-Disposition": f"inline; filename={filename}", + } diff --git a/mitmproxy/addons/onboardingapp/app.py b/mitmproxy/addons/onboardingapp/app.py deleted file mode 100644 index ab136778..00000000 --- a/mitmproxy/addons/onboardingapp/app.py +++ /dev/null @@ -1,118 +0,0 @@ -import os - -import tornado.template -import tornado.web -import tornado.wsgi - -from mitmproxy.utils import data -from mitmproxy.proxy import config - -loader = tornado.template.Loader(data.pkg_data.path("addons/onboardingapp/templates")) - - -class Adapter(tornado.wsgi.WSGIAdapter): - # Tornado doesn't make the WSGI environment available to pages, so this - # hideous monkey patch is the easiest way to get to the mitmproxy.master - # variable. - - def __init__(self, application): - self._application = application - - def application(self, request): - request.master = self.environ["mitmproxy.master"] - return self._application(request) - - def __call__(self, environ, start_response): - self.environ = environ - return tornado.wsgi.WSGIAdapter.__call__( - self, - environ, - start_response - ) - - -class Index(tornado.web.RequestHandler): - - def get(self): - t = loader.load("index.html") - self.write(t.generate()) - - -class PEM(tornado.web.RequestHandler): - - @property - def filename(self): - return config.CONF_BASENAME + "-ca-cert.pem" - - def head(self): - p = os.path.join(self.request.master.options.confdir, self.filename) - p = os.path.expanduser(p) - content_length = os.path.getsize(p) - - self.set_header("Content-Type", "application/x-x509-ca-cert") - self.set_header( - "Content-Disposition", - "inline; filename={}".format( - self.filename)) - self.set_header("Content-Length", content_length) - - def get(self): - p = os.path.join(self.request.master.options.confdir, self.filename) - p = os.path.expanduser(p) - self.set_header("Content-Type", "application/x-x509-ca-cert") - self.set_header( - "Content-Disposition", - "inline; filename={}".format( - self.filename)) - - with open(p, "rb") as f: - self.write(f.read()) - - -class P12(tornado.web.RequestHandler): - - @property - def filename(self): - return config.CONF_BASENAME + "-ca-cert.p12" - - def head(self): - p = os.path.join(self.request.master.options.confdir, self.filename) - p = os.path.expanduser(p) - content_length = os.path.getsize(p) - - self.set_header("Content-Type", "application/x-pkcs12") - self.set_header( - "Content-Disposition", - "inline; filename={}".format( - self.filename)) - - self.set_header("Content-Length", content_length) - - def get(self): - p = os.path.join(self.request.master.options.confdir, self.filename) - p = os.path.expanduser(p) - self.set_header("Content-Type", "application/x-pkcs12") - self.set_header( - "Content-Disposition", - "inline; filename={}".format( - self.filename)) - - with open(p, "rb") as f: - self.write(f.read()) - - -application = tornado.web.Application( - [ - (r"/", Index), - (r"/cert/pem", PEM), - (r"/cert/p12", P12), - ( - r"/static/(.*)", - tornado.web.StaticFileHandler, - { - "path": data.pkg_data.path("addons/onboardingapp/static") - } - ), - ], - # debug=True -) diff --git a/mitmproxy/addons/onboardingapp/static/mitmproxy.css b/mitmproxy/addons/onboardingapp/static/mitmproxy.css index 969bd62b..e654d56b 100644 --- a/mitmproxy/addons/onboardingapp/static/mitmproxy.css +++ b/mitmproxy/addons/onboardingapp/static/mitmproxy.css @@ -15,7 +15,7 @@ height: 300px; } -.bigtitle>div { +.bigtitle > div { display: table-cell; vertical-align: middle; } @@ -31,7 +31,7 @@ section { .innerlink { text-decoration: none; - border-bottom:1px dotted; + border-bottom: 1px dotted; margin-bottom: 15px; } diff --git a/mitmproxy/addons/onboardingapp/templates/frame.html b/mitmproxy/addons/onboardingapp/templates/frame.html index f00e1a66..13003f3c 100644 --- a/mitmproxy/addons/onboardingapp/templates/frame.html +++ b/mitmproxy/addons/onboardingapp/templates/frame.html @@ -3,7 +3,7 @@ <div class="row"> <div class="span12"> {% block body %} - {% end %} + {% endblock %} </div> </div> -{% end %} +{% endblock %} diff --git a/mitmproxy/addons/onboardingapp/templates/index.html b/mitmproxy/addons/onboardingapp/templates/index.html index 38aa27ed..aa471668 100644 --- a/mitmproxy/addons/onboardingapp/templates/index.html +++ b/mitmproxy/addons/onboardingapp/templates/index.html @@ -135,19 +135,19 @@ function changeTo(device) { <h2 class="text-center"> Click to install your mitmproxy certificate </h2> <div id="certbank" class="row"> <div class="col-md-3"> - <a onclick="changeTo('apple')" href="/cert/pem"><i class="fa fa-apple fa-5x"></i></a> + <a target="_blank" onclick="changeTo('apple')" href="/cert/pem"><i class="fa fa-apple fa-5x"></i></a> <p>Apple</p> </div> <div class="col-md-3"> - <a onclick="changeTo('windows')" href="/cert/p12"><i class="fa fa-windows fa-5x"></i></a> + <a target="_blank" onclick="changeTo('windows')" href="/cert/p12"><i class="fa fa-windows fa-5x"></i></a> <p>Windows</p> </div> <div class="col-md-3"> - <a onclick="changeTo('android')" href="/cert/pem"><i class="fa fa-android fa-5x"></i></a> + <a target="_blank" onclick="changeTo('android')" href="/cert/pem"><i class="fa fa-android fa-5x"></i></a> <p>Android</p> </div> <div class="col-md-3"> - <a onclick="changeTo('asterisk')" href="/cert/pem"><i class="fa fa-asterisk fa-5x"></i></a> + <a target="_blank" onclick="changeTo('asterisk')" href="/cert/pem"><i class="fa fa-asterisk fa-5x"></i></a> <p>Other</p> </div> </div> @@ -167,4 +167,4 @@ function changeTo(device) { between mitmproxy installations. </div> -{% end %} +{% endblock %} diff --git a/mitmproxy/addons/onboardingapp/templates/layout.html b/mitmproxy/addons/onboardingapp/templates/layout.html index f6e1b286..cea8373b 100644 --- a/mitmproxy/addons/onboardingapp/templates/layout.html +++ b/mitmproxy/addons/onboardingapp/templates/layout.html @@ -28,7 +28,7 @@ <div class="container"> {% block content %} - {% end %} + {% endblock %} </div> </body> diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index a39ce5ce..3b2568c9 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -16,7 +16,7 @@ from mitmproxy import ctx import mitmproxy.types as mtypes -def load_script(path: str) -> types.ModuleType: +def load_script(path: str) -> typing.Optional[types.ModuleType]: fullname = "__mitmproxy_script__.{}".format( os.path.splitext(os.path.basename(path))[0] ) diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 51ba60b4..0818696f 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -68,6 +68,13 @@ class ServerPlayback: to replay. """ ) + loader.add_option( + "server_replay_ignore_port", bool, False, + """ + Ignore request's destination port while searching for a saved flow + to replay. + """ + ) @command.command("replay.server") def load_flows(self, flows: typing.Sequence[flow.Flow]) -> None: @@ -110,7 +117,7 @@ class ServerPlayback: _, _, path, _, query, _ = urllib.parse.urlparse(r.url) queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) - key: typing.List[typing.Any] = [str(r.port), str(r.scheme), str(r.method), str(path)] + key: typing.List[typing.Any] = [str(r.scheme), str(r.method), str(path)] if not ctx.options.server_replay_ignore_content: if ctx.options.server_replay_ignore_payload_params and r.multipart_form: key.extend( @@ -129,6 +136,8 @@ class ServerPlayback: if not ctx.options.server_replay_ignore_host: key.append(r.host) + if not ctx.options.server_replay_ignore_port: + key.append(r.port) filtered = [] ignore_params = ctx.options.server_replay_ignore_params or [] diff --git a/mitmproxy/addons/session.py b/mitmproxy/addons/session.py index 63e382ec..6636b500 100644 --- a/mitmproxy/addons/session.py +++ b/mitmproxy/addons/session.py @@ -87,8 +87,8 @@ class SessionDB: def _create_session(self): script_path = pkg_data.path("io/sql/session_create.sql") - qry = open(script_path, 'r').read() - self.con.executescript(qry) + with open(script_path, 'r') as qry: + self.con.executescript(qry.read()) self.con.commit() @staticmethod @@ -215,8 +215,8 @@ class Session: def __init__(self): self.db_store: SessionDB = None self._hot_store: collections.OrderedDict = collections.OrderedDict() - self._order_store: typing.Dict[str, typing.Dict[str, typing.Union[int, float, str]]] = {} - self._view: typing.List[typing.Tuple[typing.Union[int, float, str], str]] = [] + self._order_store: typing.Dict[str, typing.Dict[str, typing.Union[int, float, str, None]]] = {} + self._view: typing.List[typing.Tuple[typing.Union[int, float, str, None], str]] = [] self.order: str = orders[0] self.filter = matchall self._flush_period: float = self._FP_DEFAULT diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index fd530aaa..1651c1f6 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -53,6 +53,7 @@ class StickyCookie: self.flt = None def response(self, flow: http.HTTPFlow): + assert flow.response if self.flt: for name, (value, attrs) in flow.response.cookies.items(multi=True): # FIXME: We now know that Cookie.py screws up some cookies with diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 8d27840f..da9d19f9 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -238,18 +238,24 @@ class View(collections.abc.Sequence): """ Set focus to the next flow. """ - idx = self.focus.index + 1 - if self.inbounds(idx): - self.focus.flow = self[idx] + if self.focus.index is not None: + idx = self.focus.index + 1 + if self.inbounds(idx): + self.focus.flow = self[idx] + else: + pass @command.command("view.focus.prev") def focus_prev(self) -> None: """ Set focus to the previous flow. """ - idx = self.focus.index - 1 - if self.inbounds(idx): - self.focus.flow = self[idx] + if self.focus.index is not None: + idx = self.focus.index - 1 + if self.inbounds(idx): + self.focus.flow = self[idx] + else: + pass # Order @command.command("view.order.options") @@ -584,7 +590,7 @@ class Focus: """ def __init__(self, v: View) -> None: self.view = v - self._flow: mitmproxy.flow.Flow = None + self._flow: typing.Optional[mitmproxy.flow.Flow] = None self.sig_change = blinker.Signal() if len(self.view): self.flow = self.view[0] diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index 6f5f8c09..d574c027 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -36,9 +36,9 @@ rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI= """ -def create_ca(organization, cn, exp): +def create_ca(organization, cn, exp, key_size): key = OpenSSL.crypto.PKey() - key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048) + key.generate_key(OpenSSL.crypto.TYPE_RSA, key_size) cert = OpenSSL.crypto.X509() cert.set_serial_number(int(time.time() * 10000)) cert.set_version(2) @@ -115,6 +115,13 @@ def dummy_cert(privkey, cacert, commonname, sans, organization): cert.set_version(2) cert.add_extensions( [OpenSSL.crypto.X509Extension(b"subjectAltName", False, ss)]) + cert.add_extensions([ + OpenSSL.crypto.X509Extension( + b"extendedKeyUsage", + False, + b"serverAuth,clientAuth" + ) + ]) cert.set_pubkey(cacert.get_pubkey()) cert.sign(privkey, "sha256") return Cert(cert) @@ -182,10 +189,10 @@ class CertStore: return dh @classmethod - def from_store(cls, path, basename): + def from_store(cls, path, basename, key_size): ca_path = os.path.join(path, basename + "-ca.pem") if not os.path.exists(ca_path): - key, ca = cls.create_store(path, basename) + key, ca = cls.create_store(path, basename, key_size) else: with open(ca_path, "rb") as f: raw = f.read() @@ -215,14 +222,14 @@ class CertStore: os.umask(original_umask) @staticmethod - def create_store(path, basename, organization=None, cn=None, expiry=DEFAULT_EXP): + def create_store(path, basename, key_size, organization=None, cn=None, expiry=DEFAULT_EXP): if not os.path.exists(path): os.makedirs(path) organization = organization or basename cn = cn or basename - key, ca = create_ca(organization=organization, cn=cn, exp=expiry) + key, ca = create_ca(organization=organization, cn=cn, exp=expiry, key_size=key_size) # Dump the CA plus private key with CertStore.umask_secret(), open(os.path.join(path, basename + "-ca.pem"), "wb") as f: f.write( @@ -308,7 +315,12 @@ class CertStore: ret.append(b"*." + b".".join(parts[i:])) return ret - def get_cert(self, commonname: typing.Optional[bytes], sans: typing.List[bytes], organization: typing.Optional[bytes] = None): + def get_cert( + self, + commonname: typing.Optional[bytes], + sans: typing.List[bytes], + organization: typing.Optional[bytes] = None + ) -> typing.Tuple["Cert", OpenSSL.SSL.PKey, str]: """ Returns an (cert, privkey, cert_chain) tuple. diff --git a/mitmproxy/command.py b/mitmproxy/command.py index 27f0921d..0998601c 100644 --- a/mitmproxy/command.py +++ b/mitmproxy/command.py @@ -44,6 +44,8 @@ def typename(t: type) -> str: class Command: + returntype: typing.Optional[typing.Type] + def __init__(self, manager, path, func) -> None: self.path = path self.manager = manager @@ -177,7 +179,7 @@ class CommandManager(mitmproxy.types._CommandBase): parse: typing.List[ParseResult] = [] params: typing.List[type] = [] - typ: typing.Type = None + typ: typing.Type for i in range(len(parts)): if i == 0: typ = mitmproxy.types.Cmd diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index 01c6d221..1e71d942 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -135,7 +135,9 @@ def get_content_view(viewmode: View, data: bytes, **metadata): # Third-party viewers can fail in unexpected ways... except Exception: desc = "Couldn't parse: falling back to Raw" - _, content = get("Raw")(data, **metadata) + raw = get("Raw") + assert raw + content = raw(data, **metadata)[1] error = "{} Content viewer failed: \n{}".format( getattr(viewmode, "name"), traceback.format_exc() diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index 6072dfb7..81f2e487 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -9,8 +9,8 @@ TViewResult = typing.Tuple[str, typing.Iterator[TViewLine]] class View: - name: str = None - content_types: typing.List[str] = [] + name: typing.ClassVar[str] + content_types: typing.ClassVar[typing.List[str]] = [] def __call__(self, data: bytes, **metadata) -> TViewResult: """ @@ -37,7 +37,7 @@ class View: def format_pairs( items: typing.Iterable[typing.Tuple[TTextType, TTextType]] -)-> typing.Iterator[TViewLine]: +) -> typing.Iterator[TViewLine]: """ Helper function that accepts a list of (k,v) pairs into a list of diff --git a/mitmproxy/contentviews/css.py b/mitmproxy/contentviews/css.py index cbe8ce62..44b33761 100644 --- a/mitmproxy/contentviews/css.py +++ b/mitmproxy/contentviews/css.py @@ -16,7 +16,7 @@ A custom CSS prettifier. Compared to other prettifiers, its main features are: CSS_SPECIAL_AREAS = ( "'" + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + "'", '"' + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + '"', - r"/\*" + strutils.MULTILINE_CONTENT + "\*/", + r"/\*" + strutils.MULTILINE_CONTENT + r"\*/", "//" + strutils.SINGLELINE_CONTENT + "$" ) CSS_SPECIAL_CHARS = "{};:" diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py index fcc50cb5..d5bb404f 100644 --- a/mitmproxy/contentviews/image/image_parser.py +++ b/mitmproxy/contentviews/image/image_parser.py @@ -54,7 +54,7 @@ def parse_gif(data: bytes) -> Metadata: entries = block.body.body.entries for entry in entries: comment = entry.bytes - if comment is not b'': + if comment != b'': parts.append(('comment', str(comment))) return parts diff --git a/mitmproxy/contentviews/javascript.py b/mitmproxy/contentviews/javascript.py index 1440ea5d..b5f09150 100644 --- a/mitmproxy/contentviews/javascript.py +++ b/mitmproxy/contentviews/javascript.py @@ -10,9 +10,9 @@ SPECIAL_AREAS = ( r"'" + strutils.MULTILINE_CONTENT_LINE_CONTINUATION + strutils.NO_ESCAPE + "'", r'"' + strutils.MULTILINE_CONTENT_LINE_CONTINUATION + strutils.NO_ESCAPE + '"', r'`' + strutils.MULTILINE_CONTENT + strutils.NO_ESCAPE + '`', - r"/\*" + strutils.MULTILINE_CONTENT + "\*/", + r"/\*" + strutils.MULTILINE_CONTENT + r"\*/", r"//" + strutils.SINGLELINE_CONTENT + "$", - r"for\(" + strutils.SINGLELINE_CONTENT + "\)", + r"for\(" + strutils.SINGLELINE_CONTENT + r"\)", ) diff --git a/mitmproxy/contentviews/xml_html.py b/mitmproxy/contentviews/xml_html.py index 658fbcd7..f2fa47cb 100644 --- a/mitmproxy/contentviews/xml_html.py +++ b/mitmproxy/contentviews/xml_html.py @@ -1,7 +1,7 @@ import io import re import textwrap -from typing import Iterable +from typing import Iterable, Optional from mitmproxy.contentviews import base from mitmproxy.utils import sliding_window @@ -18,7 +18,7 @@ The implementation is split into two main parts: tokenization and formatting of """ # http://www.xml.com/pub/a/2001/07/25/namingparts.html - this is close enough for what we do. -REGEX_TAG = re.compile("[a-zA-Z0-9._:\-]+(?!=)") +REGEX_TAG = re.compile(r"[a-zA-Z0-9._:\-]+(?!=)") # https://www.w3.org/TR/html5/syntax.html#void-elements HTML_VOID_ELEMENTS = { "area", "base", "br", "col", "embed", "hr", "img", "input", "keygen", "link", "meta", "param", @@ -124,14 +124,14 @@ def indent_text(data: str, prefix: str) -> str: return textwrap.indent(dedented, prefix[:32]) -def is_inline_text(a: Token, b: Token, c: Token) -> bool: +def is_inline_text(a: Optional[Token], b: Optional[Token], c: Optional[Token]) -> bool: if isinstance(a, Tag) and isinstance(b, Text) and isinstance(c, Tag): if a.is_opening and "\n" not in b.data and c.is_closing and a.tag == c.tag: return True return False -def is_inline(prev2: Token, prev1: Token, t: Token, next1: Token, next2: Token) -> bool: +def is_inline(prev2: Optional[Token], prev1: Optional[Token], t: Optional[Token], next1: Optional[Token], next2: Optional[Token]) -> bool: if isinstance(t, Text): return is_inline_text(prev1, t, next1) elif isinstance(t, Tag): diff --git a/mitmproxy/contrib/kaitaistruct/exif_be.py b/mitmproxy/contrib/kaitaistruct/exif_be.py index 8a6e7a2b..88ce4e54 100644 --- a/mitmproxy/contrib/kaitaistruct/exif_be.py +++ b/mitmproxy/contrib/kaitaistruct/exif_be.py @@ -1,12 +1,8 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -import array -import struct -import zlib -from enum import Enum from pkg_resources import parse_version - from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +from enum import Enum if parse_version(ks_version) < parse_version('0.7'): @@ -17,6 +13,9 @@ class ExifBe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.version = self._io.read_u2be() self.ifd0_ofs = self._io.read_u4be() @@ -25,6 +24,9 @@ class ExifBe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.num_fields = self._io.read_u2be() self.fields = [None] * (self.num_fields) for i in range(self.num_fields): @@ -54,6 +56,9 @@ class ExifBe(KaitaiStruct): word = 3 dword = 4 rational = 5 + undefined = 7 + slong = 9 + srational = 10 class TagEnum(Enum): image_width = 256 @@ -518,6 +523,9 @@ class ExifBe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.tag = self._root.IfdField.TagEnum(self._io.read_u2be()) self.field_type = self._root.IfdField.FieldTypeEnum(self._io.read_u2be()) self.length = self._io.read_u4be() @@ -552,7 +560,7 @@ class ExifBe(KaitaiStruct): if hasattr(self, '_m_data'): return self._m_data if hasattr(self, '_m_data') else None - if not self.is_immediate_data: + if not (self.is_immediate_data): io = self._root._io _pos = io.pos() io.seek(self.ofs_or_data) diff --git a/mitmproxy/contrib/kaitaistruct/exif_le.py b/mitmproxy/contrib/kaitaistruct/exif_le.py index 84e53a38..e25a2fc9 100644 --- a/mitmproxy/contrib/kaitaistruct/exif_le.py +++ b/mitmproxy/contrib/kaitaistruct/exif_le.py @@ -1,12 +1,8 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -import array -import struct -import zlib -from enum import Enum from pkg_resources import parse_version - from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +from enum import Enum if parse_version(ks_version) < parse_version('0.7'): @@ -17,6 +13,9 @@ class ExifLe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.version = self._io.read_u2le() self.ifd0_ofs = self._io.read_u4le() @@ -25,6 +24,9 @@ class ExifLe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.num_fields = self._io.read_u2le() self.fields = [None] * (self.num_fields) for i in range(self.num_fields): @@ -54,6 +56,9 @@ class ExifLe(KaitaiStruct): word = 3 dword = 4 rational = 5 + undefined = 7 + slong = 9 + srational = 10 class TagEnum(Enum): image_width = 256 @@ -518,6 +523,9 @@ class ExifLe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.tag = self._root.IfdField.TagEnum(self._io.read_u2le()) self.field_type = self._root.IfdField.FieldTypeEnum(self._io.read_u2le()) self.length = self._io.read_u4le() @@ -552,7 +560,7 @@ class ExifLe(KaitaiStruct): if hasattr(self, '_m_data'): return self._m_data if hasattr(self, '_m_data') else None - if not self.is_immediate_data: + if not (self.is_immediate_data): io = self._root._io _pos = io.pos() io.seek(self.ofs_or_data) diff --git a/mitmproxy/contrib/wbxml/ASCommandResponse.py b/mitmproxy/contrib/wbxml/ASCommandResponse.py index 2d60eb2d..34755cbe 100644 --- a/mitmproxy/contrib/wbxml/ASCommandResponse.py +++ b/mitmproxy/contrib/wbxml/ASCommandResponse.py @@ -63,8 +63,9 @@ if __name__ == "__main__": listOfSamples = os.listdir(samplesDir) for filename in listOfSamples: - byteWBXML = open(samplesDir + os.sep + filename, "rb").read() - + with open(samplesDir + os.sep + filename, "rb") as f: + byteWBXML = f.read() + logging.info("-"*100) logging.info(filename) logging.info("-"*100) diff --git a/mitmproxy/ctx.py b/mitmproxy/ctx.py index 5df6f9c1..2ce9c7c2 100644 --- a/mitmproxy/ctx.py +++ b/mitmproxy/ctx.py @@ -1,7 +1,7 @@ -import mitmproxy.master # noqa -import mitmproxy.log # noqa -import mitmproxy.options # noqa +import mitmproxy.log +import mitmproxy.master +import mitmproxy.options -master = None # type: mitmproxy.master.Master -log: mitmproxy.log.Log = None -options: mitmproxy.options.Options = None +log: "mitmproxy.log.Log" +master: "mitmproxy.master.Master" +options: "mitmproxy.options.Options" diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index 7f8df96f..b222d2a8 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -32,19 +32,17 @@ rex Equivalent to ~u rex """ +import functools import re import sys -import functools +from typing import Callable, ClassVar, Optional, Sequence, Type + +import pyparsing as pp +from mitmproxy import flow from mitmproxy import http -from mitmproxy import websocket from mitmproxy import tcp -from mitmproxy import flow - -from mitmproxy.utils import strutils - -import pyparsing as pp -from typing import Callable, Sequence, Type # noqa +from mitmproxy import websocket def only(*types): @@ -54,7 +52,9 @@ def only(*types): if isinstance(flow, types): return fn(self, flow) return False + return filter_types + return decorator @@ -69,8 +69,8 @@ class _Token: class _Action(_Token): - code: str = None - help: str = None + code: ClassVar[str] + help: ClassVar[str] @classmethod def make(klass, s, loc, toks): @@ -146,10 +146,10 @@ class _Rex(_Action): def __init__(self, expr): self.expr = expr if self.is_binary: - expr = strutils.escaped_str_to_bytes(expr) + expr = expr.encode() try: self.re = re.compile(expr, self.flags) - except: + except Exception: raise ValueError("Cannot compile expression.") @@ -336,6 +336,7 @@ class FUrl(_Rex): code = "u" help = "URL" is_binary = False + # FUrl is special, because it can be "naked". @classmethod @@ -469,45 +470,51 @@ def _make(): # Order is important - multi-char expressions need to come before narrow # ones. parts = [] - for klass in filter_unary: - f = pp.Literal("~%s" % klass.code) + pp.WordEnd() - f.setParseAction(klass.make) + for cls in filter_unary: + f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + f.setParseAction(cls.make) parts.append(f) - simplerex = "".join(c for c in pp.printables if c not in "()~'\"") - rex = pp.Word(simplerex) |\ - pp.QuotedString("\"", escChar='\\') |\ - pp.QuotedString("'", escChar='\\') - for klass in filter_rex: - f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + rex.copy() - f.setParseAction(klass.make) + # This is a bit of a hack to simulate Word(pyparsing_unicode.printables), + # which has a horrible performance with len(pyparsing.pyparsing_unicode.printables) == 1114060 + unicode_words = pp.CharsNotIn("()~'\"" + pp.ParserElement.DEFAULT_WHITE_CHARS) + unicode_words.skipWhitespace = True + regex = ( + unicode_words + | pp.QuotedString('"', escChar='\\') + | pp.QuotedString("'", escChar='\\') + ) + for cls in filter_rex: + f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + regex.copy() + f.setParseAction(cls.make) parts.append(f) - for klass in filter_int: - f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + pp.Word(pp.nums) - f.setParseAction(klass.make) + for cls in filter_int: + f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + pp.Word(pp.nums) + f.setParseAction(cls.make) parts.append(f) # A naked rex is a URL rex: - f = rex.copy() + f = regex.copy() f.setParseAction(FUrl.make) parts.append(f) atom = pp.MatchFirst(parts) - expr = pp.operatorPrecedence(atom, - [(pp.Literal("!").suppress(), - 1, - pp.opAssoc.RIGHT, - lambda x: FNot(*x)), - (pp.Literal("&").suppress(), - 2, - pp.opAssoc.LEFT, - lambda x: FAnd(*x)), - (pp.Literal("|").suppress(), - 2, - pp.opAssoc.LEFT, - lambda x: FOr(*x)), - ]) + expr = pp.infixNotation( + atom, + [(pp.Literal("!").suppress(), + 1, + pp.opAssoc.RIGHT, + lambda x: FNot(*x)), + (pp.Literal("&").suppress(), + 2, + pp.opAssoc.LEFT, + lambda x: FAnd(*x)), + (pp.Literal("|").suppress(), + 2, + pp.opAssoc.LEFT, + lambda x: FOr(*x)), + ]) expr = pp.OneOrMore(expr) return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x) @@ -516,7 +523,7 @@ bnf = _make() TFilter = Callable[[flow.Flow], bool] -def parse(s: str) -> TFilter: +def parse(s: str) -> Optional[TFilter]: try: flt = bnf.parseString(s, parseAll=True)[0] flt.pattern = s @@ -547,15 +554,15 @@ def match(flt, flow): help = [] for a in filter_unary: help.append( - ("~%s" % a.code, a.help) + (f"~{a.code}", a.help) ) for b in filter_rex: help.append( - ("~%s regex" % b.code, b.help) + (f"~{b.code} regex", b.help) ) for c in filter_int: help.append( - ("~%s int" % c.code, c.help) + (f"~{c.code} int", c.help) ) help.sort() help.extend( diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 3c16b807..6b527e75 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -1,15 +1,13 @@ import html from typing import Optional +from mitmproxy import connections from mitmproxy import flow - -from mitmproxy.net import http from mitmproxy import version -from mitmproxy import connections # noqa +from mitmproxy.net import http class HTTPRequest(http.Request): - """ A mitmproxy HTTP request. """ @@ -85,10 +83,10 @@ class HTTPRequest(http.Request): class HTTPResponse(http.Response): - """ A mitmproxy HTTP response. """ + # This is a very thin wrapper on top of :py:class:`mitmproxy.net.http.Response` and # may be removed in the future. @@ -136,34 +134,28 @@ class HTTPResponse(http.Response): class HTTPFlow(flow.Flow): - """ An HTTPFlow is a collection of objects representing a single HTTP transaction. """ + request: HTTPRequest + response: Optional[HTTPResponse] = None + error: Optional[flow.Error] = None + """ + Note that it's possible for a Flow to have both a response and an error + object. This might happen, for instance, when a response was received + from the server, but there was an error sending it back to the client. + """ + server_conn: connections.ServerConnection + client_conn: connections.ClientConnection + intercepted: bool = False + """ Is this flow currently being intercepted? """ + mode: str + """ What mode was the proxy layer in when receiving this request? """ def __init__(self, client_conn, server_conn, live=None, mode="regular"): super().__init__("http", client_conn, server_conn, live) - - self.request: HTTPRequest = None - """ :py:class:`HTTPRequest` object """ - self.response: HTTPResponse = None - """ :py:class:`HTTPResponse` object """ - self.error: flow.Error = None - """ :py:class:`Error` object - - Note that it's possible for a Flow to have both a response and an error - object. This might happen, for instance, when a response was received - from the server, but there was an error sending it back to the client. - """ - self.server_conn: connections.ServerConnection = server_conn - """ :py:class:`ServerConnection` object """ - self.client_conn: connections.ClientConnection = client_conn - """:py:class:`ClientConnection` object """ - self.intercepted: bool = False - """ Is this flow currently being intercepted? """ self.mode = mode - """ What mode was the proxy layer in when receiving this request? """ _stateobject_attributes = flow.Flow._stateobject_attributes.copy() # mypy doesn't support update with kwargs @@ -205,8 +197,8 @@ class HTTPFlow(flow.Flow): def make_error_response( status_code: int, - message: str="", - headers: Optional[http.Headers]=None, + message: str = "", + headers: Optional[http.Headers] = None, ) -> HTTPResponse: reason = http.status_codes.RESPONSES.get(status_code, "Unknown") body = """ diff --git a/mitmproxy/io/tnetstring.py b/mitmproxy/io/tnetstring.py index aa1f5670..de84279b 100644 --- a/mitmproxy/io/tnetstring.py +++ b/mitmproxy/io/tnetstring.py @@ -192,22 +192,22 @@ def parse(data_type: int, data: bytes) -> TSerializable: try: return int(data) except ValueError: - raise ValueError("not a tnetstring: invalid integer literal: {}".format(data)) + raise ValueError(f"not a tnetstring: invalid integer literal: {data!r}") if data_type == ord(b'^'): try: return float(data) except ValueError: - raise ValueError("not a tnetstring: invalid float literal: {}".format(data)) + raise ValueError(f"not a tnetstring: invalid float literal: {data!r}") if data_type == ord(b'!'): if data == b'true': return True elif data == b'false': return False else: - raise ValueError("not a tnetstring: invalid boolean literal: {}".format(data)) + raise ValueError(f"not a tnetstring: invalid boolean literal: {data!r}") if data_type == ord(b'~'): if data: - raise ValueError("not a tnetstring: invalid null literal") + raise ValueError(f"not a tnetstring: invalid null literal: {data!r}") return None if data_type == ord(b']'): l = [] @@ -236,7 +236,7 @@ def pop(data: bytes) -> typing.Tuple[TSerializable, bytes]: blength, data = data.split(b':', 1) length = int(blength) except ValueError: - raise ValueError("not a tnetstring: missing or invalid length prefix: {}".format(data)) + raise ValueError(f"not a tnetstring: missing or invalid length prefix: {data!r}") try: data, data_type, remain = data[:length], data[length], data[length + 1:] except IndexError: diff --git a/mitmproxy/net/check.py b/mitmproxy/net/check.py index aaea851f..a19ad6fe 100644 --- a/mitmproxy/net/check.py +++ b/mitmproxy/net/check.py @@ -2,7 +2,7 @@ import ipaddress import re # Allow underscore in host name -_label_valid = re.compile(b"(?!-)[A-Z\d\-_]{1,63}(?<!-)$", re.IGNORECASE) +_label_valid = re.compile(br"(?!-)[A-Z\d\-_]{1,63}(?<!-)$", re.IGNORECASE) def is_valid_host(host: bytes) -> bool: diff --git a/mitmproxy/net/http/cookies.py b/mitmproxy/net/http/cookies.py index 1472ab55..2745701f 100644 --- a/mitmproxy/net/http/cookies.py +++ b/mitmproxy/net/http/cookies.py @@ -304,7 +304,7 @@ def refresh_set_cookie_header(c: str, delta: int) -> str: e = email.utils.parsedate_tz(attrs["expires"]) if e: f = email.utils.mktime_tz(e) + delta - attrs.set_all("expires", [email.utils.formatdate(f)]) + attrs.set_all("expires", [email.utils.formatdate(f, usegmt=True)]) else: # This can happen when the expires tag is invalid. # reddit.com sends a an expires tag like this: "Thu, 31 Dec diff --git a/mitmproxy/net/http/encoding.py b/mitmproxy/net/http/encoding.py index 8cb96e5c..16d399ca 100644 --- a/mitmproxy/net/http/encoding.py +++ b/mitmproxy/net/http/encoding.py @@ -9,6 +9,7 @@ from io import BytesIO import gzip import zlib import brotli +import zstandard as zstd from typing import Union, Optional, AnyStr # noqa @@ -52,7 +53,7 @@ def decode( decoded = custom_decode[encoding](encoded) except KeyError: decoded = codecs.decode(encoded, encoding, errors) - if encoding in ("gzip", "deflate", "br"): + if encoding in ("gzip", "deflate", "br", "zstd"): _cache = CachedDecode(encoded, encoding, errors, decoded) return decoded except TypeError: @@ -93,7 +94,7 @@ def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optio encoded = custom_encode[encoding](decoded) except KeyError: encoded = codecs.encode(decoded, encoding, errors) - if encoding in ("gzip", "deflate", "br"): + if encoding in ("gzip", "deflate", "br", "zstd"): _cache = CachedDecode(encoded, encoding, errors, decoded) return encoded except TypeError: @@ -140,6 +141,23 @@ def encode_brotli(content: bytes) -> bytes: return brotli.compress(content) +def decode_zstd(content: bytes) -> bytes: + if not content: + return b"" + zstd_ctx = zstd.ZstdDecompressor() + try: + return zstd_ctx.decompress(content) + except zstd.ZstdError: + # If the zstd stream is streamed without a size header, + # try decoding with a 10MiB output buffer + return zstd_ctx.decompress(content, max_output_size=10 * 2**20) + + +def encode_zstd(content: bytes) -> bytes: + zstd_ctx = zstd.ZstdCompressor() + return zstd_ctx.compress(content) + + def decode_deflate(content: bytes) -> bytes: """ Returns decompressed data for DEFLATE. Some servers may respond with @@ -170,6 +188,7 @@ custom_decode = { "gzip": decode_gzip, "deflate": decode_deflate, "br": decode_brotli, + "zstd": decode_zstd, } custom_encode = { "none": identity, @@ -177,6 +196,7 @@ custom_encode = { "gzip": encode_gzip, "deflate": encode_deflate, "br": encode_brotli, + "zstd": encode_zstd, } __all__ = ["encode", "decode"] diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index 86782e8a..af7b032b 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -1,14 +1,18 @@ import re -from typing import Optional, Union # noqa +from typing import Optional # noqa from mitmproxy.utils import strutils from mitmproxy.net.http import encoding from mitmproxy.coretypes import serializable -from mitmproxy.net.http import headers +from mitmproxy.net.http import headers as mheaders class MessageData(serializable.Serializable): - content: bytes = None + headers: mheaders.Headers + content: bytes + http_version: bytes + timestamp_start: float + timestamp_end: float def __eq__(self, other): if isinstance(other, MessageData): @@ -18,7 +22,7 @@ class MessageData(serializable.Serializable): def set_state(self, state): for k, v in state.items(): if k == "headers": - v = headers.Headers.from_state(v) + v = mheaders.Headers.from_state(v) setattr(self, k, v) def get_state(self): @@ -28,12 +32,12 @@ class MessageData(serializable.Serializable): @classmethod def from_state(cls, state): - state["headers"] = headers.Headers.from_state(state["headers"]) + state["headers"] = mheaders.Headers.from_state(state["headers"]) return cls(**state) class Message(serializable.Serializable): - data: MessageData = None + data: MessageData def __eq__(self, other): if isinstance(other, Message): @@ -48,7 +52,7 @@ class Message(serializable.Serializable): @classmethod def from_state(cls, state): - state["headers"] = headers.Headers.from_state(state["headers"]) + state["headers"] = mheaders.Headers.from_state(state["headers"]) return cls(**state) @property @@ -78,7 +82,7 @@ class Message(serializable.Serializable): def raw_content(self, content): self.data.content = content - def get_content(self, strict: bool=True) -> bytes: + def get_content(self, strict: bool=True) -> Optional[bytes]: """ The uncompressed HTTP message body as bytes. @@ -160,7 +164,7 @@ class Message(serializable.Serializable): self.data.timestamp_end = timestamp_end def _get_content_type_charset(self) -> Optional[str]: - ct = headers.parse_content_type(self.headers.get("content-type", "")) + ct = mheaders.parse_content_type(self.headers.get("content-type", "")) if ct: return ct[2].get("charset") return None @@ -191,10 +195,9 @@ class Message(serializable.Serializable): See also: :py:attr:`content`, :py:class:`raw_content` """ - if self.raw_content is None: - return None - content = self.get_content(strict) + if content is None: + return None enc = self._guess_encoding(content) try: return encoding.decode(content, enc) @@ -213,9 +216,9 @@ class Message(serializable.Serializable): self.content = encoding.encode(text, enc) except ValueError: # Fall back to UTF-8 and update the content-type header. - ct = headers.parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {}) + ct = mheaders.parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {}) ct[2]["charset"] = "utf-8" - self.headers["content-type"] = headers.assemble_content_type(*ct) + self.headers["content-type"] = mheaders.assemble_content_type(*ct) enc = "utf8" self.content = text.encode(enc, "surrogateescape") @@ -236,7 +239,7 @@ class Message(serializable.Serializable): def encode(self, e): """ - Encodes body with the encoding e, where e is "gzip", "deflate", "identity", or "br". + Encodes body with the encoding e, where e is "gzip", "deflate", "identity", "br", or "zstd". Any existing content-encodings are overwritten, the content is not decoded beforehand. diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 959fdd33..1569ea72 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -1,5 +1,6 @@ import re import urllib +import time from typing import Optional, AnyStr, Dict, Iterable, Tuple, Union from mitmproxy.coretypes import multidict @@ -63,6 +64,8 @@ class Request(message.Message): """ An HTTP request. """ + data: RequestData + def __init__(self, *args, **kwargs): super().__init__() self.data = RequestData(*args, **kwargs) @@ -101,6 +104,7 @@ class Request(message.Message): ) req.url = url + req.timestamp_start = time.time() # Headers can be list or dict, we differentiate here. if isinstance(headers, dict): @@ -421,7 +425,7 @@ class Request(message.Message): self.headers["accept-encoding"] = ( ', '.join( e - for e in {"gzip", "identity", "deflate", "br"} + for e in {"gzip", "identity", "deflate", "br", "zstd"} if e in accept_encoding ) ) diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py index 48527d63..2e864405 100644 --- a/mitmproxy/net/http/response.py +++ b/mitmproxy/net/http/response.py @@ -47,6 +47,8 @@ class Response(message.Message): """ An HTTP response. """ + data: ResponseData + def __init__(self, *args, **kwargs): super().__init__() self.data = ResponseData(*args, **kwargs) @@ -186,7 +188,7 @@ class Response(message.Message): d = parsedate_tz(self.headers[i]) if d: new = mktime_tz(d) + delta - self.headers[i] = formatdate(new) + self.headers[i] = formatdate(new, usegmt=True) c = [] for set_cookie_header in self.headers.get_all("set-cookie"): try: diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index f938cb12..d8e14aeb 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -21,16 +21,25 @@ def parse(url): Raises: ValueError, if the URL is not properly formatted. """ - parsed = urllib.parse.urlparse(url) + # Size of Ascii character after encoding is 1 byte which is same as its size + # But non-Ascii character's size after encoding will be more than its size + def ascii_check(l): + if len(l) == len(str(l).encode()): + return True + return False + + if isinstance(url, bytes): + url = url.decode() + if not ascii_check(url): + url = urllib.parse.urlsplit(url) + url = list(url) + url[3] = urllib.parse.quote(url[3]) + url = urllib.parse.urlunsplit(url) + parsed = urllib.parse.urlparse(url) if not parsed.hostname: raise ValueError("No hostname given") - if isinstance(url, bytes): - host = parsed.hostname - - # this should not raise a ValueError, - # but we try to be very forgiving here and accept just everything. else: host = parsed.hostname.encode("idna") if isinstance(parsed, urllib.parse.ParseResult): diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index 4dc61969..d68a008f 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -295,6 +295,17 @@ def create_client_context( return context +def accept_all( + conn_: SSL.Connection, + x509: SSL.X509, + errno: int, + err_depth: int, + is_cert_verified: bool, +) -> bool: + # Return true to prevent cert verification error + return True + + def create_server_context( cert: typing.Union[certs.Cert, str], key: SSL.PKey, @@ -324,16 +335,6 @@ def create_server_context( until then we're conservative. """ - def accept_all( - conn_: SSL.Connection, - x509: SSL.X509, - errno: int, - err_depth: int, - is_cert_verified: bool, - ) -> bool: - # Return true to prevent cert verification error - return True - if request_client_cert: verify = SSL.VERIFY_PEER else: @@ -425,7 +426,7 @@ class ClientHello: return self._client_hello.cipher_suites.cipher_suites @property - def sni(self): + def sni(self) -> typing.Optional[bytes]: if self._client_hello.extensions: for extension in self._client_hello.extensions.extensions: is_valid_sni_extension = ( @@ -435,7 +436,7 @@ class ClientHello: check.is_valid_host(extension.body.server_names[0].host_name) ) if is_valid_sni_extension: - return extension.body.server_names[0].host_name.decode("idna") + return extension.body.server_names[0].host_name return None @property @@ -473,10 +474,8 @@ class ClientHello: return cls(raw_client_hello) except EOFError as e: raise exceptions.TlsProtocolException( - 'Cannot parse Client Hello: %s, Raw Client Hello: %s' % - (repr(e), binascii.hexlify(raw_client_hello)) + f"Cannot parse Client Hello: {e!r}, Raw Client Hello: {binascii.hexlify(raw_client_hello)!r}" ) def __repr__(self): - return "ClientHello(sni: %s, alpn_protocols: %s, cipher_suites: %s)" % \ - (self.sni, self.alpn_protocols, self.cipher_suites) + return f"ClientHello(sni: {self.sni}, alpn_protocols: {self.alpn_protocols})" diff --git a/mitmproxy/net/websockets/masker.py b/mitmproxy/net/websockets/masker.py index 47b1a688..6134e09e 100644 --- a/mitmproxy/net/websockets/masker.py +++ b/mitmproxy/net/websockets/masker.py @@ -1,3 +1,6 @@ +import sys + + class Masker: """ Data sent from the server must be masked to prevent malicious clients @@ -12,12 +15,13 @@ class Masker: self.offset = 0 def mask(self, offset, data): - result = bytearray(data) - for i in range(len(data)): - result[i] ^= self.key[offset % 4] - offset += 1 - result = bytes(result) - return result + datalen = len(data) + offset_mod = offset % 4 + data = int.from_bytes(data, sys.byteorder) + num_keys = (datalen + offset_mod + 3) // 4 + mask = int.from_bytes((self.key * num_keys)[offset_mod:datalen + + offset_mod], sys.byteorder) + return (data ^ mask).to_bytes(datalen, sys.byteorder) def __call__(self, data): ret = self.mask(self.offset, data) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index a6ab3d50..1583e9fc 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -5,8 +5,10 @@ from mitmproxy.net import tls CONF_DIR = "~/.mitmproxy" +CONF_BASENAME = "mitmproxy" LISTEN_PORT = 8080 CONTENT_VIEW_LINES_CUTOFF = 512 +KEY_SIZE = 2048 class Options(optmanager.OptManager): @@ -68,6 +70,10 @@ class Options(optmanager.OptManager): """ ) self.add_option( + "allow_hosts", Sequence[str], [], + "Opposite of --ignore-hosts." + ) + self.add_option( "listen_host", str, "", "Address to bind proxy to." ) @@ -169,5 +175,11 @@ class Options(optmanager.OptManager): speedup flows browsing. """ ) + self.add_option( + "key_size", int, KEY_SIZE, + """ + TLS key size for certificates and CA. + """ + ) self.update(**kwargs) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 06e696c0..f42aa645 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -320,7 +320,9 @@ class OptManager: update = {} for optname, optval in self.deferred.items(): if optname in self._options: - update[optname] = self.parse_setval(self._options[optname], optval) + if isinstance(optval, str): + optval = self.parse_setval(self._options[optname], optval) + update[optname] = optval self.update(**update) for k in update.keys(): del self.deferred[k] @@ -549,7 +551,9 @@ def serialize(opts: OptManager, text: str, defaults: bool = False) -> str: for k in list(data.keys()): if k not in opts._options: del data[k] - return ruamel.yaml.round_trip_dump(data) + ret = ruamel.yaml.round_trip_dump(data) + assert ret + return ret def save(opts: OptManager, path: str, defaults: bool =False) -> None: diff --git a/mitmproxy/platform/__init__.py b/mitmproxy/platform/__init__.py index 61946ec4..7e690789 100644 --- a/mitmproxy/platform/__init__.py +++ b/mitmproxy/platform/__init__.py @@ -1,7 +1,7 @@ import re import socket import sys -from typing import Tuple +from typing import Callable, Optional, Tuple def init_transparent_mode() -> None: @@ -10,30 +10,34 @@ def init_transparent_mode() -> None: """ -def original_addr(csock: socket.socket) -> Tuple[str, int]: - """ - Get the original destination for the given socket. - This function will be None if transparent mode is not supported. - """ - +original_addr: Optional[Callable[[socket.socket], Tuple[str, int]]] +""" +Get the original destination for the given socket. +This function will be None if transparent mode is not supported. +""" if re.match(r"linux(?:2)?", sys.platform): from . import linux - original_addr = linux.original_addr # noqa + original_addr = linux.original_addr elif sys.platform == "darwin" or sys.platform.startswith("freebsd"): from . import osx - original_addr = osx.original_addr # noqa + original_addr = osx.original_addr elif sys.platform.startswith("openbsd"): from . import openbsd - original_addr = openbsd.original_addr # noqa + original_addr = openbsd.original_addr elif sys.platform == "win32": from . import windows resolver = windows.Resolver() init_transparent_mode = resolver.setup # noqa - original_addr = resolver.original_addr # noqa + original_addr = resolver.original_addr else: - original_addr = None # noqa + original_addr = None + +__all__ = [ + "original_addr", + "init_transparent_mode" +] diff --git a/mitmproxy/platform/pf.py b/mitmproxy/platform/pf.py index bb5eb515..74e077a4 100644 --- a/mitmproxy/platform/pf.py +++ b/mitmproxy/platform/pf.py @@ -11,11 +11,17 @@ def lookup(address, port, s): """ # We may get an ipv4-mapped ipv6 address here, e.g. ::ffff:127.0.0.1. # Those still appear as "127.0.0.1" in the table, so we need to strip the prefix. - address = re.sub("^::ffff:(?=\d+.\d+.\d+.\d+$)", "", address) + address = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", address) s = s.decode() - spec = "%s:%s" % (address, port) + + # ALL tcp 192.168.1.13:57474 -> 23.205.82.58:443 ESTABLISHED:ESTABLISHED + specv4 = "%s:%s" % (address, port) + + # ALL tcp 2a01:e35:8bae:50f0:9d9b:ef0d:2de3:b733[58505] -> 2606:4700:30::681f:4ad0[443] ESTABLISHED:ESTABLISHED + specv6 = "%s[%s]" % (address, port) + for i in s.split("\n"): - if "ESTABLISHED:ESTABLISHED" in i and spec in i: + if "ESTABLISHED:ESTABLISHED" in i and specv4 in i: s = i.split() if len(s) > 4: if sys.platform.startswith("freebsd"): @@ -26,4 +32,11 @@ def lookup(address, port, s): if len(s) == 2: return s[0], int(s[1]) + elif "ESTABLISHED:ESTABLISHED" in i and specv6 in i: + s = i.split() + if len(s) > 4: + s = s[4].split("[") + port = s[1].split("]") + port = port[0] + return s[0], int(port) raise RuntimeError("Could not resolve original destination.") diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index b849afa5..19d9abd4 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -13,6 +13,7 @@ import typing import click import collections +import collections.abc import pydivert import pydivert.consts @@ -58,7 +59,7 @@ class Resolver: def original_addr(self, csock: socket.socket): ip, port = csock.getpeername()[:2] - ip = re.sub("^::ffff:(?=\d+.\d+.\d+.\d+$)", "", ip) + ip = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", ip) ip = ip.split("%", 1)[0] with self.lock: try: @@ -171,7 +172,7 @@ def MIB_TCPTABLE_OWNER_PID(size): TCP_TABLE_OWNER_PID_CONNECTIONS = 4 -class TcpConnectionTable(collections.Mapping): +class TcpConnectionTable(collections.abc.Mapping): DEFAULT_TABLE_SIZE = 4096 def __init__(self): diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index f32d3086..e98faabf 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -4,17 +4,15 @@ import typing from OpenSSL import crypto +from mitmproxy import certs from mitmproxy import exceptions from mitmproxy import options as moptions -from mitmproxy import certs from mitmproxy.net import server_spec -CONF_BASENAME = "mitmproxy" - class HostMatcher: - - def __init__(self, patterns=tuple()): + def __init__(self, handle, patterns=tuple()): + self.handle = handle self.patterns = list(patterns) self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] @@ -22,10 +20,10 @@ class HostMatcher: if not address: return False host = "%s:%s" % address - if any(rex.search(host) for rex in self.regexes): - return True - else: - return False + if self.handle in ["ignore", "tcp"]: + return any(rex.search(host) for rex in self.regexes) + else: # self.handle == "allow" + return any(not rex.search(host) for rex in self.regexes) def __bool__(self): return bool(self.patterns) @@ -36,18 +34,26 @@ class ProxyConfig: def __init__(self, options: moptions.Options) -> None: self.options = options - self.check_ignore: HostMatcher = None - self.check_tcp: HostMatcher = None - self.certstore: certs.CertStore = None + self.certstore: certs.CertStore + self.check_filter: typing.Optional[HostMatcher] = None + self.check_tcp: typing.Optional[HostMatcher] = None self.upstream_server: typing.Optional[server_spec.ServerSpec] = None self.configure(options, set(options.keys())) options.changed.connect(self.configure) def configure(self, options: moptions.Options, updated: typing.Any) -> None: - if "ignore_hosts" in updated: - self.check_ignore = HostMatcher(options.ignore_hosts) + if options.allow_hosts and options.ignore_hosts: + raise exceptions.OptionsError("--ignore-hosts and --allow-hosts are mutually " + "exclusive; please choose one.") + + if options.ignore_hosts: + self.check_filter = HostMatcher("ignore", options.ignore_hosts) + elif options.allow_hosts: + self.check_filter = HostMatcher("allow", options.allow_hosts) + else: + self.check_filter = HostMatcher(False) if "tcp_hosts" in updated: - self.check_tcp = HostMatcher(options.tcp_hosts) + self.check_tcp = HostMatcher("tcp", options.tcp_hosts) certstore_path = os.path.expanduser(options.confdir) if not os.path.exists(os.path.dirname(certstore_path)): @@ -55,9 +61,11 @@ class ProxyConfig: "Certificate Authority parent directory does not exist: %s" % os.path.dirname(certstore_path) ) + key_size = options.key_size self.certstore = certs.CertStore.from_store( certstore_path, - CONF_BASENAME + moptions.CONF_BASENAME, + key_size ) for c in options.certs: diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index 42b61f4d..a5870e6c 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -1,7 +1,7 @@ import threading import time import functools -from typing import Dict, Callable, Any, List # noqa +from typing import Dict, Callable, Any, List, Optional # noqa import h2.exceptions from h2 import connection @@ -382,15 +382,15 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr ctx, name="Http2SingleStreamLayer-{}".format(stream_id) ) self.h2_connection = h2_connection - self.zombie: float = None + self.zombie: Optional[float] = None self.client_stream_id: int = stream_id - self.server_stream_id: int = None + self.server_stream_id: Optional[int] = None self.request_headers = request_headers - self.response_headers: mitmproxy.net.http.Headers = None + self.response_headers: Optional[mitmproxy.net.http.Headers] = None self.pushed = False - self.timestamp_start: float = None - self.timestamp_end: float = None + self.timestamp_start: Optional[float] = None + self.timestamp_end: Optional[float] = None self.request_arrived = threading.Event() self.request_data_queue: queue.Queue[bytes] = queue.Queue() @@ -404,9 +404,9 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr self.no_body = False - self.priority_exclusive: bool = None - self.priority_depends_on: int = None - self.priority_weight: int = None + self.priority_exclusive: bool + self.priority_depends_on: Optional[int] = None + self.priority_weight: Optional[int] = None self.handled_priority_event: Any = None def kill(self): diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index 096aae9f..282df60d 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -196,17 +196,14 @@ CIPHER_ID_NAME_MAP = { } # We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. -# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old +# https://ssl-config.mozilla.org/#config=old DEFAULT_CLIENT_CIPHERS = ( - "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:" - "ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:" - "ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:" - "ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:" - "DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:" - "DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:" - "AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:" - "HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:" - "!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" + "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:" + "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:" + "DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:" + "ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:" + "ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:" + "AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA" ) @@ -323,14 +320,18 @@ class TlsLayer(base.Layer): return self._server_tls @property - def server_sni(self): + def server_sni(self) -> Optional[str]: """ The Server Name Indication we want to send with the next server TLS handshake. """ if self._custom_server_sni is False: return None + elif self._custom_server_sni: + return self._custom_server_sni + elif self._client_hello and self._client_hello.sni: + return self._client_hello.sni.decode("idna") else: - return self._custom_server_sni or self._client_hello and self._client_hello.sni + return None @property def alpn_for_client_connection(self): @@ -391,11 +392,12 @@ class TlsLayer(base.Layer): # raises ann error. self.client_conn.rfile.peek(1) except exceptions.TlsException as e: + sni_str = self._client_hello.sni and self._client_hello.sni.decode("idna") raise exceptions.ClientHandshakeException( "Cannot establish TLS with client (sni: {sni}): {e}".format( - sni=self._client_hello.sni, e=repr(e) + sni=sni_str, e=repr(e) ), - self._client_hello.sni or repr(self.server_conn.address) + sni_str or repr(self.server_conn.address) ) def _establish_tls_with_server(self): @@ -493,7 +495,7 @@ class TlsLayer(base.Layer): organization = upstream_cert.organization # Also add SNI values. if self._client_hello.sni: - sans.add(self._client_hello.sni.encode("idna")) + sans.add(self._client_hello.sni) if self._custom_server_sni: sans.add(self._custom_server_sni.encode("idna")) diff --git a/mitmproxy/proxy/protocol/websocket.py b/mitmproxy/proxy/protocol/websocket.py index 0d1964a6..f5ac6a29 100644 --- a/mitmproxy/proxy/protocol/websocket.py +++ b/mitmproxy/proxy/protocol/websocket.py @@ -4,8 +4,9 @@ from OpenSSL import SSL import wsproto -from wsproto import events -from wsproto.connection import ConnectionType, WSConnection +from wsproto import events, WSConnection +from wsproto.connection import ConnectionType +from wsproto.events import AcceptConnection, CloseConnection, Message, Ping, Request from wsproto.extensions import PerMessageDeflate from mitmproxy import exceptions @@ -52,51 +53,52 @@ class WebSocketLayer(base.Layer): self.connections: dict[object, WSConnection] = {} - extensions = [] + client_extensions = [] + server_extensions = [] if 'Sec-WebSocket-Extensions' in handshake_flow.response.headers: if PerMessageDeflate.name in handshake_flow.response.headers['Sec-WebSocket-Extensions']: - extensions = [PerMessageDeflate()] - self.connections[self.client_conn] = WSConnection(ConnectionType.SERVER, - extensions=extensions) - self.connections[self.server_conn] = WSConnection(ConnectionType.CLIENT, - host=handshake_flow.request.host, - resource=handshake_flow.request.path, - extensions=extensions) - if extensions: - for conn in self.connections.values(): - conn.extensions[0].finalize(conn, handshake_flow.response.headers['Sec-WebSocket-Extensions']) - - data = self.connections[self.server_conn].bytes_to_send() - self.connections[self.client_conn].receive_bytes(data) + client_extensions = [PerMessageDeflate()] + server_extensions = [PerMessageDeflate()] + self.connections[self.client_conn] = WSConnection(ConnectionType.SERVER) + self.connections[self.server_conn] = WSConnection(ConnectionType.CLIENT) + + if client_extensions: + client_extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) + if server_extensions: + server_extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) + + request = Request(extensions=client_extensions, host=handshake_flow.request.host, target=handshake_flow.request.path) + data = self.connections[self.server_conn].send(request) + self.connections[self.client_conn].receive_data(data) event = next(self.connections[self.client_conn].events()) - assert isinstance(event, events.ConnectionRequested) + assert isinstance(event, events.Request) - self.connections[self.client_conn].accept(event) - self.connections[self.server_conn].receive_bytes(self.connections[self.client_conn].bytes_to_send()) - assert isinstance(next(self.connections[self.server_conn].events()), events.ConnectionEstablished) + data = self.connections[self.client_conn].send(AcceptConnection(extensions=server_extensions)) + self.connections[self.server_conn].receive_data(data) + assert isinstance(next(self.connections[self.server_conn].events()), events.AcceptConnection) def _handle_event(self, event, source_conn, other_conn, is_server): - if isinstance(event, events.DataReceived): - return self._handle_data_received(event, source_conn, other_conn, is_server) - elif isinstance(event, events.PingReceived): - return self._handle_ping_received(event, source_conn, other_conn, is_server) - elif isinstance(event, events.PongReceived): - return self._handle_pong_received(event, source_conn, other_conn, is_server) - elif isinstance(event, events.ConnectionClosed): - return self._handle_connection_closed(event, source_conn, other_conn, is_server) + if isinstance(event, events.Message): + return self._handle_message(event, source_conn, other_conn, is_server) + elif isinstance(event, events.Ping): + return self._handle_ping(event, source_conn, other_conn, is_server) + elif isinstance(event, events.Pong): + return self._handle_pong(event, source_conn, other_conn, is_server) + elif isinstance(event, events.CloseConnection): + return self._handle_close_connection(event, source_conn, other_conn, is_server) # fail-safe for unhandled events return True # pragma: no cover - def _handle_data_received(self, event, source_conn, other_conn, is_server): + def _handle_message(self, event, source_conn, other_conn, is_server): fb = self.server_frame_buffer if is_server else self.client_frame_buffer fb.append(event.data) if event.message_finished: original_chunk_sizes = [len(f) for f in fb] - if isinstance(event, events.TextReceived): + if isinstance(event, events.TextMessage): message_type = wsproto.frame_protocol.Opcode.TEXT payload = ''.join(fb) else: @@ -127,19 +129,20 @@ class WebSocketLayer(base.Layer): yield (payload[i:i + chunk_size], True if i + chunk_size >= len(payload) else False) for chunk, final in get_chunk(websocket_message.content): - self.connections[other_conn].send_data(chunk, final) - other_conn.send(self.connections[other_conn].bytes_to_send()) + data = self.connections[other_conn].send(Message(data=chunk, message_finished=final)) + other_conn.send(data) if self.flow.stream: - self.connections[other_conn].send_data(event.data, event.message_finished) - other_conn.send(self.connections[other_conn].bytes_to_send()) + data = self.connections[other_conn].send(Message(data=event.data, message_finished=event.message_finished)) + other_conn.send(data) return True - def _handle_ping_received(self, event, source_conn, other_conn, is_server): - # PING is automatically answered with a PONG by wsproto - self.connections[other_conn].ping() - other_conn.send(self.connections[other_conn].bytes_to_send()) - source_conn.send(self.connections[source_conn].bytes_to_send()) + def _handle_ping(self, event, source_conn, other_conn, is_server): + # Use event.response to create the approprate Pong response + data = self.connections[other_conn].send(Ping()) + other_conn.send(data) + data = self.connections[source_conn].send(event.response()) + source_conn.send(data) self.log( "Ping Received from {}".format("server" if is_server else "client"), "info", @@ -147,7 +150,7 @@ class WebSocketLayer(base.Layer): ) return True - def _handle_pong_received(self, event, source_conn, other_conn, is_server): + def _handle_pong(self, event, source_conn, other_conn, is_server): self.log( "Pong Received from {}".format("server" if is_server else "client"), "info", @@ -155,14 +158,15 @@ class WebSocketLayer(base.Layer): ) return True - def _handle_connection_closed(self, event, source_conn, other_conn, is_server): + def _handle_close_connection(self, event, source_conn, other_conn, is_server): self.flow.close_sender = "server" if is_server else "client" self.flow.close_code = event.code self.flow.close_reason = event.reason - self.connections[other_conn].close(event.code, event.reason) - other_conn.send(self.connections[other_conn].bytes_to_send()) - source_conn.send(self.connections[source_conn].bytes_to_send()) + data = self.connections[other_conn].send(CloseConnection(code=event.code, reason=event.reason)) + other_conn.send(data) + data = self.connections[source_conn].send(event.response()) + source_conn.send(data) return False @@ -170,8 +174,7 @@ class WebSocketLayer(base.Layer): while True: try: payload = message_queue.get_nowait() - self.connections[endpoint].send_data(payload, final=True) - data = self.connections[endpoint].bytes_to_send() + data = self.connections[endpoint].send(Message(data=payload, message_finished=True)) endpoint.send(data) except queue.Empty: break @@ -197,8 +200,8 @@ class WebSocketLayer(base.Layer): is_server = (source_conn == self.server_conn) frame = websockets.Frame.from_file(source_conn.rfile) - self.connections[source_conn].receive_bytes(bytes(frame)) - source_conn.send(self.connections[source_conn].bytes_to_send()) + data = self.connections[source_conn].receive_data(bytes(frame)) + source_conn.send(data) if close_received: return diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index eb0008cf..3d4e8660 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -48,17 +48,18 @@ class RootContext: raise exceptions.ProtocolException(str(e)) client_tls = tls.is_tls_record_magic(d) - # 1. check for --ignore - if self.config.check_ignore: - ignore = self.config.check_ignore(top_layer.server_conn.address) - if not ignore and client_tls: + # 1. check for filter + if self.config.check_filter: + is_filtered = self.config.check_filter(top_layer.server_conn.address) + if not is_filtered and client_tls: try: client_hello = tls.ClientHello.from_file(self.client_conn.rfile) except exceptions.TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") else: - ignore = self.config.check_ignore((client_hello.sni, 443)) - if ignore: + sni_str = client_hello.sni and client_hello.sni.decode("idna") + is_filtered = self.config.check_filter((sni_str, 443)) + if is_filtered: return protocol.RawTCPLayer(top_layer, ignore=True) # 2. Always insert a TLS layer, even if there's neither client nor server tls. diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 44ae5697..3688b677 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -35,6 +35,7 @@ class DummyServer: class ProxyServer(tcp.TCPServer): allow_reuse_address = True bound = True + channel: controller.Channel def __init__(self, config: config.ProxyConfig) -> None: """ @@ -53,7 +54,6 @@ class ProxyServer(tcp.TCPServer): raise exceptions.ServerException( 'Error starting proxy server: ' + repr(e) ) from e - self.channel: controller.Channel = None def set_channel(self, channel): self.channel = channel diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index 2c16dcda..76329236 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -1,7 +1,5 @@ -import typing -from typing import Any # noqa -from typing import MutableMapping # noqa import json +import typing from mitmproxy.coretypes import serializable from mitmproxy.utils import typecheck @@ -15,7 +13,7 @@ class StateObject(serializable.Serializable): or StateObject instances themselves. """ - _stateobject_attributes: MutableMapping[str, Any] = None + _stateobject_attributes: typing.ClassVar[typing.MutableMapping[str, typing.Any]] """ An attribute-name -> class-or-type dict containing all attributes that should be serialized. If the attribute is a class, it must implement the @@ -42,7 +40,7 @@ class StateObject(serializable.Serializable): if val is None: setattr(self, attr, val) else: - curr = getattr(self, attr) + curr = getattr(self, attr, None) if hasattr(curr, "set_state"): curr.set_state(val) else: diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py index f1c763b2..a00a3e98 100644 --- a/mitmproxy/tools/_main.py +++ b/mitmproxy/tools/_main.py @@ -6,19 +6,16 @@ Feel free to import and use whatever new package you deem necessary. import os import sys import asyncio -import argparse # noqa -import signal # noqa -import typing # noqa +import argparse +import signal +import typing -from mitmproxy.tools import cmdline # noqa -from mitmproxy import exceptions, master # noqa -from mitmproxy import options # noqa -from mitmproxy import optmanager # noqa -from mitmproxy import proxy # noqa -from mitmproxy import log # noqa -from mitmproxy.utils import debug, arg_check # noqa - -OPTIONS_FILE_NAME = "config.yaml" +from mitmproxy.tools import cmdline +from mitmproxy import exceptions, master +from mitmproxy import options +from mitmproxy import optmanager +from mitmproxy import proxy +from mitmproxy.utils import debug, arg_check def assert_utf8_env(): @@ -87,10 +84,11 @@ def run( arg_check.check() sys.exit(1) try: - opts.confdir = args.confdir + opts.set(*args.setoptions, defer=True) optmanager.load_paths( opts, - os.path.join(opts.confdir, OPTIONS_FILE_NAME), + os.path.join(opts.confdir, "config.yaml"), + os.path.join(opts.confdir, "config.yml"), ) pconf = process_options(parser, opts, args) server: typing.Any = None @@ -110,7 +108,6 @@ def run( if args.commands: master.commands.dump() sys.exit(0) - opts.set(*args.setoptions, defer=True) if extra: opts.update(**extra(args)) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index ad934ca2..e9ff973f 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -1,7 +1,5 @@ import argparse -from mitmproxy.addons import core - def common_options(parser, opts): parser.add_argument( @@ -21,12 +19,6 @@ def common_options(parser, opts): help="Show all commands and their signatures", ) parser.add_argument( - "--confdir", - type=str, dest="confdir", default=core.CONF_DIR, - metavar="PATH", - help="Path to the mitmproxy config directory" - ) - parser.add_argument( "--set", type=str, dest="setoptions", default=[], action="append", @@ -65,6 +57,7 @@ def common_options(parser, opts): opts.make_parser(group, "listen_port", metavar="PORT", short="p") opts.make_parser(group, "server", short="n") opts.make_parser(group, "ignore_hosts", metavar="HOST") + opts.make_parser(group, "allow_hosts", metavar="HOST") opts.make_parser(group, "tcp_hosts", metavar="HOST") opts.make_parser(group, "upstream_auth", metavar="USER:PASS") opts.make_parser(group, "proxyauth", metavar="SPEC") @@ -75,6 +68,7 @@ def common_options(parser, opts): group = parser.add_argument_group("SSL") opts.make_parser(group, "certs", metavar="SPEC") opts.make_parser(group, "ssl_insecure", short="k") + opts.make_parser(group, "key_size", metavar="KEY_SIZE") # Client replay group = parser.add_argument_group("Client Replay") @@ -85,6 +79,7 @@ def common_options(parser, opts): opts.make_parser(group, "server_replay", metavar="PATH", short="S") opts.make_parser(group, "server_replay_kill_extra") opts.make_parser(group, "server_replay_nopop") + opts.make_parser(group, "server_replay_refresh") # Replacements group = parser.add_argument_group("Replacements") diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py index e8550f86..f291b8fd 100644 --- a/mitmproxy/tools/console/commander/commander.py +++ b/mitmproxy/tools/console/commander/commander.py @@ -55,7 +55,7 @@ class CommandBuffer: self.text = self.flatten(start) # Cursor is always within the range [0:len(buffer)]. self._cursor = len(self.text) - self.completion: CompletionState = None + self.completion: typing.Optional[CompletionState] = None @property def cursor(self) -> int: diff --git a/mitmproxy/tools/console/commandexecutor.py b/mitmproxy/tools/console/commandexecutor.py index 3db03d3e..c738e349 100644 --- a/mitmproxy/tools/console/commandexecutor.py +++ b/mitmproxy/tools/console/commandexecutor.py @@ -34,4 +34,4 @@ class CommandExecutor: ret, ), valign="top" - )
\ No newline at end of file + ) diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 72282015..3a5b4aeb 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -1,6 +1,10 @@ import platform import typing +import datetime +import time +import math from functools import lru_cache +from publicsuffix2 import get_sld, get_tld import urwid import urwid.util @@ -34,7 +38,7 @@ KEY_MAX = 30 def format_keyvals( - entries: typing.List[typing.Tuple[str, typing.Union[None, str, urwid.Widget]]], + entries: typing.Iterable[typing.Tuple[str, typing.Union[None, str, urwid.Widget]]], key_format: str = "key", value_format: str = "text", indent: int = 0 @@ -97,16 +101,180 @@ if urwid.util.detected_encoding and not IS_WSL: SYMBOL_MARK = u"\u25cf" SYMBOL_UP = u"\u21E7" SYMBOL_DOWN = u"\u21E9" + SYMBOL_ELLIPSIS = u"\u2026" else: SYMBOL_REPLAY = u"[r]" SYMBOL_RETURN = u"<-" SYMBOL_MARK = "[m]" SYMBOL_UP = "^" SYMBOL_DOWN = " " + SYMBOL_ELLIPSIS = "~" + + +def fixlen(s, maxlen): + if len(s) <= maxlen: + return s.ljust(maxlen) + else: + return s[0:maxlen - len(SYMBOL_ELLIPSIS)] + SYMBOL_ELLIPSIS + + +def fixlen_r(s, maxlen): + if len(s) <= maxlen: + return s.rjust(maxlen) + else: + return SYMBOL_ELLIPSIS + s[len(s) - maxlen + len(SYMBOL_ELLIPSIS):] + + +class TruncatedText(urwid.Widget): + def __init__(self, text, attr, align='left'): + self.text = text + self.attr = attr + self.align = align + super(TruncatedText, self).__init__() + + def pack(self, size, focus=False): + return (len(self.text), 1) + + def rows(self, size, focus=False): + return 1 + + def render(self, size, focus=False): + text = self.text + attr = self.attr + if self.align == 'right': + text = text[::-1] + attr = attr[::-1] + + text_len = len(text) # TODO: unicode? + if size is not None and len(size) > 0: + width = size[0] + else: + width = text_len + + if width >= text_len: + remaining = width - text_len + if remaining > 0: + c_text = text + ' ' * remaining + c_attr = attr + [('text', remaining)] + else: + c_text = text + c_attr = attr + else: + visible_len = width - len(SYMBOL_ELLIPSIS) + visible_text = text[0:visible_len] + c_text = visible_text + SYMBOL_ELLIPSIS + c_attr = (urwid.util.rle_subseg(attr, 0, len(visible_text.encode())) + + [('focus', len(SYMBOL_ELLIPSIS.encode()))]) + + if self.align == 'right': + c_text = c_text[::-1] + c_attr = c_attr[::-1] + + return urwid.TextCanvas([c_text.encode()], [c_attr], maxcol=width) + + +def truncated_plain(text, attr, align='left'): + return TruncatedText(text, [(attr, len(text.encode()))], align) + + +# Work around https://github.com/urwid/urwid/pull/330 +def rle_append_beginning_modify(rle, a_r): + """ + Append (a, r) (unpacked from *a_r*) to BEGINNING of rle. + Merge with first run when possible + + MODIFIES rle parameter contents. Returns None. + """ + a, r = a_r + if not rle: + rle[:] = [(a, r)] + else: + al, run = rle[0] + if a == al: + rle[0] = (a, run + r) + else: + rle[0:0] = [(a, r)] + + +def colorize_host(host): + tld = get_tld(host) + sld = get_sld(host) + + attr = [] + + tld_size = len(tld) + sld_size = len(sld) - tld_size + + for letter in reversed(range(len(host))): + character = host[letter] + if tld_size > 0: + style = 'url_domain' + tld_size -= 1 + elif tld_size == 0: + style = 'text' + tld_size -= 1 + elif sld_size > 0: + sld_size -= 1 + style = 'url_extension' + else: + style = 'text' + rle_append_beginning_modify(attr, (style, len(character.encode()))) + return attr + + +def colorize_req(s): + path = s.split('?', 2)[0] + i_query = len(path) + i_last_slash = path.rfind('/') + i_ext = path[i_last_slash + 1:].rfind('.') + i_ext = i_last_slash + i_ext if i_ext >= 0 else len(s) + in_val = False + attr = [] + for i in range(len(s)): + c = s[i] + if ((i < i_query and c == '/') or + (i < i_query and i > i_last_slash and c == '.') or + (i == i_query)): + a = 'url_punctuation' + elif i > i_query: + if in_val: + if c == '&': + in_val = False + a = 'url_punctuation' + else: + a = 'url_query_value' + else: + if c == '=': + in_val = True + a = 'url_punctuation' + else: + a = 'url_query_key' + elif i > i_ext: + a = 'url_extension' + elif i > i_last_slash: + a = 'url_filename' + else: + a = 'text' + urwid.util.rle_append_modify(attr, (a, len(c.encode()))) + return attr + + +def colorize_url(url): + parts = url.split('/', 3) + if len(parts) < 4 or len(parts[1]) > 0 or parts[0][-1:] != ':': + return [('error', len(url))] # bad URL + schemes = { + 'http:': 'scheme_http', + 'https:': 'scheme_https', + } + return [ + (schemes.get(parts[0], "scheme_other"), len(parts[0]) - 1), + ('url_punctuation', 3), # :// + ] + colorize_host(parts[2]) + colorize_req('/' + parts[3]) @lru_cache(maxsize=800) -def raw_format_flow(f, flow): +def raw_format_list(f): f = dict(f) pile = [] req = [] @@ -126,8 +294,7 @@ def raw_format_flow(f, flow): if f["req_is_replay"]: req.append(fcol(SYMBOL_REPLAY, "replay")) - pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else '' - req.append(fcol(f["req_method"] + pushed, "method")) + req.append(fcol(f["req_method"], "method")) preamble = sum(i[1] for i in req) + len(req) - 1 @@ -140,8 +307,8 @@ def raw_format_flow(f, flow): url = f["req_url"] - if f["max_url_len"] and len(url) > f["max_url_len"]: - url = url[:f["max_url_len"]] + "…" + if f["cols"] and len(url) > f["cols"]: + url = url[:f["cols"]] + "…" if f["req_http_version"] not in ("HTTP/1.0", "HTTP/1.1"): url += " " + f["req_http_version"] @@ -178,7 +345,8 @@ def raw_format_flow(f, flow): if f["resp_ctype"]: resp.append(fcol(f["resp_ctype"], rc)) resp.append(fcol(f["resp_clen"], rc)) - resp.append(fcol(f["roundtrip"], rc)) + pretty_duration = human.pretty_duration(f["duration"]) + resp.append(fcol(pretty_duration, rc)) elif f["err_msg"]: resp.append(fcol(SYMBOL_RETURN, "error")) @@ -194,48 +362,203 @@ def raw_format_flow(f, flow): return urwid.Pile(pile) -def format_flow(f, focus, extended=False, hostheader=False, max_url_len=False): +@lru_cache(maxsize=800) +def raw_format_table(f): + f = dict(f) + pile = [] + req = [] + + cursor = [' ', 'focus'] + if f.get('resp_is_replay', False): + cursor[0] = SYMBOL_REPLAY + cursor[1] = 'replay' + if f['marked']: + if cursor[0] == ' ': + cursor[0] = SYMBOL_MARK + cursor[1] = 'mark' + if f['focus']: + cursor[0] = '>' + + req.append(fcol(*cursor)) + + if f["two_line"]: + req.append(TruncatedText(f["req_url"], colorize_url(f["req_url"]), 'left')) + pile.append(urwid.Columns(req, dividechars=1)) + + req = [] + req.append(fcol(' ', 'text')) + + if f["intercepted"] and not f["acked"]: + uc = "intercept" + elif "resp_code" in f or f["err_msg"] is not None: + uc = "highlight" + else: + uc = "title" + + if f["extended"]: + s = human.format_timestamp(f["req_timestamp"]) + else: + s = datetime.datetime.fromtimestamp(time.mktime(time.localtime(f["req_timestamp"]))).strftime("%H:%M:%S") + req.append(fcol(s, uc)) + + methods = { + 'GET': 'method_get', + 'POST': 'method_post', + 'DELETE': 'method_delete', + 'HEAD': 'method_head', + 'PUT': 'method_put' + } + uc = methods.get(f["req_method"], "method_other") + if f['extended']: + req.append(fcol(f["req_method"], uc)) + if f["req_promise"]: + req.append(fcol('PUSH_PROMISE', 'method_http2_push')) + else: + if f["req_promise"]: + uc = 'method_http2_push' + req.append(("fixed", 4, truncated_plain(f["req_method"], uc))) + + if f["two_line"]: + req.append(fcol(f["req_http_version"], 'text')) + else: + schemes = { + 'http': 'scheme_http', + 'https': 'scheme_https', + } + req.append(fcol(fixlen(f["req_scheme"].upper(), 5), schemes.get(f["req_scheme"], "scheme_other"))) + + req.append(('weight', 0.25, TruncatedText(f["req_host"], colorize_host(f["req_host"]), 'right'))) + req.append(('weight', 1.0, TruncatedText(f["req_path"], colorize_req(f["req_path"]), 'left'))) + + ret = (' ' * len(SYMBOL_RETURN), 'text') + status = ('', 'text') + content = ('', 'text') + size = ('', 'text') + duration = ('', 'text') + + if "resp_code" in f: + codes = { + 2: "code_200", + 3: "code_300", + 4: "code_400", + 5: "code_500", + } + ccol = codes.get(f["resp_code"] // 100, "code_other") + ret = (SYMBOL_RETURN, ccol) + status = (str(f["resp_code"]), ccol) + + if f["resp_len"] < 0: + if f["intercepted"] and f["resp_code"] and not f["acked"]: + rc = "intercept" + else: + rc = "content_none" + + if f["resp_len"] == -1: + contentdesc = "[content missing]" + else: + contentdesc = "[no content]" + content = (contentdesc, rc) + else: + if f["resp_ctype"]: + ctype = f["resp_ctype"].split(";")[0] + if ctype.endswith('/javascript'): + rc = 'content_script' + elif ctype.startswith('text/'): + rc = 'content_text' + elif (ctype.startswith('image/') or + ctype.startswith('video/') or + ctype.startswith('font/') or + "/x-font-" in ctype): + rc = 'content_media' + elif ctype.endswith('/json') or ctype.endswith('/xml'): + rc = 'content_data' + elif ctype.startswith('application/'): + rc = 'content_raw' + else: + rc = 'content_other' + content = (ctype, rc) + + rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + f["resp_len"]) / 20, 0.99)) + + size_str = human.pretty_size(f["resp_len"]) + if not f['extended']: + # shorten to 5 chars max + if len(size_str) > 5: + size_str = size_str[0:4].rstrip('.') + size_str[-1:] + size = (size_str, rc) + + if f['duration'] is not None: + rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + 1000 * f['duration']) / 12, 0.99)) + duration = (human.pretty_duration(f['duration']), rc) + + elif f["err_msg"]: + status = ('Err', 'error') + content = f["err_msg"], 'error' + + if f["two_line"]: + req.append(fcol(*ret)) + req.append(fcol(fixlen(status[0], 3), status[1])) + req.append(('weight', 0.15, truncated_plain(content[0], content[1], 'right'))) + if f['extended']: + req.append(fcol(*size)) + else: + req.append(fcol(fixlen_r(size[0], 5), size[1])) + req.append(fcol(fixlen_r(duration[0], 5), duration[1])) + + pile.append(urwid.Columns(req, dividechars=1, min_width=15)) + + return urwid.Pile(pile) + + +def format_flow(f, focus, extended=False, hostheader=False, cols=False, layout='default'): acked = False if f.reply and f.reply.state == "committed": acked = True d = dict( focus=focus, extended=extended, - max_url_len=max_url_len, + two_line=extended or cols < 100, + cols=cols, intercepted=f.intercepted, acked=acked, req_timestamp=f.request.timestamp_start, req_is_replay=f.request.is_replay, req_method=f.request.method, + req_promise='h2-pushed-stream' in f.metadata, req_url=f.request.pretty_url if hostheader else f.request.url, + req_scheme=f.request.scheme, + req_host=f.request.pretty_host if hostheader else f.request.host, + req_path=f.request.path, req_http_version=f.request.http_version, err_msg=f.error.msg if f.error else None, marked=f.marked, ) if f.response: if f.response.raw_content: + content_len = len(f.response.raw_content) contentdesc = human.pretty_size(len(f.response.raw_content)) elif f.response.raw_content is None: + content_len = -1 contentdesc = "[content missing]" else: + content_len = -2 contentdesc = "[no content]" - duration = 0 + + duration = None if f.response.timestamp_end and f.request.timestamp_start: duration = f.response.timestamp_end - f.request.timestamp_start - roundtrip = human.pretty_duration(duration) d.update(dict( resp_code=f.response.status_code, resp_reason=f.response.reason, resp_is_replay=f.response.is_replay, + resp_len=content_len, + resp_ctype=f.response.headers.get("content-type"), resp_clen=contentdesc, - roundtrip=roundtrip, + duration=duration, )) - t = f.response.headers.get("content-type") - if t: - d["resp_ctype"] = t.split(";")[0] - else: - d["resp_ctype"] = "" - - return raw_format_flow(tuple(sorted(d.items())), f) + if ((layout == 'default' and cols < 100) or layout == "list"): + return raw_format_list(tuple(sorted(d.items()))) + else: + return raw_format_table(tuple(sorted(d.items()))) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index a40cdeaa..b6602413 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -37,6 +37,12 @@ console_layouts = [ "horizontal", ] +console_flowlist_layout = [ + "default", + "table", + "list" +] + class UnsupportedLog: """ @@ -114,6 +120,13 @@ class ConsoleAddon: "Console mouse interaction." ) + loader.add_option( + "console_flowlist_layout", + str, "default", + "Set the flowlist layout", + choices=sorted(console_flowlist_layout) + ) + @command.command("console.layout.options") def layout_options(self) -> typing.Sequence[str]: """ @@ -428,7 +441,12 @@ class ConsoleAddon: message.content = c.rstrip(b"\n") elif part == "set-cookies": self.master.switch_view("edit_focus_setcookies") - elif part in ["url", "method", "status_code", "reason"]: + elif part == "url": + url = flow.request.url.encode() + edited_url = self.master.spawn_editor(url) + url = edited_url.rstrip(b"\n") + flow.request.url = url.decode() + elif part in ["method", "status_code", "reason"]: self.master.commands.execute( "console.command flow.set @focus %s " % part ) diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index e947a582..9650c0d3 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -18,7 +18,8 @@ class FlowItem(urwid.WidgetWrap): self.flow, self.flow is self.master.view.focus.flow, hostheader=self.master.options.showhost, - max_url_len=cols, + cols=cols, + layout=self.master.options.console_flowlist_layout ) def selectable(self): @@ -84,6 +85,10 @@ class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget): ) -> None: self.master: "mitmproxy.tools.console.master.ConsoleMaster" = master super().__init__(FlowListWalker(master)) + self.master.options.subscribe( + self.set_flowlist_layout, + ["console_flowlist_layout"] + ) def keypress(self, size, key): if key == "m_start": @@ -96,3 +101,6 @@ class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget): def view_changed(self): self.body.view_changed() + + def set_flowlist_layout(self, opts, updated): + self.master.ui.clear() diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index b4e3876f..807c9714 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -38,7 +38,8 @@ class FlowViewHeader(urwid.WidgetWrap): False, extended=True, hostheader=self.master.options.showhost, - max_url_len=cols, + cols=cols, + layout=self.master.options.console_flowlist_layout ) else: self._w = urwid.Pile([]) diff --git a/mitmproxy/tools/console/grideditor/base.py b/mitmproxy/tools/console/grideditor/base.py index 3badf1a6..64b6e5d5 100644 --- a/mitmproxy/tools/console/grideditor/base.py +++ b/mitmproxy/tools/console/grideditor/base.py @@ -254,7 +254,7 @@ FIRST_WIDTH_MAX = 40 class BaseGridEditor(urwid.WidgetWrap): - title = "" + title: str = "" keyctx = "grideditor" def __init__( @@ -402,8 +402,8 @@ class BaseGridEditor(urwid.WidgetWrap): class GridEditor(BaseGridEditor): - title: str = None - columns: typing.Sequence[Column] = None + title = "" + columns: typing.Sequence[Column] = () keyctx = "grideditor" def __init__( diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index 61fcf6b4..b4d59384 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -107,7 +107,7 @@ class CookieAttributeEditor(base.FocusEditor): col_text.Column("Name"), col_text.Column("Value"), ] - grideditor: base.BaseGridEditor = None + grideditor: base.BaseGridEditor def data_in(self, data): return [(k, v or "") for k, v in data] @@ -169,7 +169,7 @@ class SetCookieEditor(base.FocusEditor): class OptionsEditor(base.GridEditor, layoutwidget.LayoutWidget): - title: str = None + title = "" columns = [ col_text.Column("") ] @@ -189,7 +189,7 @@ class OptionsEditor(base.GridEditor, layoutwidget.LayoutWidget): class DataViewer(base.GridEditor, layoutwidget.LayoutWidget): - title: str = None + title = "" def __init__( self, diff --git a/mitmproxy/tools/console/help.py b/mitmproxy/tools/console/help.py index 1b4b9ac6..fb4e0051 100644 --- a/mitmproxy/tools/console/help.py +++ b/mitmproxy/tools/console/help.py @@ -91,9 +91,9 @@ class HelpView(tabs.Tabs, layoutwidget.LayoutWidget): ) ) examples = [ - ("google\.com", "Url containing \"google.com"), - ("~q ~b test", "Requests where body contains \"test\""), - ("!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."), + (r"google\.com", r"Url containing \"google.com"), + ("~q ~b test", r"Requests where body contains \"test\""), + (r"!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."), ] text.extend( common.format_keyvals(examples, indent=4) diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 7930c4a3..6033ff25 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -22,7 +22,12 @@ class Palette: 'option_selected_key', # List and Connections - 'method', 'focus', + 'method', + 'method_get', 'method_post', 'method_delete', 'method_other', 'method_head', 'method_put', 'method_http2_push', + 'scheme_http', 'scheme_https', 'scheme_other', + 'url_punctuation', 'url_domain', 'url_filename', 'url_extension', 'url_query_key', 'url_query_value', + 'content_none', 'content_text', 'content_script', 'content_media', 'content_data', 'content_raw', 'content_other', + 'focus', 'code_200', 'code_300', 'code_400', 'code_500', 'code_other', 'error', "warn", "alert", 'header', 'highlight', 'intercept', 'replay', 'mark', @@ -36,7 +41,8 @@ class Palette: # Commander 'commander_command', 'commander_invalid', 'commander_hint' ] - high: typing.Mapping[str, typing.Sequence[str]] = None + _fields.extend(['gradient_%02d' % i for i in range(100)]) + high: typing.Optional[typing.Mapping[str, typing.Sequence[str]]] = None def palette(self, transparent): l = [] @@ -68,6 +74,27 @@ class Palette: return l +def gen_gradient(palette, cols): + for i in range(100): + palette['gradient_%02d' % i] = (cols[i * len(cols) // 100], 'default') + + +def gen_rgb_gradient(palette, cols): + parts = len(cols) - 1 + for i in range(100): + p = i / 100 + idx = int(p * parts) + t0 = cols[idx] + t1 = cols[idx + 1] + pp = p * parts % 1 + t = ( + round(t0[0] + (t1[0] - t0[0]) * pp), + round(t0[1] + (t1[1] - t0[1]) * pp), + round(t0[2] + (t1[2] - t0[2]) * pp), + ) + palette['gradient_%02d' % i] = ("#%x%x%x" % t, 'default') + + class LowDark(Palette): """ @@ -95,6 +122,33 @@ class LowDark(Palette): # List and Connections method = ('dark cyan', 'default'), + method_get = ('light green', 'default'), + method_post = ('brown', 'default'), + method_delete = ('light red', 'default'), + method_head = ('dark cyan', 'default'), + method_put = ('dark red', 'default'), + method_other = ('dark magenta', 'default'), + method_http2_push = ('dark gray', 'default'), + + scheme_http = ('dark cyan', 'default'), + scheme_https = ('dark green', 'default'), + scheme_other = ('dark magenta', 'default'), + + url_punctuation = ('light gray', 'default'), + url_domain = ('white', 'default'), + url_filename = ('dark cyan', 'default'), + url_extension = ('light gray', 'default'), + url_query_key = ('white', 'default'), + url_query_value = ('light gray', 'default'), + + content_none = ('dark gray', 'default'), + content_text = ('light gray', 'default'), + content_script = ('dark green', 'default'), + content_media = ('light blue', 'default'), + content_data = ('brown', 'default'), + content_raw = ('dark red', 'default'), + content_other = ('dark magenta', 'default'), + focus = ('yellow', 'default'), code_200 = ('dark green', 'default'), @@ -127,6 +181,7 @@ class LowDark(Palette): commander_invalid = ('light red', 'default'), commander_hint = ('dark gray', 'default'), ) + gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue']) class Dark(LowDark): @@ -167,6 +222,33 @@ class LowLight(Palette): # List and Connections method = ('dark cyan', 'default'), + method_get = ('dark green', 'default'), + method_post = ('brown', 'default'), + method_head = ('dark cyan', 'default'), + method_put = ('light red', 'default'), + method_delete = ('dark red', 'default'), + method_other = ('light magenta', 'default'), + method_http2_push = ('light gray', 'default'), + + scheme_http = ('dark cyan', 'default'), + scheme_https = ('light green', 'default'), + scheme_other = ('light magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('dark gray', 'default'), + url_filename = ('black', 'default'), + url_extension = ('dark gray', 'default'), + url_query_key = ('light blue', 'default'), + url_query_value = ('dark blue', 'default'), + + content_none = ('black', 'default'), + content_text = ('dark gray', 'default'), + content_script = ('light green', 'default'), + content_media = ('light blue', 'default'), + content_data = ('brown', 'default'), + content_raw = ('light red', 'default'), + content_other = ('light magenta', 'default'), + focus = ('black', 'default'), code_200 = ('dark green', 'default'), @@ -198,6 +280,7 @@ class LowLight(Palette): commander_invalid = ('light red', 'default'), commander_hint = ('light gray', 'default'), ) + gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue']) class Light(LowLight): @@ -256,7 +339,27 @@ class SolarizedLight(LowLight): option_active_selected = (sol_orange, sol_base2), # List and Connections - method = (sol_cyan, 'default'), + + method = ('dark cyan', 'default'), + method_get = (sol_green, 'default'), + method_post = (sol_orange, 'default'), + method_head = (sol_cyan, 'default'), + method_put = (sol_red, 'default'), + method_delete = (sol_red, 'default'), + method_other = (sol_magenta, 'default'), + method_http2_push = ('light gray', 'default'), + + scheme_http = (sol_cyan, 'default'), + scheme_https = ('light green', 'default'), + scheme_other = ('light magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('dark gray', 'default'), + url_filename = ('black', 'default'), + url_extension = ('dark gray', 'default'), + url_query_key = (sol_blue, 'default'), + url_query_value = ('dark blue', 'default'), + focus = (sol_base01, 'default'), code_200 = (sol_green, 'default'), @@ -311,9 +414,28 @@ class SolarizedDark(LowDark): option_active_selected = (sol_orange, sol_base00), # List and Connections - method = (sol_cyan, 'default'), focus = (sol_base1, 'default'), + method = (sol_cyan, 'default'), + method_get = (sol_green, 'default'), + method_post = (sol_orange, 'default'), + method_delete = (sol_red, 'default'), + method_head = (sol_cyan, 'default'), + method_put = (sol_red, 'default'), + method_other = (sol_magenta, 'default'), + method_http2_push = (sol_base01, 'default'), + + url_punctuation = ('h242', 'default'), + url_domain = ('h252', 'default'), + url_filename = ('h132', 'default'), + url_extension = ('h96', 'default'), + url_query_key = ('h37', 'default'), + url_query_value = ('h30', 'default'), + + content_none = (sol_base01, 'default'), + content_text = (sol_base1, 'default'), + content_media = (sol_blue, 'default'), + code_200 = (sol_green, 'default'), code_300 = (sol_blue, 'default'), code_400 = (sol_orange, 'default',), @@ -342,6 +464,7 @@ class SolarizedDark(LowDark): commander_invalid = (sol_orange, 'default'), commander_hint = (sol_base00, 'default'), ) + gen_rgb_gradient(high, [(15, 0, 0), (15, 15, 0), (0, 15, 0), (0, 15, 15), (0, 0, 15)]) DEFAULT = "dark" diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index 2d32f487..56f0674f 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -215,6 +215,10 @@ class StatusBar(urwid.WidgetWrap): r.append("[") r.append(("heading_key", "I")) r.append("gnore:%d]" % len(self.master.options.ignore_hosts)) + elif self.master.options.allow_hosts: + r.append("[") + r.append(("heading_key", "A")) + r.append("llow:%d]" % len(self.master.options.allow_hosts)) if self.master.options.tcp_hosts: r.append("[") r.append(("heading_key", "T")) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index b72e0d77..a0803755 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -1,24 +1,26 @@ +import asyncio import hashlib import json import logging import os.path import re from io import BytesIO -import asyncio +from typing import ClassVar, Optional -import mitmproxy.flow import tornado.escape import tornado.web import tornado.websocket + +import mitmproxy.flow +import mitmproxy.tools.web.master # noqa from mitmproxy import contentviews from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import http from mitmproxy import io from mitmproxy import log -from mitmproxy import version from mitmproxy import optmanager -import mitmproxy.tools.web.master # noqa +from mitmproxy import version def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: @@ -49,6 +51,8 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: f["error"] = flow.error.get_state() if isinstance(flow, http.HTTPFlow): + content_length: Optional[int] + content_hash: Optional[str] if flow.request: if flow.request.raw_content: content_length = len(flow.request.raw_content) @@ -108,6 +112,8 @@ class APIError(tornado.web.HTTPError): class RequestHandler(tornado.web.RequestHandler): + application: "Application" + def write(self, chunk): # Writing arrays on the top level is ok nowadays. # http://flask.pocoo.org/docs/0.11/security/#json-security @@ -190,7 +196,7 @@ class FilterHelp(RequestHandler): class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler): # raise an error if inherited class doesn't specify its own instance. - connections: set = None + connections: ClassVar[set] def open(self): self.connections.add(self) @@ -210,7 +216,7 @@ class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler): class ClientConnection(WebSocketEventBroadcaster): - connections: set = set() + connections: ClassVar[set] = set() class Flows(RequestHandler): @@ -370,7 +376,7 @@ class FlowContent(RequestHandler): original_cd = message.headers.get("Content-Disposition", None) filename = None if original_cd: - filename = re.search('filename=([-\w" .()]+)', original_cd) + filename = re.search(r'filename=([-\w" .()]+)', original_cd) if filename: filename = filename.group(1) if not filename: @@ -432,7 +438,7 @@ class Settings(RequestHandler): def put(self): update = self.json option_whitelist = { - "intercept", "showhost", "upstream_cert", + "intercept", "showhost", "upstream_cert", "ssl_insecure", "rawtcp", "http2", "websocket", "anticache", "anticomp", "stickycookie", "stickyauth", "stream_large_bodies" } @@ -473,7 +479,9 @@ class DnsRebind(RequestHandler): class Application(tornado.web.Application): - def __init__(self, master, debug): + master: "mitmproxy.tools.web.master.WebMaster" + + def __init__(self, master: "mitmproxy.tools.web.master.WebMaster", debug: bool) -> None: self.master = master super().__init__( default_host="dns-rebind-protection", diff --git a/mitmproxy/types.py b/mitmproxy/types.py index f2a26b40..0634e4d7 100644 --- a/mitmproxy/types.py +++ b/mitmproxy/types.py @@ -423,7 +423,7 @@ class TypeManager: for t in types: self.typemap[t.typ] = t() - def get(self, t: type, default=None) -> _BaseType: + def get(self, t: typing.Optional[typing.Type], default=None) -> _BaseType: if type(t) in self.typemap: return self.typemap[type(t)] return self.typemap.get(t, default) diff --git a/mitmproxy/utils/human.py b/mitmproxy/utils/human.py index 5c02b072..3158a294 100644 --- a/mitmproxy/utils/human.py +++ b/mitmproxy/utils/human.py @@ -48,12 +48,14 @@ def parse_size(s: typing.Optional[str]) -> typing.Optional[int]: raise ValueError("Invalid size specification.") -def pretty_duration(secs): +def pretty_duration(secs: typing.Optional[float]) -> str: formatters = [ (100, "{:.0f}s"), (10, "{:2.1f}s"), (1, "{:1.2f}s"), ] + if secs is None: + return "" for limit, formatter in formatters: if secs >= limit: diff --git a/mitmproxy/utils/sliding_window.py b/mitmproxy/utils/sliding_window.py index 0a65f5e4..cb31756d 100644 --- a/mitmproxy/utils/sliding_window.py +++ b/mitmproxy/utils/sliding_window.py @@ -1,5 +1,5 @@ import itertools -from typing import TypeVar, Iterable, Iterator, Tuple, Optional +from typing import TypeVar, Iterable, Iterator, Tuple, Optional, List T = TypeVar('T') @@ -18,7 +18,7 @@ def window(iterator: Iterable[T], behind: int = 0, ahead: int = 0) -> Iterator[T 2 3 None """ # TODO: move into utils - iters = list(itertools.tee(iterator, behind + 1 + ahead)) + iters: List[Iterator[Optional[T]]] = list(itertools.tee(iterator, behind + 1 + ahead)) for i in range(behind): iters[i] = itertools.chain((behind - i) * [None], iters[i]) for i in range(ahead): diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py index 71d1c54c..6e399d8f 100644 --- a/mitmproxy/utils/strutils.py +++ b/mitmproxy/utils/strutils.py @@ -1,10 +1,10 @@ +import codecs import io import re -import codecs -from typing import AnyStr, Optional, cast, Iterable +from typing import Iterable, Optional, Union, cast -def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes]: +def always_bytes(str_or_bytes: Union[str, bytes, None], *encode_args) -> Optional[bytes]: if isinstance(str_or_bytes, bytes) or str_or_bytes is None: return cast(Optional[bytes], str_or_bytes) elif isinstance(str_or_bytes, str): @@ -13,13 +13,15 @@ def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes raise TypeError("Expected str or bytes, but got {}.".format(type(str_or_bytes).__name__)) -def always_str(str_or_bytes: Optional[AnyStr], *decode_args) -> Optional[str]: +def always_str(str_or_bytes: Union[str, bytes, None], *decode_args) -> Optional[str]: """ Returns, str_or_bytes unmodified, if """ - if isinstance(str_or_bytes, str) or str_or_bytes is None: - return cast(Optional[str], str_or_bytes) + if str_or_bytes is None: + return None + if isinstance(str_or_bytes, str): + return cast(str, str_or_bytes) elif isinstance(str_or_bytes, bytes): return str_or_bytes.decode(*decode_args) else: @@ -39,7 +41,6 @@ _control_char_trans_newline = _control_char_trans.copy() for x in ("\r", "\n", "\t"): del _control_char_trans_newline[ord(x)] - _control_char_trans = str.maketrans(_control_char_trans) _control_char_trans_newline = str.maketrans(_control_char_trans_newline) @@ -169,7 +170,7 @@ def split_special_areas( >>> split_special_areas( >>> "test /* don't modify me */ foo", - >>> [r"/\*[\s\S]*?\*/"]) # (regex matching comments) + >>> [r"/\\*[\\s\\S]*?\\*/"]) # (regex matching comments) ["test ", "/* don't modify me */", " foo"] "".join(split_special_areas(x, ...)) == x always holds true. @@ -201,7 +202,7 @@ def escape_special_areas( >>> x = escape_special_areas(x, "{", ["'" + SINGLELINE_CONTENT + "'"]) >>> print(x) if (true) { console.log('�}'); } - >>> x = re.sub(r"\s*{\s*", " {\n ", x) + >>> x = re.sub(r"\\s*{\\s*", " {\n ", x) >>> x = unescape_special_areas(x) >>> print(x) if (true) { diff --git a/mitmproxy/version.py b/mitmproxy/version.py index b40fae8b..363a4bf6 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -25,9 +25,9 @@ def get_dev_version() -> str: stderr=subprocess.STDOUT, cwd=here, ) - last_tag, tag_dist, commit = git_describe.decode().strip().rsplit("-", 2) + last_tag, tag_dist_str, commit = git_describe.decode().strip().rsplit("-", 2) commit = commit.lstrip("g")[:7] - tag_dist = int(tag_dist) + tag_dist = int(tag_dist_str) except Exception: pass else: diff --git a/pathod/pathod.py b/pathod/pathod.py index b330a293..3fe2f901 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -21,6 +21,7 @@ CONFDIR = "~/.mitmproxy" CERTSTORE_BASENAME = "mitmproxy" CA_CERT_NAME = "mitmproxy-ca.pem" DEFAULT_CRAFT_ANCHOR = "/p/" +KEY_SIZE = 2048 logger = logging.getLogger('pathod') @@ -54,7 +55,8 @@ class SSLOptions: self.alpn_select = alpn_select self.certstore = mcerts.CertStore.from_store( os.path.expanduser(confdir), - CERTSTORE_BASENAME + CERTSTORE_BASENAME, + KEY_SIZE ) for i in certs or []: self.certstore.add_cert_file(*i) diff --git a/pathod/test.py b/pathod/test.py index e8c3c84a..b6e5e4d0 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -25,7 +25,7 @@ class Daemon: def __enter__(self): return self - def __exit__(self, type, value, traceback) -> bool: + def __exit__(self, type, value, traceback): self.logfp.truncate(0) self.shutdown() return False diff --git a/release/docker/README.md b/release/docker/README.md index 4511a33a..b676d3ae 100644 --- a/release/docker/README.md +++ b/release/docker/README.md @@ -1,6 +1,6 @@ # mitmproxy -Containerized version of [mitmproxy](https://mitmproxy.org/), an interactive SSL-capable intercepting HTTP proxy. +Containerized version of [mitmproxy](https://mitmproxy.org/): an interactive, SSL/TLS-capable intercepting proxy for HTTP/1, HTTP/2, and WebSockets. # Usage @@ -1,7 +1,7 @@ [flake8] max-line-length = 140 max-complexity = 25 -ignore = E251,E252,C901,W292,W503,W504,W605,E722,E741 +ignore = E251,E252,C901,W292,W503,W504,W605,E722,E741,E126 exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/*,mitmproxy/io/proto/* addons = file,open,basestring,xrange,unicode,long,cmp @@ -22,6 +22,9 @@ exclude_lines = [mypy-mitmproxy.contrib.*] ignore_errors = True +[mypy-tornado.*] +ignore_errors = True + [tool:full_coverage] exclude = mitmproxy/proxy/protocol/base.py @@ -59,7 +62,6 @@ exclude = mitmproxy/net/http/headers.py mitmproxy/net/http/message.py mitmproxy/net/http/multipart.py - mitmproxy/net/http/url.py mitmproxy/net/tcp.py mitmproxy/net/tls.py mitmproxy/options.py @@ -18,7 +18,7 @@ with open(os.path.join(here, "mitmproxy", "version.py")) as f: setup( name="mitmproxy", version=VERSION, - description="An interactive, SSL-capable, man-in-the-middle HTTP proxy for penetration testers and software developers.", + description="An interactive, SSL/TLS-capable intercepting proxy for HTTP/1, HTTP/2, and WebSockets.", long_description=long_description, url="http://mitmproxy.org", author="Aldo Cortesi", @@ -62,25 +62,28 @@ setup( # It is not considered best practice to use install_requires to pin dependencies to specific versions. install_requires=[ "blinker>=1.4, <1.5", - "brotlipy>=0.7.0,<0.8", - "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! - "click>=6.2, <7", + "Brotli>=1.0,<1.1", + "certifi>=2019.9.11", # no semver here - this should always be on the last release! + "click>=7.0,<8", "cryptography>=2.1.4,<2.5", + "flask>=1.1.1,<1.2", "h2>=3.0.1,<4", "hyperframe>=5.1.0,<6", "kaitaistruct>=0.7,<0.9", - "ldap3>=2.5,<2.6", + "ldap3>=2.6.1,<2.7", "passlib>=1.6.5, <1.8", - "protobuf>=3.6.0, <3.7", + "protobuf>=3.6.0, <3.11", "pyasn1>=0.3.1,<0.5", - "pyOpenSSL>=17.5,<18.1", - "pyparsing>=2.1.3,<2.4", + "pyOpenSSL>=19.0.0,<20", + "pyparsing>=2.4.2,<2.5", "pyperclip>=1.6.0,<1.8", - "ruamel.yaml>=0.15,<0.16", - "sortedcontainers>=1.5.4,<2.1", - "tornado>=4.3,<5.2", + "ruamel.yaml>=0.16,<0.17", + "sortedcontainers>=2.1.0,<2.2", + "tornado>=4.3,<7", "urwid>=2.0.1,<2.1", - "wsproto>=0.12.0,<0.13.0", + "wsproto>=0.14.0,<0.15.0", + "publicsuffix2>=2.20190812,<3", + "zstandard>=0.11.0,<0.13.0", ], extras_require={ ':sys_platform == "win32"': [ @@ -88,22 +91,21 @@ setup( ], 'dev': [ "asynctest>=0.12.0", - "flake8>=3.5,<3.7", - "Flask>=1.0,<1.1", - "mypy>=0.590,<0.591", + "flake8>=3.7.8,<3.8", + "Flask>=1.0,<1.2", + "mypy>=0.740,<0.741", "parver>=0.1,<2.0", - "pytest-asyncio>=0.8", - "pytest-cov>=2.5.1,<3", - "pytest-faulthandler>=1.3.1,<2", - "pytest-timeout>=1.2.1,<2", - "pytest-xdist>=1.22,<2", - "pytest>=4.0,<5", - "requests>=2.9.1, <3", - "tox>=3.5,<3.6", - "rstcheck>=2.2, <4.0", + "pytest-asyncio>=0.10.0,<0.11", + "pytest-cov>=2.7.1,<3", + "pytest-timeout>=1.3.3,<2", + "pytest-xdist>=1.29,<2", + "pytest>=5.1.3,<6", + "requests>=2.9.1,<3", + "tox>=3.5,<3.15", + "rstcheck>=2.2,<4.0", ], 'examples': [ - "beautifulsoup4>=4.4.1, <4.7" + "beautifulsoup4>=4.4.1,<4.9" ] } ) diff --git a/test/bench/benchmark.py b/test/bench/benchmark.py index 84ec6005..076ad6c9 100644 --- a/test/bench/benchmark.py +++ b/test/bench/benchmark.py @@ -31,7 +31,8 @@ class Benchmark: stdout=asyncio.subprocess.PIPE ) stdout, _ = await traf.communicate() - open(ctx.options.benchmark_save_path + ".bench", mode="wb").write(stdout) + with open(ctx.options.benchmark_save_path + ".bench", mode="wb") as f: + f.write(stdout) ctx.log.error("Proxy saw %s requests, %s responses" % (self.reqs, self.resps)) ctx.log.error(stdout.decode("ascii")) backend.kill() diff --git a/test/full_coverage_plugin.py b/test/full_coverage_plugin.py index ec30a9f9..ab1206ea 100644 --- a/test/full_coverage_plugin.py +++ b/test/full_coverage_plugin.py @@ -31,8 +31,8 @@ def pytest_configure(config): global no_full_cov enable_coverage = ( - len(config.getoption('file_or_dir')) == 0 and - len(config.getoption('full_cov')) > 0 and + config.getoption('file_or_dir') and len(config.getoption('file_or_dir')) == 0 and + config.getoption('full_cov') and len(config.getoption('full_cov')) > 0 and config.pluginmanager.getplugin("_cov") is not None and config.pluginmanager.getplugin("_cov").cov_controller is not None and config.pluginmanager.getplugin("_cov").cov_controller.cov is not None @@ -55,7 +55,7 @@ def pytest_runtestloop(session): yield return - cov = pytest.config.pluginmanager.getplugin("_cov").cov_controller.cov + cov = session.config.pluginmanager.getplugin("_cov").cov_controller.cov if os.name == 'nt': cov.exclude('pragma: windows no cover') @@ -68,7 +68,7 @@ def pytest_runtestloop(session): yield - coverage_values = dict([(name, 0) for name in pytest.config.option.full_cov]) + coverage_values = dict([(name, 0) for name in session.config.option.full_cov]) prefix = os.getcwd() @@ -92,7 +92,7 @@ def pytest_runtestloop(session): coverage_passed = False -def pytest_terminal_summary(terminalreporter, exitstatus): +def pytest_terminal_summary(terminalreporter, exitstatus, config): global enable_coverage global coverage_values global coverage_passed @@ -119,7 +119,7 @@ def pytest_terminal_summary(terminalreporter, exitstatus): terminalreporter.write(msg, **markup) else: msg = 'SUCCESS: Full test coverage reached in modules and files:\n' - msg += '{}\n\n'.format('\n'.join(pytest.config.option.full_cov)) + msg += '{}\n\n'.format('\n'.join(config.option.full_cov)) terminalreporter.write(msg, green=True) msg = '\nExcluded files:\n' diff --git a/test/individual_coverage.py b/test/individual_coverage.py index 097b290f..54180eac 100755 --- a/test/individual_coverage.py +++ b/test/individual_coverage.py @@ -19,29 +19,29 @@ def run_tests(src, test, fail): e = pytest.main([ '-qq', '--disable-pytest-warnings', - '--no-faulthandler', '--cov', src.replace('.py', '').replace('/', '.'), '--cov-fail-under', '100', '--cov-report', 'term-missing:skip-covered', + '-o', 'faulthandler_timeout=0', test ]) if e == 0: if fail: - print("UNEXPECTED SUCCESS:", src, "Please remove this file from setup.cfg tool:individual_coverage/exclude.") + print("FAIL DUE TO UNEXPECTED SUCCESS:", src, "Please remove this file from setup.cfg tool:individual_coverage/exclude.") e = 42 else: - print("SUCCESS: ", src) + print("Success:", src) else: if fail: - print("IGNORING FAIL: ", src) + print("Ignoring allowed fail:", src) e = 0 else: cov = [l for l in stdout.getvalue().split("\n") if (src in l) or ("was never imported" in l)] if len(cov) == 1: - print("FAIL: ", cov[0]) + print("FAIL:", cov[0]) else: - print("FAIL: ", src, test, stdout.getvalue(), stdout.getvalue()) + print("FAIL:", src, test, stdout.getvalue(), stdout.getvalue()) print(stderr.getvalue()) print(stdout.getvalue()) diff --git a/test/mitmproxy/addons/onboardingapp/test_app.py b/test/mitmproxy/addons/onboardingapp/test_app.py deleted file mode 100644 index 777ab4dd..00000000 --- a/test/mitmproxy/addons/onboardingapp/test_app.py +++ /dev/null @@ -1 +0,0 @@ -# TODO: write tests diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index f4bb0f64..c86e0c7d 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -14,29 +14,23 @@ from unittest import mock @pytest.fixture def get_request(): return tflow.tflow( - req=tutils.treq( - method=b'GET', - content=b'', - path=b"/path?a=foo&a=bar&b=baz" - ) - ) + req=tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz")) @pytest.fixture def post_request(): return tflow.tflow( - req=tutils.treq( - method=b'POST', - headers=(), - content=bytes(range(256)) - ) - ) + req=tutils.treq(method=b'POST', headers=(), content=bytes(range(256)))) @pytest.fixture def patch_request(): return tflow.tflow( - req=tutils.treq(method=b'PATCH', path=b"/path?query=param") + req=tutils.treq( + method=b'PATCH', + content=b'content', + path=b"/path?query=param" + ) ) @@ -47,7 +41,7 @@ def tcp_flow(): class TestExportCurlCommand: def test_get(self, get_request): - result = """curl -H 'header:qvalue' -H 'content-length:0' 'http://address:22/path?a=foo&a=bar&b=baz'""" + result = """curl -H 'header:qvalue' 'http://address:22/path?a=foo&a=bar&b=baz'""" assert export.curl_command(get_request) == result def test_post(self, post_request): @@ -67,7 +61,7 @@ class TestExportCurlCommand: class TestExportHttpieCommand: def test_get(self, get_request): - result = """http GET http://address:22/path?a=foo&a=bar&b=baz 'header:qvalue' 'content-length:0'""" + result = """http GET http://address:22/path?a=foo&a=bar&b=baz 'header:qvalue'""" assert export.httpie_command(get_request) == result def test_post(self, post_request): diff --git a/test/mitmproxy/addons/test_session.py b/test/mitmproxy/addons/test_session.py index 20feb69d..97351426 100644 --- a/test/mitmproxy/addons/test_session.py +++ b/test/mitmproxy/addons/test_session.py @@ -68,7 +68,8 @@ class TestSession: os.remove(path) con = sqlite3.connect(path) script_path = pkg_data.path("io/sql/session_create.sql") - qry = open(script_path, 'r').read() + with open(script_path) as f: + qry = f.read() with con: con.executescript(qry) blob = b'blob_of_data' diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index 976c14b7..f5088a68 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -471,7 +471,7 @@ def test_focus(): v = view.View() v.add([tft()]) f = view.Focus(v) - assert f.index is 0 + assert f.index == 0 assert f.flow is v[0] # Start empty diff --git a/test/mitmproxy/coretypes/test_basethread.py b/test/mitmproxy/coretypes/test_basethread.py index 4a383fea..6b0ae154 100644 --- a/test/mitmproxy/coretypes/test_basethread.py +++ b/test/mitmproxy/coretypes/test_basethread.py @@ -4,4 +4,4 @@ from mitmproxy.coretypes import basethread def test_basethread(): t = basethread.BaseThread('foobar') - assert re.match('foobar - age: \d+s', t._threadinfo()) + assert re.match(r'foobar - age: \d+s', t._threadinfo()) diff --git a/test/mitmproxy/data/pf01 b/test/mitmproxy/data/pf01 index 3139a289..019a6b76 100644 --- a/test/mitmproxy/data/pf01 +++ b/test/mitmproxy/data/pf01 @@ -1,4 +1,10 @@ No ALTQ support in kernel ALTQ related functions disabled +ALL tcp 192.168.1.111:40001 -> 5.5.5.6:80 FIN_WAIT_2:FIN_WAIT_2 ALL tcp 127.0.0.1:8080 <- 5.5.5.6:80 <- 192.168.1.111:40001 FIN_WAIT_2:FIN_WAIT_2 +ALL tcp 192.168.1.111:40000 -> 5.5.5.5:80 ESTABLISHED:ESTABLISHED ALL tcp 127.0.0.1:8080 <- 5.5.5.5:80 <- 192.168.1.111:40000 ESTABLISHED:ESTABLISHED +ALL tcp 2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db[40002] -> 2a03:2880:f21f:c5:face:b00c::167[443] ESTABLISHED:ESTABLISHED +ALL tcp ::1[8080] <- 2a03:2880:f21f:c5:face:b00c::167[443] <- 2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db[40002] ESTABLISHED:ESTABLISHED +ALL tcp 2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db[40003] -> 2a03:2880:f21f:c5:face:b00c::167[443] FIN_WAIT_2:FIN_WAIT_2 +ALL tcp ::1[6970] <- 2a03:2880:f21f:c5:face:b00c::167[443] <- 2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db[40003] FIN_WAIT_2:FIN_WAIT_2
\ No newline at end of file diff --git a/test/mitmproxy/net/http/test_cookies.py b/test/mitmproxy/net/http/test_cookies.py index 74233cca..06cfe1d3 100644 --- a/test/mitmproxy/net/http/test_cookies.py +++ b/test/mitmproxy/net/http/test_cookies.py @@ -27,7 +27,7 @@ cookie_pairs = [ [["one", "uno"], ["two", "due"]] ], [ - 'one="uno"; two="\due"', + 'one="uno"; two="\\due"', [["one", "uno"], ["two", "due"]] ], [ @@ -70,7 +70,7 @@ def test_read_key(): def test_read_quoted_string(): tokens = [ [('"foo" x', 0), ("foo", 5)], - [('"f\oo" x', 0), ("foo", 6)], + [('"f\\oo" x', 0), ("foo", 6)], [(r'"f\\o" x', 0), (r"f\o", 6)], [(r'"f\\" x', 0), (r"f" + '\\', 5)], [('"fo\\\"" x', 0), ("fo\"", 6)], diff --git a/test/mitmproxy/net/http/test_encoding.py b/test/mitmproxy/net/http/test_encoding.py index 8dac12cb..7f768f39 100644 --- a/test/mitmproxy/net/http/test_encoding.py +++ b/test/mitmproxy/net/http/test_encoding.py @@ -19,6 +19,7 @@ def test_identity(encoder): 'gzip', 'br', 'deflate', + 'zstd', ]) def test_encoders(encoder): """ diff --git a/test/mitmproxy/net/http/test_response.py b/test/mitmproxy/net/http/test_response.py index f3470384..27c16be6 100644 --- a/test/mitmproxy/net/http/test_response.py +++ b/test/mitmproxy/net/http/test_response.py @@ -148,7 +148,7 @@ class TestResponseUtils: def test_refresh(self): r = tresp() n = time.time() - r.headers["date"] = email.utils.formatdate(n) + r.headers["date"] = email.utils.formatdate(n, usegmt=True) pre = r.headers["date"] r.refresh(946681202) assert pre == r.headers["date"] diff --git a/test/mitmproxy/net/http/test_url.py b/test/mitmproxy/net/http/test_url.py index ecf8e896..48277859 100644 --- a/test/mitmproxy/net/http/test_url.py +++ b/test/mitmproxy/net/http/test_url.py @@ -49,6 +49,17 @@ def test_parse(): url.parse('http://lo[calhost') +def test_ascii_check(): + + test_url = "https://xyz.tax-edu.net?flag=selectCourse&lc_id=42825&lc_name=茅莽莽猫氓猫氓".encode() + scheme, host, port, full_path = url.parse(test_url) + assert scheme == b'https' + assert host == b'xyz.tax-edu.net' + assert port == 443 + assert full_path == b'/?flag%3DselectCourse%26lc_id%3D42825%26lc_name%3D%E8%8C%85%E8%8E%BD%E8%8E' \ + b'%BD%E7%8C%AB%E6%B0%93%E7%8C%AB%E6%B0%93' + + @pytest.mark.skipif(sys.version_info < (3, 6), reason='requires Python 3.6 or higher') def test_parse_port_range(): # Port out of range @@ -61,6 +72,7 @@ def test_unparse(): assert url.unparse("http", "foo.com", 80, "/bar") == "http://foo.com/bar" assert url.unparse("https", "foo.com", 80, "") == "https://foo.com:80" assert url.unparse("https", "foo.com", 443, "") == "https://foo.com" + assert url.unparse("https", "foo.com", 443, "*") == "https://foo.com" # We ignore the byte 126: '~' because of an incompatibility in Python 3.6 and 3.7 @@ -131,3 +143,7 @@ def test_unquote(): assert url.unquote("foo") == "foo" assert url.unquote("foo%20bar") == "foo bar" assert url.unquote(surrogates_quoted) == surrogates + + +def test_hostport(): + assert url.hostport(b"https", b"foo.com", 8080) == b"foo.com:8080" diff --git a/test/mitmproxy/net/test_tcp.py b/test/mitmproxy/net/test_tcp.py index b6bb7cc1..22a306dc 100644 --- a/test/mitmproxy/net/test_tcp.py +++ b/test/mitmproxy/net/test_tcp.py @@ -102,7 +102,7 @@ class TestServerBind(tservers.ServerTestBase): # We may get an ipv4-mapped ipv6 address here, e.g. ::ffff:127.0.0.1. # Those still appear as "127.0.0.1" in the table, so we need to strip the prefix. peername = self.connection.getpeername() - address = re.sub("^::ffff:(?=\d+.\d+.\d+.\d+$)", "", peername[0]) + address = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", peername[0]) port = peername[1] self.wfile.write(str((address, port)).encode()) diff --git a/test/mitmproxy/net/test_tls.py b/test/mitmproxy/net/test_tls.py index 489bf89f..c4e76bc6 100644 --- a/test/mitmproxy/net/test_tls.py +++ b/test/mitmproxy/net/test_tls.py @@ -87,13 +87,13 @@ def test_get_client_hello(): rfile = io.BufferedReader(io.BytesIO( FULL_CLIENT_HELLO_NO_EXTENSIONS[:30] )) - with pytest.raises(exceptions.TlsProtocolException, message="Unexpected EOF"): + with pytest.raises(exceptions.TlsProtocolException, match="Unexpected EOF"): tls.get_client_hello(rfile) rfile = io.BufferedReader(io.BytesIO( b"GET /" )) - with pytest.raises(exceptions.TlsProtocolException, message="Expected TLS record"): + with pytest.raises(exceptions.TlsProtocolException, match="Expected TLS record"): tls.get_client_hello(rfile) @@ -116,7 +116,7 @@ class TestClientHello: ) c = tls.ClientHello(data) assert repr(c) - assert c.sni == 'example.com' + assert c.sni == b'example.com' assert c.cipher_suites == [ 49195, 49199, 49196, 49200, 52393, 52392, 52244, 52243, 49161, 49171, 49162, 49172, 156, 157, 47, 53, 10 @@ -153,5 +153,5 @@ class TestClientHello: b"\x01\x00\x00\x03" + # handshake header b"foo" )) - with pytest.raises(exceptions.TlsProtocolException, message='Cannot parse Client Hello'): + with pytest.raises(exceptions.TlsProtocolException, match='Cannot parse Client Hello'): tls.ClientHello.from_file(rfile) diff --git a/test/mitmproxy/platform/test_pf.py b/test/mitmproxy/platform/test_pf.py index 9795a2db..4a7dfe75 100644 --- a/test/mitmproxy/platform/test_pf.py +++ b/test/mitmproxy/platform/test_pf.py @@ -19,3 +19,8 @@ class TestLookup: pf.lookup("192.168.1.112", 40000, d) with pytest.raises(Exception, match="Could not resolve original destination"): pf.lookup("192.168.1.111", 40001, d) + assert pf.lookup("2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db", 40002, d) == ("2a03:2880:f21f:c5:face:b00c::167", 443) + with pytest.raises(Exception, match="Could not resolve original destination"): + pf.lookup("2a01:e35:8bae:50f0:396f:e6c7:f4f1:f3db", 40003, d) + with pytest.raises(Exception, match="Could not resolve original destination"): + pf.lookup("2a01:e35:face:face:face:face:face:face", 40003, d) diff --git a/test/mitmproxy/proxy/test_config.py b/test/mitmproxy/proxy/test_config.py index 1da031c6..1319d1a9 100644 --- a/test/mitmproxy/proxy/test_config.py +++ b/test/mitmproxy/proxy/test_config.py @@ -17,3 +17,12 @@ class TestProxyConfig: opts.certs = [tdata.path("mitmproxy/data/dumpfile-011")] with pytest.raises(exceptions.OptionsError, match="Invalid certificate format"): ProxyConfig(opts) + + def test_cannot_set_both_allow_and_filter_options(self): + opts = options.Options() + opts.ignore_hosts = ["foo"] + opts.allow_hosts = ["bar"] + with pytest.raises(exceptions.OptionsError, match="--ignore-hosts and --allow-hosts are " + "mutually exclusive; please choose " + "one."): + ProxyConfig(opts) diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 01ab068d..b5852d60 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -78,6 +78,16 @@ class TcpMixin: self.options.ignore_hosts = self._ignore_backup del self._ignore_backup + def _allow_on(self): + assert not hasattr(self, "_allow_backup") + self._allow_backup = self.options.allow_hosts + self.options.allow_hosts = ["(127.0.0.1|None):\\d+"] + self.options.allow_hosts + + def _allow_off(self): + assert hasattr(self, "_allow_backup") + self.options.allow_hosts = self._allow_backup + del self._allow_backup + def test_ignore(self): n = self.pathod("304") self._ignore_on() @@ -111,6 +121,40 @@ class TcpMixin: self._ignore_off() + def test_allow(self): + n = self.pathod("304") + self._allow_on() + i = self.pathod("305") + i2 = self.pathod("306") + self._allow_off() + + assert n.status_code == 304 + assert i.status_code == 305 + assert i2.status_code == 306 + + assert any(f.response.status_code == 304 for f in self.master.state.flows) + assert any(f.response.status_code == 305 for f in self.master.state.flows) + assert any(f.response.status_code == 306 for f in self.master.state.flows) + + # Test that we get the original SSL cert + if self.ssl: + i_cert = certs.Cert(i.sslinfo.certchain[0]) + i2_cert = certs.Cert(i2.sslinfo.certchain[0]) + n_cert = certs.Cert(n.sslinfo.certchain[0]) + + assert i_cert == i2_cert + assert i_cert != n_cert + + # Test Non-HTTP traffic + spec = "200:i0,@100:d0" # this results in just 100 random bytes + # mitmproxy responds with bad gateway + assert self.pathod(spec).status_code == 502 + self._allow_on() + + self.pathod(spec) # pathoc parses answer as HTTP + + self._allow_off() + def _tcpproxy_on(self): assert not hasattr(self, "_tcpproxy_backup") self._tcpproxy_backup = self.options.tcp_hosts @@ -852,10 +896,12 @@ class TestUpstreamProxySSL( def _host_pattern_on(self, attr): """ - Updates config.check_tcp or check_ignore, depending on attr. + Updates config.check_tcp or check_filter, depending on attr. """ assert not hasattr(self, "_ignore_%s_backup" % attr) backup = [] + handle = attr + attr = "filter" if attr in ["allow", "ignore"] else attr for proxy in self.chain: old_matcher = getattr( proxy.tmaster.server.config, @@ -865,12 +911,13 @@ class TestUpstreamProxySSL( setattr( proxy.tmaster.server.config, "check_%s" % attr, - HostMatcher([".+:%s" % self.server.port] + old_matcher.patterns) + HostMatcher(handle, [".+:%s" % self.server.port] + old_matcher.patterns) ) setattr(self, "_ignore_%s_backup" % attr, backup) def _host_pattern_off(self, attr): + attr = "filter" if attr in ["allow", "ignore"] else attr backup = getattr(self, "_ignore_%s_backup" % attr) for proxy in reversed(self.chain): setattr( diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index 3ec58760..70d41511 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -43,17 +43,17 @@ class TestConcurrent(tservers.MasterTest): assert await tctx.master.await_log("decorator not supported") def test_concurrent_class(self, tdata): - with taddons.context() as tctx: - sc = tctx.script( - tdata.path( - "mitmproxy/data/addonscripts/concurrent_decorator_class.py" - ) + with taddons.context() as tctx: + sc = tctx.script( + tdata.path( + "mitmproxy/data/addonscripts/concurrent_decorator_class.py" ) - f1, f2 = tflow.tflow(), tflow.tflow() - tctx.cycle(sc, f1) - tctx.cycle(sc, f2) - start = time.time() - while time.time() - start < 5: - if f1.reply.state == f2.reply.state == "committed": - return - raise ValueError("Script never acked") + ) + f1, f2 = tflow.tflow(), tflow.tflow() + tctx.cycle(sc, f1) + tctx.cycle(sc, f2) + start = time.time() + while time.time() - start < 5: + if f1.reply.state == f2.reply.state == "committed": + return + raise ValueError("Script never acked") diff --git a/test/mitmproxy/test_certs.py b/test/mitmproxy/test_certs.py index b8ad1d36..37604f54 100644 --- a/test/mitmproxy/test_certs.py +++ b/test/mitmproxy/test_certs.py @@ -35,20 +35,20 @@ from ..conftest import skip_windows class TestCertStore: def test_create_explicit(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) assert ca.get_cert(b"foo", []) - ca2 = certs.CertStore.from_store(str(tmpdir), "test") + ca2 = certs.CertStore.from_store(str(tmpdir), "test", 2048) assert ca2.get_cert(b"foo", []) assert ca.default_ca.get_serial_number() == ca2.default_ca.get_serial_number() def test_create_no_common_name(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) assert ca.get_cert(None, [])[0].cn is None def test_create_tmp(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) assert ca.get_cert(b"foo.com", []) assert ca.get_cert(b"foo.com", []) assert ca.get_cert(b"*.foo.com", []) @@ -57,7 +57,7 @@ class TestCertStore: assert r[1] == ca.default_privatekey def test_sans(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) c1 = ca.get_cert(b"foo.com", [b"*.bar.com"]) ca.get_cert(b"foo.bar.com", []) # assert c1 == c2 @@ -65,13 +65,13 @@ class TestCertStore: assert not c1 == c3 def test_sans_change(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) ca.get_cert(b"foo.com", [b"*.bar.com"]) cert, key, chain_file = ca.get_cert(b"foo.bar.com", [b"*.baz.com"]) assert b"*.baz.com" in cert.altnames def test_expire(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) ca.STORE_CAP = 3 ca.get_cert(b"one.com", []) ca.get_cert(b"two.com", []) @@ -95,8 +95,8 @@ class TestCertStore: assert (b"four.com", ()) in ca.certs def test_overrides(self, tmpdir): - ca1 = certs.CertStore.from_store(str(tmpdir.join("ca1")), "test") - ca2 = certs.CertStore.from_store(str(tmpdir.join("ca2")), "test") + ca1 = certs.CertStore.from_store(str(tmpdir.join("ca1")), "test", 2048) + ca2 = certs.CertStore.from_store(str(tmpdir.join("ca2")), "test", 2048) assert not ca1.default_ca.get_serial_number() == ca2.default_ca.get_serial_number() dc = ca2.get_cert(b"foo.com", [b"sans.example.com"]) @@ -124,7 +124,7 @@ class TestCertStore: class TestDummyCert: def test_with_ca(self, tmpdir): - ca = certs.CertStore.from_store(str(tmpdir), "test") + ca = certs.CertStore.from_store(str(tmpdir), "test", 2048) r = certs.dummy_cert( ca.default_privatekey, ca.default_ca, diff --git a/test/mitmproxy/test_flowfilter.py b/test/mitmproxy/test_flowfilter.py index 4eb37d81..d53cec7d 100644 --- a/test/mitmproxy/test_flowfilter.py +++ b/test/mitmproxy/test_flowfilter.py @@ -28,6 +28,9 @@ class TestParsing: self._dump(p) assert len(p.lst) == 2 + def test_non_ascii(self): + assert flowfilter.parse("~s шгн") + def test_naked_url(self): a = flowfilter.parse("foobar ~h rex") assert a.lst[0].expr == "foobar" @@ -173,10 +176,30 @@ class TestMatchingHTTPFlow: assert not self.q("~bq message", q) assert not self.q("~bq message", s) + s.response.text = 'яч' # Cyrillic + assert self.q("~bs яч", s) + s.response.text = '测试' # Chinese + assert self.q('~bs 测试', s) + s.response.text = 'ॐ' # Hindi + assert self.q('~bs ॐ', s) + s.response.text = 'لله' # Arabic + assert self.q('~bs لله', s) + s.response.text = 'θεός' # Greek + assert self.q('~bs θεός', s) + s.response.text = 'לוהים' # Hebrew + assert self.q('~bs לוהים', s) + s.response.text = '神' # Japanese + assert self.q('~bs 神', s) + s.response.text = '하나님' # Korean + assert self.q('~bs 하나님', s) + s.response.text = 'Äÿ' # Latin + assert self.q('~bs Äÿ', s) + assert not self.q("~bs nomatch", s) assert not self.q("~bs content", q) assert not self.q("~bs content", s) assert not self.q("~bs message", q) + s.response.text = 'message' assert self.q("~bs message", s) def test_body(self): diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 00086c4b..c8cf6c33 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -1,4 +1,5 @@ import argparse +import platform from unittest import mock import pytest @@ -52,8 +53,11 @@ class TestProcessProxyOptions: class TestProxyServer: @skip_windows + @pytest.mark.skipif(platform.mac_ver()[0].split('.')[:2] == ['10', '14'], + reason='Skipping due to macOS Mojave') def test_err(self): - # binding to 0.0.0.0:1 works without special permissions on Windows + # binding to 0.0.0.0:1 works without special permissions on Windows and + # macOS Mojave conf = ProxyConfig(options.Options(listen_port=1)) with pytest.raises(Exception, match="Error starting proxy server"): ProxyServer(conf) diff --git a/test/mitmproxy/utils/test_human.py b/test/mitmproxy/utils/test_human.py index faf35f72..6f8bf732 100644 --- a/test/mitmproxy/utils/test_human.py +++ b/test/mitmproxy/utils/test_human.py @@ -47,6 +47,7 @@ def test_pretty_duration(): assert human.pretty_duration(10000) == "10000s" assert human.pretty_duration(1.123) == "1.12s" assert human.pretty_duration(0.123) == "123ms" + assert human.pretty_duration(None) == "" def test_format_address(): @@ -1,5 +1,5 @@ [tox] -envlist = py36, py37, lint +envlist = py35, py36, py37, lint, individual_coverage, wheeltest, docs skipsdist = True toxworkdir={env:TOX_WORK_DIR:.tox} @@ -44,8 +44,8 @@ commands = passenv = TRAVIS_* APPVEYOR_* AWS_* TWINE_* DOCKER_* RTOOL_KEY WHEEL DOCKER PYINSTALLER WININSTALLER deps = -rrequirements.txt - pyinstaller==3.4 - twine==1.12.1 + pyinstaller==3.5 + twine==2.0.0 awscli commands = mitmdump --version diff --git a/web/src/js/ducks/ui/keyboard.js b/web/src/js/ducks/ui/keyboard.js index ed4dbba5..007d24db 100644 --- a/web/src/js/ducks/ui/keyboard.js +++ b/web/src/js/ducks/ui/keyboard.js @@ -6,7 +6,7 @@ import * as modalActions from "./modal" export function onKeyDown(e) { //console.debug("onKeyDown", e) - if (e.ctrlKey) { + if (e.ctrlKey || e.metaKey) { return () => { } } |