diff options
| author | Marcelo Glezer <mg@tekii.com.ar> | 2014-12-11 14:54:14 -0300 |
|---|---|---|
| committer | Marcelo Glezer <mg@tekii.com.ar> | 2014-12-11 14:54:14 -0300 |
| commit | 4952643a0d76eb1e9bd51cbbe95c565ae48b97a2 (patch) | |
| tree | f43fc647bdfabb522bdef32e21ea4a36404cc311 | |
| parent | 83b1d4e0e0490e5be05943da459c925a3ee3ff14 (diff) | |
| parent | ffb95a1db742d71d7671f9e9c6db552774bb0ead (diff) | |
| download | mitmproxy-4952643a0d76eb1e9bd51cbbe95c565ae48b97a2.tar.gz mitmproxy-4952643a0d76eb1e9bd51cbbe95c565ae48b97a2.tar.bz2 mitmproxy-4952643a0d76eb1e9bd51cbbe95c565ae48b97a2.zip | |
Merge remote-tracking branch 'base/master'
70 files changed, 1979 insertions, 781 deletions
diff --git a/.coveragerc b/.coveragerc index 7a4e3ab7..70ff48e7 100644 --- a/.coveragerc +++ b/.coveragerc @@ -2,5 +2,5 @@ branch = True [report] -omit = *contrib*, *tnetstring*, *platform*, *console* +omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py include = *libmproxy* @@ -1,3 +1,53 @@ + +15 November 2014: mitmproxy 0.11.1: + + * Bug fixes: connection leaks some crashes + + +7 November 2014: mitmproxy 0.11: + + * Performance improvements for mitmproxy console + + * SOCKS5 proxy mode allows mitmproxy to act as a SOCKS5 proxy server + + * Data streaming for response bodies exceeding a threshold + (bradpeabody@gmail.com) + + * Ignore hosts or IP addresses, forwarding both HTTP and HTTPS traffic + untouched + + * Finer-grained control of traffic replay, including options to ignore + contents or parameters when matching flows (marcelo.glezer@gmail.com) + + * Pass arguments to inline scripts + + * Configurable size limit on HTTP request and response bodies + + * Per-domain specification of interception certificates and keys (see + --cert option) + + * Certificate forwarding, relaying upstream SSL certificates verbatim (see + --cert-forward) + + * Search and highlighting for HTTP request and response bodies in + mitmproxy console (pedro@worcel.com) + + * Transparent proxy support on Windows + + * Improved error messages and logging + + * Support for FreeBSD in transparent mode, using pf (zbrdge@gmail.com) + + * Content view mode for WBXML (davidshaw835@air-watch.com) + + * Better documentation, with a new section on proxy modes + + * Generic TCP proxy mode + + * Countless bugfixes and other small improvements + + + 28 January 2014: mitmproxy 0.10: * Support for multiple scripts and multiple script arguments diff --git a/CONTRIBUTORS b/CONTRIBUTORS index bed636fa..a9688d92 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1,51 +1,65 @@ - 854 Aldo Cortesi - 64 Maximilian Hils + 902 Aldo Cortesi + 323 Maximilian Hils 18 Henrik Nordstrom 13 Thomas Roth + 12 Pedro Worcel 11 Stephen Altamirano 10 András Veres-Szentkirályi - 8 Jason A. Novak 8 Rouli + 8 Jason A. Novak 7 Alexis Hildebrandt - 6 Pedro Worcel 5 Tomaz Muraus + 5 Brad Peabody 5 Matthias Urlichs 4 root - 4 Bryan Bishop 4 Marc Liyanage 4 Valtteri Virtanen - 3 Kyle Manna + 4 Bryan Bishop 3 Chris Neasbitt - 2 alts - 2 Heikki Hannikainen - 2 Jim Lloyd + 3 Zack B + 3 Eli Shvartsman + 3 Kyle Manna 2 Michael Frister + 2 Bennett Blodinger + 2 Jim Lloyd 2 Rob Wills - 2 Jaime Soriano Pastor 2 israel + 2 Jaime Soriano Pastor + 2 Heikki Hannikainen 2 Mark E. Haase + 2 alts + 1 davidpshaw + 1 deployable + 1 joebowbeer + 1 meeee + 1 phil plante + 1 Michael Bisbjerg + 1 Andy Smith + 1 Dan Wilbraham + 1 David Shaw + 1 Eric Entzel + 1 Felix Wolfsteller + 1 Henrik Nordström + 1 Ivaylo Popov + 1 JC + 1 Jakub Nawalaniec + 1 James Billingham + 1 Jean Regisser + 1 Kit Randel + 1 Marcelo Glezer + 1 Mathieu Mitchell + 1 Mikhail Korobov + 1 Nicolas Esteves + 1 Oleksandr Sheremet 1 Paul 1 Rich Somerfield 1 Rory McCann - 1 Felix Wolfsteller 1 Rune Halvorsen 1 Sahn Lam - 1 Eric Entzel - 1 Dan Wilbraham + 1 Seppo Yli-Olli + 1 Sergey Chipiga + 1 Steven Van Acker 1 Ulrich Petri - 1 Andy Smith + 1 Vyacheslav Bakhmutov 1 Yuangxuan Wang 1 capt8bit - 1 joebowbeer - 1 meeee - 1 James Billingham - 1 Jakub Nawalaniec - 1 JC - 1 Kit Randel - 1 phil plante - 1 Mathieu Mitchell - 1 Ivaylo Popov - 1 Henrik Nordström - 1 Michael Bisbjerg - 1 Nicolas Esteves - 1 Oleksandr Sheremet diff --git a/MANIFEST.in b/MANIFEST.in index efe18f43..cc048b61 100644 --- a/MANIFEST.in +++ b/MANIFEST.in @@ -2,10 +2,7 @@ include mitmproxy mitmdump include LICENSE CHANGELOG CONTRIBUTORS README.txt exclude README.mkd recursive-include examples * -recursive-exclude examples *.pyc *.pyo *.swo *.swp recursive-include doc * -recursive-exclude doc *.pyc *.pyo *.swo *.swp recursive-include test * -recursive-exclude test *.pyc *.pyo *.swo *.swp recursive-include libmproxy * -recursive-exclude libmproxy *.pyc *.pyo *.swo *.swp
\ No newline at end of file +recursive-exclude * *.pyc *.pyo *.swo *.swp
\ No newline at end of file @@ -13,6 +13,9 @@ mitmproxy.org website: [mitmproxy.org](http://mitmproxy.org). +You can find complete directions for installing mitmproxy [here](http://mitmproxy.org/doc/install.html). + + Features -------- @@ -26,17 +29,17 @@ Features - SSL certificates for interception are generated on the fly. - And much, much more. +__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. On Windows, +only mitmdump is supported, which does not have a graphical user interface. -Installation ------------- -The recommended way to install mitmproxy is running <code>pip install mitmproxy</code>. -For convenience, we provide binary packages on [mitmproxy.org](http://mitmproxy.org/). +Hacking +------- -Requirements ------------- +### Requirements + * [Python](http://www.python.org) 2.7.x. * [netlib](http://pypi.python.org/pypi/netlib), version matching mitmproxy. @@ -49,28 +52,35 @@ Optional packages for extended content decoding: * [cssutils](http://cthedot.de/cssutils/) version 1.0 or newer. For convenience, all optional dependencies can be installed with -`pip install mitmproxy[contenviews]` - -__mitmproxy__ is tested and developed on OSX, Linux and OpenBSD. On Windows, -only mitmdump is supported, which does not have a graphical user interface. - -Hacking -------- +`pip install "mitmproxy[contentviews]"` -The following components are needed if you plan to hack on mitmproxy: +### Setting up a dev environment -* The test suite requires the `dev` extra requirements listed in [setup.py](https://github.com/mitmproxy/mitmproxy/blob/master/setup.py) and [pathod](http://pathod.net), version matching mitmproxy. -* Rendering the documentation requires [countershape](http://github.com/cortesi/countershape). +The following procedure is recommended to set up your dev environment: -For convenience, the following procedure is recommended to set up your environment: ``` $ git clone https://github.com/mitmproxy/mitmproxy.git $ cd mitmproxy $ pip install --src . -r requirements.txt ``` + This installs the latest GitHub versions of mitmproxy, netlib and pathod into `mitmproxy/`. All other development dependencies save countershape are installed into their usual locations. + +### Testing + +The test suite requires the `dev` extra requirements listed in [setup.py](https://github.com/mitmproxy/mitmproxy/blob/master/setup.py) and [pathod](http://pathod.net), version matching mitmproxy. Install these with: + +`pip install "mitmproxy[dev]"` + + Please ensure that all patches are accompanied by matching changes in the test suite. The project maintains 100% test coverage. + +### Docs + +Rendering the documentation requires [countershape](http://github.com/cortesi/countershape). After installation, you can render the documentation to the doc like this: + +`cshape doc-src doc` diff --git a/doc-src/_nav.html b/doc-src/_nav.html index 822e9fa6..0ae0fa67 100644 --- a/doc-src/_nav.html +++ b/doc-src/_nav.html @@ -17,12 +17,14 @@ $!nav("serverreplay.html", this, state)!$ $!nav("setheaders.html", this, state)!$ $!nav("passthrough.html", this, state)!$ - $!nav("sticky.html", this, state)!$ + $!nav("proxyauth.html", this, state)!$ $!nav("reverseproxy.html", this, state)!$ + $!nav("responsestreaming.html", this, state)!$ + $!nav("socksproxy.html", this, state)!$ + $!nav("sticky.html", this, state)!$ + $!nav("tcpproxy.html", this, state)!$ $!nav("upstreamproxy.html", this, state)!$ $!nav("upstreamcerts.html", this, state)!$ - $!nav("proxyauth.html", this, state)!$ - $!nav("responsestreaming.html", this, state)!$ <li class="nav-header">Installing Certificates</li> diff --git a/doc-src/features/index.py b/doc-src/features/index.py index 477bb8af..693b4439 100644 --- a/doc-src/features/index.py +++ b/doc-src/features/index.py @@ -9,9 +9,11 @@ pages = [ Page("replacements.html", "Replacements"), Page("responsestreaming.html", "Response Streaming"), Page("reverseproxy.html", "Reverse proxy mode"), + Page("socksproxy.html", "SOCKS Mode"), Page("setheaders.html", "Set Headers"), Page("serverreplay.html", "Server-side replay"), Page("sticky.html", "Sticky cookies and auth"), + Page("tcpproxy.html", "TCP Proxy"), Page("upstreamcerts.html", "Upstream Certs"), Page("upstreamproxy.html", "Upstream proxy mode"), ]
\ No newline at end of file diff --git a/doc-src/features/passthrough.html b/doc-src/features/passthrough.html index 039d6b58..7c830639 100644 --- a/doc-src/features/passthrough.html +++ b/doc-src/features/passthrough.html @@ -1,13 +1,12 @@ -There are a couple of reasons why you may want to exempt some traffic from mitmproxy's interception mechanism: +There are two main reasons why you may want to exempt some traffic from mitmproxy's interception mechanism: - **Certificate pinning:** Some traffic is is protected using [certificate pinning](https://security.stackexchange.com/questions/29988/what-is-certificate-pinning) and mitmproxy's interception leads to errors. For example, Windows Update or the Apple App Store fail to work if mitmproxy is active. -- **Non-HTTP traffic:** WebSockets or other non-http protocols are not supported by mitmproxy yet. You can exempt the - domain from processing, which would otherwise fail. - **Convenience:** You really don't care about some parts of the traffic and just want them to go away. -If you want to ignore traffic from mitmproxy's processing because of large response bodies, check out the +If you want to peek into (SSL-protected) non-HTTP connections, check out the [tcp proxy](@!urlTo("tcpproxy.html")!@) feature. +If you want to ignore traffic from mitmproxy's processing because of large response bodies, take a look at the [response streaming](@!urlTo("responsestreaming.html")!@) feature. ## How it works @@ -74,4 +73,9 @@ Here are some other examples for ignore patterns: --ignore 17\.178\.\d+\.\d+:443 </pre> +### See Also + +- [TCP Proxy](@!urlTo("tcpproxy.html")!@) +- [Response Streaming](@!urlTo("responsestreaming.html")!@) + [^explicithttp]: This stems from an limitation of explicit HTTP proxying: A single connection can be re-used for multiple target domains - a <code>GET http://example.com/</code> request may be followed by a <code>GET http://evil.com/</code> request on the same connection. If we start to ignore the connection after the first request, we would miss the relevant second one.
\ No newline at end of file diff --git a/doc-src/features/responsestreaming.html b/doc-src/features/responsestreaming.html index d20af65c..47fafef7 100644 --- a/doc-src/features/responsestreaming.html +++ b/doc-src/features/responsestreaming.html @@ -47,4 +47,8 @@ When response streaming is enabled, portions of the code which would have otherw on the response body will see an empty response body instead (<code>libmproxy.protocol.http.CONTENT_MISSING</code>). Any modifications will be ignored. Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a <code>Transfer-Encoding: - chunked</code> header, the response will be streamed one chunk at a time.
\ No newline at end of file + chunked</code> header, the response will be streamed one chunk at a time. + +### See Also + +- [Ignore Domains](@!urlTo("passthrough.html")!@) diff --git a/doc-src/features/reverseproxy.html b/doc-src/features/reverseproxy.html index e6de4f33..5ef4efc5 100644 --- a/doc-src/features/reverseproxy.html +++ b/doc-src/features/reverseproxy.html @@ -7,10 +7,46 @@ mitmproxy forwards HTTP proxy requests to an upstream proxy server. <table class="table"> <tbody> <tr> - <th width="20%">command-line</th> <td>-R http[s]://hostname[:port]</td> - </tr> - <tr> - <th>mitmproxy shortcut</th> <td><b>P</b></td> + <th width="20%">command-line</th> <td>-R <i>schema</i>://hostname[:port]</td> </tr> </tbody> </table> + +Here, **schema** is one of http, https, http2https or https2http. The latter +two extended schema specifications control the use of HTTP and HTTPS on +mitmproxy and the upstream server. You can indicate that mitmproxy should use +HTTP, and the upstream server uses HTTPS like this: + + http2https://hostname:port + +And you can indicate that mitmproxy should use HTTPS while the upstream +service uses HTTP like this: + + https2http://hostname:port + + +### Host Header + +In reverse proxy mode, mitmproxy does not rewrite the host header. While often useful, this +may lead to issues with public web servers. For example, consider the following scenario: + + $ python mitmdump -d -R http://example.com/ & + $ curl http://localhost:8080/ + + >> GET https://example.com/ + Host: localhost:8080 + User-Agent: curl/7.35.0 + [...] + + << 404 Not Found 345B + +Since the Host header doesn't match <samp>example.com</samp>, an error is returned.<br> +There are two ways to solve this: +<ol> + <li>Modify the hosts file of your OS so that example.com resolves to 127.0.0.1.</li> + <li> + Instruct mitmproxy to rewrite the host header by passing <kbd>‑‑setheader :~q:Host:example.com</kbd>. + However, keep in mind that absolute URLs within the returned document or HTTP redirects will cause the client application + to bypass the proxy. + </li> +</ol>
\ No newline at end of file diff --git a/doc-src/features/socksproxy.html b/doc-src/features/socksproxy.html new file mode 100644 index 00000000..f436cbf5 --- /dev/null +++ b/doc-src/features/socksproxy.html @@ -0,0 +1,10 @@ + +In this mode, mitmproxy acts as a SOCKS5 proxy server. + +<table class="table"> + <tbody> + <tr> + <th width="20%">command-line</th> <td>--socks</td> + </tr> + </tbody> +</table> diff --git a/doc-src/features/tcpproxy.html b/doc-src/features/tcpproxy.html new file mode 100644 index 00000000..819cf297 --- /dev/null +++ b/doc-src/features/tcpproxy.html @@ -0,0 +1,30 @@ +WebSockets or other non-HTTP protocols are not supported by mitmproxy yet. However, you can exempt hostnames from +processing, so that mitmproxy acts as a generic TCP forwarder. This feature is closely related to the +[ignore domains](@!urlTo("passthrough.html")!@) functionality, but differs in two important aspects: + +- The raw TCP messages are printed to the event log. +- SSL connections will be intercepted. + +Please note that message interception or modification are not possible yet. +If you are not interested in the raw TCP messages, you should use the ignore domains feature. + +## How it works + + +<table class="table"> + <tbody> + <tr> + <th width="20%">command-line</th> <td>--tcp HOST</td> + </tr> + <tr> + <th>mitmproxy shortcut</th> <td><b>T</b></td> + </tr> + </tbody> +</table> + +For a detailed description on the structure of the hostname pattern, please refer to the [Ignore Domains](@!urlTo("passthrough.html")!@) feature. + +### See Also + +- [Ignore Domains](@!urlTo("passthrough.html")!@) +- [Response Streaming](@!urlTo("responsestreaming.html")!@) diff --git a/doc-src/features/upstreamproxy.html b/doc-src/features/upstreamproxy.html index 6039f4df..47bc115d 100644 --- a/doc-src/features/upstreamproxy.html +++ b/doc-src/features/upstreamproxy.html @@ -9,8 +9,19 @@ mitmproxy forwards ordinary HTTP requests to an upstream server. <tr> <th width="20%">command-line</th> <td>-U http://hostname[:port]</td> </tr> - <tr> - <th>mitmproxy shortcut</th> <td><b>U</b></td> - </tr> </tbody> </table> + +Here, **schema** is one of http, https, http2https or https2http. The latter +two extended schema specifications control the use of HTTP and HTTPS on +mitmproxy and the upstream server. You can indicate that mitmproxy should use +HTTP, and the upstream server uses HTTPS like this: + + http2https://hostname:port + +And you can indicate that mitmproxy should use HTTPS while the upstream +service uses HTTP like this: + + https2http://hostname:port + + diff --git a/doc-src/index.html b/doc-src/index.html index 79687ec6..23da7223 100644 --- a/doc-src/index.html +++ b/doc-src/index.html @@ -1,4 +1,27 @@ -@!index_contents!@ +__mitmproxy__ is an interactive, SSL-capable man-in-the-middle proxy for HTTP +with a console interface. +__mitmdump__ is the command-line version of mitmproxy. Think tcpdump for HTTP. + +__libmproxy__ is the library that mitmproxy and mitmdump are built on. + +Documentation, tutorials and distribution packages can be found on the +mitmproxy.org website: + +[mitmproxy.org](http://mitmproxy.org). + + +Features +-------- + +- Intercept HTTP requests and responses and modify them on the fly. +- Save complete HTTP conversations for later replay and analysis. +- Replay the client-side of an HTTP conversations. +- Replay HTTP responses of a previously recorded server. +- Reverse proxy mode to forward traffic to a specified server. +- Transparent proxy mode on OSX and Linux. +- Make scripted changes to HTTP traffic using Python. +- SSL certificates for interception are generated on the fly. +- And much, much more. diff --git a/doc-src/index.py b/doc-src/index.py index b7ab9995..e6064e3a 100644 --- a/doc-src/index.py +++ b/doc-src/index.py @@ -1,6 +1,8 @@ -import os, sys, datetime +import os +import sys +import datetime import countershape -from countershape import Page, Directory, PythonModule, markup, model +from countershape import Page, Directory, markup, model import countershape.template sys.path.insert(0, "..") from libmproxy import filt, version @@ -23,18 +25,18 @@ ns.docMaintainer = "Aldo Cortesi" ns.docMaintainerEmail = "aldo@corte.si" ns.copyright = u"\u00a9 mitmproxy project, %s" % datetime.date.today().year + def mpath(p): p = os.path.join(MITMPROXY_SRC, p) return os.path.expanduser(p) -with open(mpath("README.mkd")) as f: - readme = f.read() - ns.index_contents = readme.split("\n", 1)[1] #remove first line (contains build status) def example(s): d = file(mpath(s)).read().rstrip() extemp = """<div class="example">%s<div class="example_legend">(%s)</div></div>""" return extemp%(countershape.template.Syntax("py")(d), s) + + ns.example = example @@ -73,6 +75,7 @@ def nav(page, current, state): ns.nav = nav ns.navbar = countershape.template.File(None, "_nav.html") + pages = [ Page("index.html", "Introduction"), Page("install.html", "Installation"), diff --git a/doc-src/install.html b/doc-src/install.html index 5d412459..682e317e 100644 --- a/doc-src/install.html +++ b/doc-src/install.html @@ -1,40 +1,33 @@ +## Installing from source + The preferred way to install mitmproxy - whether you're installing the latest release or from source - is to use [pip](http://www.pip-installer.org/). If you don't already have pip on your system, you can find installation instructions [here](http://www.pip-installer.org/en/latest/installing.html). - -## Installing the latest release - -A single command will download and install the latest release of mitmproxy, -along with all its dependencies: - <pre class="terminal"> pip install mitmproxy </pre> +If you also want to install the optional packages AMF, protobuf and CSS +content views, do this: -## Installing from source - -When installing from source, the easiest method is still to use pip. In this -case run: - <pre class="terminal"> -pip install /path/to/source +pip install "mitmproxy[contentviews]" </pre> -Note that if you're installing current git master, you will also have to -install the current git master of [netlib](http://github.com/mitmproxy/netlib) by -hand. ## OSX +The easiest way to get up and running on OSX is to download the pre-built +binary packages from [mitmproxy.org](http://mitmproxy.org). If you still want +to install using pip, there are a few things to keep in mind: + - If you're running a Python interpreter installed with homebrew (or similar), you may have to install some dependencies by hand. - Make sure that XCode is installed from the App Store, and that the command-line tools have been downloaded (XCode/Preferences/Downloads). -- Now use __pip__ to do the installation, as above. There are a few bits of customization you might want to do to make mitmproxy comfortable to use on OSX. The default color scheme is optimized for a dark @@ -64,8 +57,3 @@ from source: - libxslt1-dev - - - - - diff --git a/doc-src/modes.html b/doc-src/modes.html index 77bd1b05..8870009d 100644 --- a/doc-src/modes.html +++ b/doc-src/modes.html @@ -1,210 +1,222 @@ -Mitmproxy comes with several modes of operation, which allow you to use mitmproxy in a variety of scenarios. -This documents briefly explains each mode and possible setups. -<hr> -Mitmproxy has four modes of operation: -<ul> - <li>Regular Mode (this is what you get by default)</li> - <li>Transparent Mode</li> - <li>Reverse Proxy Mode</li> - <li>Upstream Proxy Mode</li> -</ul> - -<p>Now, which one should you pick? Use this flow chart: -</p> - -<img src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"><br><br> + +Mitmproxy has four modes of operation that allow you to use mitmproxy in a +variety of scenarios: + +- **Regular** (the default) +- **Transparent** +- **Reverse Proxy** +- **Upstream Proxy** + +Now, which one should you pick? Use this flow chart: + +<img src="@!urlTo('schematics/proxy-modes-flowchart.png')!@"/> <div class="page-header"> <h1>Regular Proxy</h1> </div> -Mitmproxy's regular mode it the most simple one and the easiest to set up. +Mitmproxy's regular mode is the simplest and the easiest to set up. -<ol> - <li>Start mitmproxy.</li> - <li>Configure your client to use mitmproxy. This means that you either adjust the proxy setting of your local browser - or point an external device to your proxy (which should look like - <a href="@!urlTo('screenshots/ios-manual.png')!@">this</a>).</li> - <li>Quick Check: You can already visit an unencrypted HTTP site over the proxy.</li> - <li>Open the magic domain <strong>mitm.it</strong> and install the certificate for your device.</li> -</ol> +1. Start mitmproxy. +2. Configure your client to use mitmproxy. For instance on IOS, the settings might look like <a href="@!urlTo('screenshots/ios-manual.png')!@">this</a>. +3. Quick Check: You should already be able to visit an unencrypted HTTP site +through the proxy. +4. Open the magic domain <strong>mitm.it</strong> and install the certificate for your device. <div class="well"> - <strong>Heads Up:</strong> Unfortunately, some applications prefer to bypass the HTTP proxy settings of the system - - Android applications are a common example. In these cases, you need to use mitmproxy's transparent mode. +<strong>Heads Up:</strong> Unfortunately, some applications bypass the +system HTTP proxy settings - Android applications are a common example. In +these cases, you need to use mitmproxy's transparent mode. </div> -<p>If you are proxying an external device, your network will probably look like this:</p> +If you are proxying an external device, your network will probably look like this: + <img src="@!urlTo('schematics/proxy-modes-regular.png')!@"> -<br><br> -<p>The square brackets signify the source and destination IP addresses. Your client explicitly connects - to mitmproxy and mitmproxy explicitly connects to the target server. -</p> + +The square brackets signify the source and destination IP addresses. Your +client explicitly connects to mitmproxy and mitmproxy explicitly connects +to the target server. <div class="page-header"> <h1>Transparent Proxy</h1> </div> -When a transparent proxy is used, traffic is redirected into a proxy at the network layer, without any client -configuration being required. This makes transparent proxying ideal for those situations where you can't change client -behaviour. The basic principle is that mitmproxy sits somewhere on the line from the client to the internet and -transparently intercepts the request. In the graphic below, a machine running mitmproxy has been inserted between -the router and the internet: +In transparent mode, traffic is directed into a proxy at the network layer, +without any client configuration required. This makes transparent proxying +ideal for situations where you can't change client behaviour. In the graphic +below, a machine running mitmproxy has been inserted between the router and +the internet: <a href="@!urlTo('schematics/proxy-modes-transparent-1.png')!@"> - <img src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@"></a> -<p>The square brackets signify the source and destination IP addresses. Round brackets mark the next - hop on the <strong>Ethernet</strong>/data link layer. This distinction is important to make: When the packet arrives - at the mitmproxy machine, it must still be addressed to the target server. In other words: A simple IP redirect on - the router does not work - this would remove the target information, leaving mitmproxy unable to - determine the real destination. -</p> + <img src="@!urlTo('schematics/proxy-modes-transparent-1.png')!@"> +</a> + +The square brackets signify the source and destination IP addresses. Round +brackets mark the next hop on the *Ethernet/data link* layer. This distinction +is important: when the packet arrives at the mitmproxy machine, it must still +be addressed to the target server. This means that Network Address Translation +should not be applied before the traffic reaches mitmproxy, since this would +remove the target information, leaving mitmproxy unable to determine the real +destination. + <a href="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"> <img src="@!urlTo('schematics/proxy-modes-transparent-wrong.png')!@"></a> <h2>Common Configurations</h2> -The first graphic is a little bit idealistic: Usually, you'll have your local wireless lan network and no -machines between your router and the internet. Fortunately, there are other ways to configure your network: -(a) Configuring the client to use a custom gateway/router/"next hop", (b) Implementing custom routing on the router -or (c) setting up a separate wireless network router which gets proxied. -There are of course other options, but we'll look at these three. In most cases, setting (a) is recommended due to its -ease of use. +There are many ways to configure your network for transparent proxying. We'll +look at three common scenarios: + +1. Configuring the client to use a custom gateway/router/"next hop" +2. Implementing custom routing on the router + +In most cases, the first option is recommended due to its ease of use. <h3>(a) Custom Gateway</h3> -<p>Looking at your local home network, it's clear what happens if you enter "example.com" into your address bar: After you -press enter, your OS sends a packet to your router, which then sends this to your ISP, which then sends it to some -Tier-1 carrier, which then sends it... I think you get the idea. The important part for us is the first step here: -Your machine is configured to use your router as the next hop. Your router certainly doesn't host example.com, but your -machine knows that your router will forward it upstream. On the technical level, your router probably provides a DHCP -server, which instructs all clients to use his address as the <em>Default Gateway</em> for connections that leave the -current subnet (your local network).</p> -<p> -How does this help us? Here comes our trick: By configuring the client to use our machine as its Gateway, all traffic -will be sent to our machine, which then forwards it to the router. This provides us with the scenario we'd like to have, -namely packets on our doorstep that are addressed for someone else: -</p> +One simple way to get traffic to the mitmproxy machine with the destination IP +intact, is to simply configure the client with the mitmproxy box as the +default gateway. + <a href="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"> <img src="@!urlTo('schematics/proxy-modes-transparent-2.png')!@"></a> -Given this concept, we can set up mitmproxy: -<ol> - <li>Configure your proxy machine for transparent mode.<br>You can find instructions - in the <em>Transparent Proxying</em> section of the mitmproxy docs.</li> - <li>Configure your client to use your proxy machine's IP as the default gateway. This setting is usually called - <em>Standard Gateway, Router</em> or something along these lines - (<a href="@!urlTo('screenshots/ios-gateway.png')!@">iOS screenshot</a>).</li> - <li>Quick Check: You can already visit an unencrypted HTTP site over the proxy.</li> - <li>Open the magic domain <strong>mitm.it</strong> and install the certificate for your device.</li> -</ol> +In this scenario, we would: + +- Configure the proxy machine for transparent mode. You can find instructions +in the <em>Transparent Proxying</em> section of the mitmproxy docs. + +- Configure the client to use the proxy machine's IP as the default gateway. +<a href="@!urlTo('screenshots/ios-gateway.png')!@">Here</a> is what this would +look like on IOS. + +- Quick Check: At this point, you should already be able to visit an +unencrypted HTTP site over the proxy. + +- Open the magic domain <strong>mitm.it</strong> and install the certificate +for your device. + +Setting the custom gateway on clients can be automated by serving the settings +out to clients over DHCP. This lets set up an interception network where all +clients are proxied automatically, which can save time and effort. + <div class="well"> <strong style="text-align: center; display: block">Troubleshooting Transparent Mode</strong> - <p>Wrong transparent mode configurations are a frequent source of + + <p>Incorrect transparent mode configurations are a frequent source of error. If it doesn't work for you, try the following things:</p> + <ul> - <li>Open mitmproxy's event log (press `e`) - can you spot clientconnect messages? - If not, the packets are not arriving at the proxy. A common source is the occurence of ICMP redirects, - which means that your machine is telling the client that there's a faster way to the internet by contacting - your router directly (see the <em>Transparent Proxying</em> section on how to disable them). If in doubt, - <a href="https://wireshark.org/">Wireshark</a> may help you to see whether something arrives at your machine - or not. + <li> + Open mitmproxy's event log (press `e`) - do you see clientconnect + messages? If not, the packets are not arriving at the proxy. One common + cause is the occurrence of ICMP redirects, which means that your + machine is telling the client that there's a faster way to the + internet by contacting your router directly (see the + <em>Transparent Proxying</em> section on how to disable them). If in + doubt, <a href="https://wireshark.org/">Wireshark</a> may help you + to see whether something arrives at your machine or not. </li> <li> - Have you explicitly configured an HTTP proxy on your device? You do not need mitmproxy's transparent mode - then, just start mitmproxy normally. Explicitly setting a proxy and transparent mode contradict each other, - settle for one. Do not explicitly redirect traffic to mitmproxy anywhere except for the Gateway setting. + Make sure you have not explicitly configured an HTTP proxy on the + client. This is not needed in transparent mode. </li> <li> Re-check the instructions in the <em>Transparent Proxying</em> section. Anything you missed? </li> </ul> + If you encounter any other pitfalls that should be listed here, please let us know! + </div> <h3>(b) Custom Routing</h3> -Custom routing is a fairly advanced setup which we'll only document briefly here. -First and foremost, it usually requires root on your router. The basic idea is to teach your router a custom routing -table that says "for requests from ip X, the proxy machine is the next gateway". +In some cases, you may need more fine-grained control of which traffic reaches +the mitmproxy instance, and which doesn't. You may, for instance, choose only +to divert traffic to some hosts into the transparent proxy. There are a huge +number of ways to accomplish this, and much will depend on the router or +packet filter you're using. In most cases, the configuration will look like +this: <a href="@!urlTo('schematics/proxy-modes-transparent-3.png')!@"> - <img src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@"></a> - -For this setup, we expect you to have a basic understanding of networking in general. In short, you should get started -with <a href="@!urlTo('custom-routing.txt')!@">these routing commands</a>. The Troubleshooting part directly above this -section might be helpful for you as well. - -<h3>(c) Separate Network</h3> - -Setting up a separate network using a cheap router might be a viable option, too. Such a configuration mostly resembles -the idealistic graphic from the beginning (Variant 1). Take a look at the -<a href="@!urlTo('tutorials/transparent-dhcp.html')!@">Transparently proxify virtual machines</a> tutorial to see how -such a network could be implemented. The troubleshooting section for custom gateways may be helpful for you, too. + <img src="@!urlTo('schematics/proxy-modes-transparent-3.png')!@"> +</a> <div class="page-header"> <h1>Reverse Proxy</h1> </div> -Mitmproxy is usually used with a client that uses the proxy to access the Internet. Using reverse proxy mode, you can -use mitmproxy to represent a server: +Mitmproxy is usually used with a client that uses the proxy to access the +Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal +HTTP server: <a href="@!urlTo('schematics/proxy-modes-reverse.png')!@"> - <img src="@!urlTo('schematics/proxy-modes-reverse.png')!@"></a> + <img src="@!urlTo('schematics/proxy-modes-reverse.png')!@"> +</a> There are various use-cases: -<ul> -<li> - Say you have an internal API running at http://example.local/. You could now setup mitmproxy in - reverse proxy mode at http://debug.example.local/ and dynamically point clients to this new API endpoint, - which provides clients with the same data and you with debug information. Similarly, you could move your real server - to a different ip/port and setup mitmproxy at the original place to debug all sessions. -</li> -<li> - Say you're a web developer working on example.com (with a development version running on localhost:8000). - You can modify your hosts file so that example.com points to 127.0.0.1 and then run mitmproxy in reverse proxy - mode on port 80. You can test your app on the example.com domain and get all requests recorded in mitmproxy. -</li> -<li> - Say you have some toy project that should get SSL support. Simply setup mitmproxy with SSL termination and you're - done (<code>mitmdump -p 443 -R https2http://localhost:80/</code>). There are better tools for this specific task (we don't - have C performance obviously), but it's definitely a nice and very quick way to setup an SSL-speaking server. -</li> -<li> - Want to add a non-SSL-capable compression proxy in front of your server? You could even spawn a mitmproxy instance - that terminates SSL (https2http://...), point it to the compression proxy and let the compression proxy point - to a SSL-initiating mitmproxy (http2https://...), which then points to the real server. As you see, it's a fairly - flexible thing. -</li> -</ul> - -<p> -Please note that cloning Google by using <code>mitmproxy -R http://google.com/</code> does <em>not</em> really work -(as in <a href="@!urlTo('screenshots/ios-reverse.png')!@">this screenshot</a>). -This may work for the first request, but the HTML remains unchanged: As soon as the user clicks on an non-relative URL -(or downloads a non-relative image resource), they speak with Google directly again. -</p> -<p> - On another note, mitmproxy either supports an HTTP or an HTTPS upstream server, not both at the same time. You can - simply work around this by spawning a second mitmproxy instance. Each instance listens to one port and talks to one - port. -</p> + +- Say you have an internal API running at http://example.local/. You could now +set up mitmproxy in reverse proxy mode at http://debug.example.local/ and +dynamically point clients to this new API endpoint, which provides clients +with the same data and you with debug information. Similarly, you could move +your real server to a different IP/port and set up mitmproxy at the original +place to debug all sessions. + +- Say you're a web developer working on example.com (with a development +version running on localhost:8000). You can modify your hosts file so that +example.com points to 127.0.0.1 and then run mitmproxy in reverse proxy mode +on port 80. You can test your app on the example.com domain and get all +requests recorded in mitmproxy. + +- Say you have some toy project that should get SSL support. Simply set up +mitmproxy with SSL termination and you're done (<code>mitmdump -p 443 -R +https2http://localhost:80/</code>). There are better tools for this specific +task, but mitmproxy is very quick and simple way to set up an SSL-speaking +server. + +- Want to add a non-SSL-capable compression proxy in front of your server? You +could even spawn a mitmproxy instance that terminates SSL (https2http://...), +point it to the compression proxy and let the compression proxy point to a +SSL-initiating mitmproxy (http2https://...), which then points to the real +server. As you see, it's a fairly flexible thing. + +Note that mitmproxy supports either an HTTP or an HTTPS upstream server, not +both at the same time. You can work around this by spawning a second mitmproxy +instance. + +<div class="well"> + <strong style="text-align: center; display: block">Caveat: Interactive Use</strong> + + +One caveat is that reverse proxy mode is often not sufficient for interactive +browsing. Consider trying to clone Google by using: + +<code>mitmproxy -R http://google.com/</code> + +This works for the initial request, but the HTML served to the client remains +unchanged. As soon as the user clicks on an non-relative URL (or downloads a +non-relative image resource), traffic no longer passes through mitmproxy, and +the client connects to Google directly again. + +</div> + + <div class="page-header"> <h1>Upstream Proxy</h1> </div> -<p> -If you want to add mitmproxy in front of a different proxy appliance, you can use mitmproxy's upstream mode. -In upstream mode, all requests are unconditionally transferred to an upstream proxy or your choice. -</p> +If you want to chain proxies by adding mitmproxy in front of a different proxy +appliance, you can use mitmproxy's upstream mode. In upstream mode, all +requests are unconditionally transferred to an upstream proxy of your choice. <a href="@!urlTo('schematics/proxy-modes-upstream.png')!@"> <img src="@!urlTo('schematics/proxy-modes-upstream.png')!@"></a> -<p> -mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy mode. You could in theory chain multiple -mitmproxy instances in a row, but that doesn't make any sense in practice (i.e. outside of our tests). -</p>
\ No newline at end of file +mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy +mode. You could in theory chain multiple mitmproxy instances in a row, but +that doesn't make any sense in practice (i.e. outside of our tests). diff --git a/doc-src/schematics/proxy-modes-transparent-1.png b/doc-src/schematics/proxy-modes-transparent-1.png Binary files differindex c2027432..002e0e76 100644 --- a/doc-src/schematics/proxy-modes-transparent-1.png +++ b/doc-src/schematics/proxy-modes-transparent-1.png diff --git a/doc-src/schematics/proxy-modes-transparent-2.png b/doc-src/schematics/proxy-modes-transparent-2.png Binary files differindex 1129e343..41997b05 100644 --- a/doc-src/schematics/proxy-modes-transparent-2.png +++ b/doc-src/schematics/proxy-modes-transparent-2.png diff --git a/doc-src/schematics/proxy-modes-transparent-wrong.png b/doc-src/schematics/proxy-modes-transparent-wrong.png Binary files differindex 6bac491f..ca501e93 100644 --- a/doc-src/schematics/proxy-modes-transparent-wrong.png +++ b/doc-src/schematics/proxy-modes-transparent-wrong.png diff --git a/doc-src/schematics/proxy-modes.vsdx b/doc-src/schematics/proxy-modes.vsdx Binary files differindex 74d425fc..c78cf8d0 100644 --- a/doc-src/schematics/proxy-modes.vsdx +++ b/doc-src/schematics/proxy-modes.vsdx diff --git a/doc-src/ssl.html b/doc-src/ssl.html index 91225d79..3fa0e070 100644 --- a/doc-src/ssl.html +++ b/doc-src/ssl.html @@ -41,10 +41,26 @@ The files created by mitmproxy in the .mitmproxy directory are as follows: Using a custom certificate -------------------------- -You can use your own certificate by passing the __--cert__ option to mitmproxy. +You can use your own certificate by passing the <kbd>--cert</kbd> option to mitmproxy. mitmproxy then uses the provided +certificate for interception of the specified domains instead of generating a cert signed by its own CA. -The certificate file is expected to be in the PEM format. You can generate -a certificate in this format using these instructions: +The certificate file is expected to be in the PEM format. +You can include intermediary certificates right below your leaf certificate, so that you PEM file roughly looks like +this: + +<pre> +-----BEGIN PRIVATE KEY----- +<private key> +-----END PRIVATE KEY----- +-----BEGIN CERTIFICATE----- +<cert> +-----END CERTIFICATE----- +-----BEGIN CERTIFICATE----- +<intermediary cert (optional)> +-----END CERTIFICATE----- +</pre> + +For example, you can generate a certificate in this format using these instructions: <pre class="terminal"> > openssl genrsa -out cert.key 8192 @@ -55,6 +71,15 @@ a certificate in this format using these instructions: </pre> +Using a custom certificate authority +------------------------------------ + +By default, mitmproxy will (generate and) use <samp>~/.mitmproxy/mitmproxy-ca.pem</samp> as the default certificate +authority to generate certificates for all domains for which no custom certificate is provided (see above). +You can use your own certificate authority by passing the <kbd>--confdir</kbd> option to mitmproxy. +mitmproxy will then look for <samp>mitmproxy-ca.pem</samp> in the specified directory. If no such file exists, +it will be generated automatically. + Installing the mitmproxy CA --------------------------- diff --git a/examples/change_upstream_proxy.py b/examples/change_upstream_proxy.py index e063ca4f..74a43bd0 100644 --- a/examples/change_upstream_proxy.py +++ b/examples/change_upstream_proxy.py @@ -1,7 +1,7 @@ -# This scripts demonstrates how mitmproxy can switch to a different upstream proxy +# This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy # in upstream proxy mode. # -# Usage: mitmdump -s "change_upstream_proxy.py host" +# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s "change_upstream_proxy.py host" from libmproxy.protocol.http import send_connect_request alternative_upstream_proxy = ("localhost", 8082) diff --git a/examples/flowbasic b/examples/flowbasic index 21d31efa..c71debc9 100755 --- a/examples/flowbasic +++ b/examples/flowbasic @@ -36,7 +36,7 @@ class MyMaster(flow.FlowMaster): config = proxy.ProxyConfig( port=8080, - ca_file=os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem") + cadir="~/.mitmproxy/" # use ~/.mitmproxy/mitmproxy-ca.pem as default CA file. ) state = flow.State() server = ProxyServer(config) diff --git a/examples/har_extractor.py b/examples/har_extractor.py new file mode 100644 index 00000000..531f32aa --- /dev/null +++ b/examples/har_extractor.py @@ -0,0 +1,212 @@ +""" + This inline script utilizes harparser.HAR from https://github.com/JustusW/harparser + to generate a HAR log object. +""" +try: + from harparser import HAR + from pytz import UTC +except ImportError as e: + import sys + print >> sys.stderr, "\r\nMissing dependencies: please run `pip install mitmproxy[examples]`.\r\n" + raise + +from datetime import datetime, timedelta, tzinfo + + +class _HARLog(HAR.log): + # The attributes need to be registered here for them to actually be available later via self. This is + # due to HAREncodable linking __getattr__ to __getitem__. Anything that is set only in __init__ will + # just be added as key/value pair to self.__classes__. + __page_list__ = [] + __page_count__ = 0 + __page_ref__ = {} + + def __init__(self, page_list): + self.__page_list__ = page_list + self.__page_count__ = 0 + self.__page_ref__ = {} + + HAR.log.__init__(self, {"version": "1.2", + "creator": {"name": "MITMPROXY HARExtractor", + "version": "0.1", + "comment": ""}, + "pages": [], + "entries": []}) + + def reset(self): + self.__init__(self.__page_list__) + + def add(self, obj): + if isinstance(obj, HAR.pages): + self['pages'].append(obj) + if isinstance(obj, HAR.entries): + self['entries'].append(obj) + + def create_page_id(self): + self.__page_count__ += 1 + return "autopage_%s" % str(self.__page_count__) + + def set_page_ref(self, page, ref): + self.__page_ref__[page] = ref + + def get_page_ref(self, page): + return self.__page_ref__.get(page, None) + + def get_page_list(self): + return self.__page_list__ + + +def start(context, argv): + """ + On start we create a HARLog instance. You will have to adapt this to suit your actual needs + of HAR generation. As it will probably be necessary to cluster logs by IPs or reset them + from time to time. + """ + context.dump_file = None + if len(argv) > 1: + context.dump_file = argv[1] + else: + raise ValueError('Usage: -s "har_extractor.py filename" ' + '(- will output to stdout, filenames ending with .zhar will result in compressed har)') + context.HARLog = _HARLog(['https://github.com']) + context.seen_server = set() + + +def response(context, flow): + """ + Called when a server response has been received. At the time of this message both + a request and a response are present and completely done. + """ + # Values are converted from float seconds to int milliseconds later. + ssl_time = -.001 + connect_time = -.001 + if flow.server_conn not in context.seen_server: + # Calculate the connect_time for this server_conn. Afterwards add it to seen list, in + # order to avoid the connect_time being present in entries that use an existing connection. + connect_time = flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start + context.seen_server.add(flow.server_conn) + + if flow.server_conn.timestamp_ssl_setup is not None: + # Get the ssl_time for this server_conn as the difference between the start of the successful + # tcp setup and the successful ssl setup. If no ssl setup has been made it is left as -1 since + # it doesn't apply to this connection. + ssl_time = flow.server_conn.timestamp_ssl_setup - flow.server_conn.timestamp_tcp_setup + + # Calculate the raw timings from the different timestamps present in the request and response object. + # For lack of a way to measure it dns timings can not be calculated. The same goes for HAR blocked: + # MITMProxy will open a server connection as soon as it receives the host and port from the client + # connection. So the time spent waiting is actually spent waiting between request.timestamp_end and + # response.timestamp_start thus it correlates to HAR wait instead. + timings_raw = {'send': flow.request.timestamp_end - flow.request.timestamp_start, + 'wait': flow.response.timestamp_start - flow.request.timestamp_end, + 'receive': flow.response.timestamp_end - flow.response.timestamp_start, + 'connect': connect_time, + 'ssl': ssl_time} + + # HAR timings are integers in ms, so we have to re-encode the raw timings to that format. + timings = dict([(key, int(1000 * value)) for key, value in timings_raw.iteritems()]) + + # The full_time is the sum of all timings. Timings set to -1 will be ignored as per spec. + full_time = 0 + for item in timings.values(): + if item > -1: + full_time += item + + started_date_time = datetime.fromtimestamp(flow.request.timestamp_start, tz=utc).isoformat() + + request_query_string = [{"name": k, "value": v} for k, v in flow.request.get_query()] + request_http_version = ".".join([str(v) for v in flow.request.httpversion]) + # Cookies are shaped as tuples by MITMProxy. + request_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.request.get_cookies() or {}).iteritems()] + request_headers = [{"name": k, "value": v} for k, v in flow.request.headers] + request_headers_size = len(str(flow.request.headers)) + request_body_size = len(flow.request.content) + + response_http_version = ".".join([str(v) for v in flow.response.httpversion]) + # Cookies are shaped as tuples by MITMProxy. + response_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.response.get_cookies() or {}).iteritems()] + response_headers = [{"name": k, "value": v} for k, v in flow.response.headers] + response_headers_size = len(str(flow.response.headers)) + response_body_size = len(flow.response.content) + response_body_decoded_size = len(flow.response.get_decoded_content()) + response_body_compression = response_body_decoded_size - response_body_size + response_mime_type = flow.response.headers.get_first('Content-Type', '') + response_redirect_url = flow.response.headers.get_first('Location', '') + + entry = HAR.entries({"startedDateTime": started_date_time, + "time": full_time, + "request": {"method": flow.request.method, + "url": flow.request.url, + "httpVersion": request_http_version, + "cookies": request_cookies, + "headers": request_headers, + "queryString": request_query_string, + "headersSize": request_headers_size, + "bodySize": request_body_size, }, + "response": {"status": flow.response.code, + "statusText": flow.response.msg, + "httpVersion": response_http_version, + "cookies": response_cookies, + "headers": response_headers, + "content": {"size": response_body_size, + "compression": response_body_compression, + "mimeType": response_mime_type}, + "redirectURL": response_redirect_url, + "headersSize": response_headers_size, + "bodySize": response_body_size, }, + "cache": {}, + "timings": timings, }) + + # If the current url is in the page list of context.HARLog or does not have a referrer we add it as a new + # pages object. + if flow.request.url in context.HARLog.get_page_list() or flow.request.headers.get('Referer', None) is None: + page_id = context.HARLog.create_page_id() + context.HARLog.add(HAR.pages({"startedDateTime": entry['startedDateTime'], + "id": page_id, + "title": flow.request.url, })) + context.HARLog.set_page_ref(flow.request.url, page_id) + entry['pageref'] = page_id + + # Lookup the referer in the page_ref of context.HARLog to point this entries pageref attribute to the right + # pages object, then set it as a new reference to build a reference tree. + elif context.HARLog.get_page_ref(flow.request.headers.get('Referer', (None, ))[0]) is not None: + entry['pageref'] = context.HARLog.get_page_ref(flow.request.headers['Referer'][0]) + context.HARLog.set_page_ref(flow.request.headers['Referer'][0], entry['pageref']) + + context.HARLog.add(entry) + + +def done(context): + """ + Called once on script shutdown, after any other events. + """ + from pprint import pprint + import json + + json_dump = context.HARLog.json() + compressed_json_dump = context.HARLog.compress() + + print "=" * 100 + if context.dump_file == '-': + pprint(json.loads(json_dump)) + elif context.dump_file.endswith('.zhar'): + file(context.dump_file, "w").write(compressed_json_dump) + else: + file(context.dump_file, "w").write(json_dump) + print "=" * 100 + print "HAR log finished with %s bytes (%s bytes compressed)" % (len(json_dump), len(compressed_json_dump)) + print "Compression rate is %s%%" % str(100. * len(compressed_json_dump) / len(json_dump)) + print "=" * 100 + + +def print_attributes(obj, filter_string=None, hide_privates=False): + """ + Useful helper method to quickly get all attributes of an object and its values. + """ + for attr in dir(obj): + if hide_privates and "__" in attr: + continue + if filter_string is not None and filter_string not in attr: + continue + value = getattr(obj, attr) + print "%s.%s" % ('obj', attr), value, type(value)
\ No newline at end of file diff --git a/examples/ignore_websocket.py b/examples/ignore_websocket.py new file mode 100644 index 00000000..48093951 --- /dev/null +++ b/examples/ignore_websocket.py @@ -0,0 +1,34 @@ +# This script makes mitmproxy switch to passthrough mode for all HTTP +# responses with "Connection: Upgrade" header. This is useful to make +# WebSockets work in untrusted environments. +# +# Note: Chrome (and possibly other browsers), when explicitly configured +# to use a proxy (i.e. mitmproxy's regular mode), send a CONNECT request +# to the proxy before they initiate the websocket connection. +# To make WebSockets work in these cases, supply +# `--ignore :80$` as an additional parameter. +# (see http://mitmproxy.org/doc/features/passthrough.html) + +from libmproxy.protocol.http import HTTPRequest +from libmproxy.protocol.tcp import TCPHandler +from libmproxy.protocol import KILL +from libmproxy.script import concurrent + + +def start(context, argv): + HTTPRequest._headers_to_strip_off.remove("Connection") + HTTPRequest._headers_to_strip_off.remove("Upgrade") + + +def done(context): + HTTPRequest._headers_to_strip_off.append("Connection") + HTTPRequest._headers_to_strip_off.append("Upgrade") + +@concurrent +def response(context, flow): + if flow.response.headers.get_first("Connection", None) == "Upgrade": + # We need to send the response manually now... + flow.client_conn.send(flow.response.assemble()) + # ...and then delegate to tcp passthrough. + TCPHandler(flow.live.c, log=False).handle_messages() + flow.reply(KILL)
\ No newline at end of file diff --git a/examples/stickycookies b/examples/stickycookies index 132e4dc7..67b31da1 100755 --- a/examples/stickycookies +++ b/examples/stickycookies @@ -36,10 +36,7 @@ class StickyMaster(controller.Master): flow.reply() -config = proxy.ProxyConfig( - port=8080, - ca_file=os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem") -) +config = proxy.ProxyConfig(port=8080) server = ProxyServer(config) m = StickyMaster(server) m.run() diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index f6cd1ab8..b892f1fd 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -1,9 +1,9 @@ from __future__ import absolute_import +import os import re -import argparse -from argparse import ArgumentTypeError +import configargparse from netlib import http -from . import filt, utils +from . import filt, utils, version from .proxy import config APP_HOST = "mitm.it" @@ -23,7 +23,9 @@ def _parse_hook(s): elif len(parts) == 3: patt, a, b = parts else: - raise ParseException("Malformed hook specifier - too few clauses: %s" % s) + raise ParseException( + "Malformed hook specifier - too few clauses: %s" % s + ) if not a: raise ParseException("Empty clause: %s" % str(patt)) @@ -102,7 +104,9 @@ def parse_server_spec(url): p = http.parse_url(normalized_url) if not p or not p[1]: - raise ArgumentTypeError("Invalid server specification: %s" % url) + raise configargparse.ArgumentTypeError( + "Invalid server specification: %s" % url + ) if url.lower().startswith("https2http"): ssl = [True, False] @@ -131,17 +135,19 @@ def get_common_options(options): try: p = parse_replace_hook(i) except ParseException, e: - raise ArgumentTypeError(e.message) + raise configargparse.ArgumentTypeError(e.message) reps.append(p) for i in options.replace_file: try: patt, rex, path = parse_replace_hook(i) except ParseException, e: - raise ArgumentTypeError(e.message) + raise configargparse.ArgumentTypeError(e.message) try: v = open(path, "rb").read() except IOError, e: - raise ArgumentTypeError("Could not read replace file: %s" % path) + raise configargparse.ArgumentTypeError( + "Could not read replace file: %s" % path + ) reps.append((patt, rex, v)) setheaders = [] @@ -149,7 +155,7 @@ def get_common_options(options): try: p = parse_setheader(i) except ParseException, e: - raise ArgumentTypeError(e.message) + raise configargparse.ArgumentTypeError(e.message) setheaders.append(p) return dict( @@ -183,14 +189,23 @@ def get_common_options(options): def common_options(parser): parser.add_argument( + '--version', + action= 'version', + version= "%(prog)s" + " " + version.VERSION + ) + parser.add_argument( "--anticache", action="store_true", dest="anticache", default=False, - help="Strip out request headers that might cause the server to return 304-not-modified." + + help=""" + Strip out request headers that might cause the server to return + 304-not-modified. + """ ) parser.add_argument( - "--confdir", - action="store", type=str, dest="confdir", default='~/.mitmproxy', - help="Configuration directory. (~/.mitmproxy)" + "--cadir", + action="store", type=str, dest="cadir", default=config.CA_DIR, + help="Location of the default mitmproxy CA files. (%s)"%config.CA_DIR ) parser.add_argument( "--host", @@ -198,112 +213,150 @@ def common_options(parser): help="Use the Host header to construct URLs for display." ) parser.add_argument( - "-q", + "-q", "--quiet", action="store_true", dest="quiet", help="Quiet." ) parser.add_argument( - "-r", + "-r", "--read-flows", action="store", dest="rfile", default=None, help="Read flows from file." ) parser.add_argument( - "-s", + "-s", "--script", action="append", type=str, dest="scripts", default=[], metavar='"script.py --bar"', - help="Run a script. Surround with quotes to pass script arguments. Can be passed multiple times." + help=""" + Run a script. Surround with quotes to pass script arguments. Can be + passed multiple times. + """ ) parser.add_argument( - "-t", - action="store", dest="stickycookie_filt", default=None, metavar="FILTER", + "-t", "--stickycookie", + action="store", + dest="stickycookie_filt", + default=None, + metavar="FILTER", help="Set sticky cookie filter. Matched against requests." ) parser.add_argument( - "-u", + "-u", "--stickyauth", action="store", dest="stickyauth_filt", default=None, metavar="FILTER", help="Set sticky auth filter. Matched against requests." ) parser.add_argument( - "-v", + "-v", "--verbose", action="store_const", dest="verbose", default=1, const=2, help="Increase event log verbosity." ) parser.add_argument( - "-w", + "-w", "--wfile", action="store", dest="wfile", default=None, help="Write flows to file." ) parser.add_argument( - "-z", + "-z", "--anticomp", action="store_true", dest="anticomp", default=False, help="Try to convince servers to send us un-compressed data." ) parser.add_argument( - "-Z", + "-Z", "--body-size-limit", action="store", dest="body_size_limit", default=None, metavar="SIZE", - help="Byte size limit of HTTP request and response bodies." \ + help="Byte size limit of HTTP request and response bodies." " Understands k/m/g suffixes, i.e. 3m for 3 megabytes." ) parser.add_argument( "--stream", action="store", dest="stream_large_bodies", default=None, metavar="SIZE", - help="Stream data to the client if response body exceeds the given threshold. " - "If streamed, the body will not be stored in any way. Understands k/m/g suffixes, i.e. 3m for 3 megabytes." + help=""" + Stream data to the client if response body exceeds the given + threshold. If streamed, the body will not be stored in any way. + Understands k/m/g suffixes, i.e. 3m for 3 megabytes. + """ ) group = parser.add_argument_group("Proxy Options") - # We could make a mutually exclusive group out of -R, -U, -T, but we don't do that because - # - --upstream-server should be in that group as well, but it's already in a different group. - # - our own error messages are more helpful + # We could make a mutually exclusive group out of -R, -U, -T, but we don't + # do that because - --upstream-server should be in that group as well, but + # it's already in a different group. - our own error messages are more + # helpful group.add_argument( - "-b", + "-b", "--bind-address", action="store", type=str, dest="addr", default='', help="Address to bind proxy to (defaults to all interfaces)" ) group.add_argument( "-I", "--ignore", - action="append", type=str, dest="ignore", default=[], + action="append", type=str, dest="ignore_hosts", default=[], metavar="HOST", - help="Ignore host and forward all traffic without processing it. " - "In transparent mode, it is recommended to use an IP address (range), not the hostname. " - "In regular mode, only SSL traffic is ignored and the hostname should be used. " - "The supplied value is interpreted as a regular expression and matched on the ip or the hostname. " - "Can be passed multiple times. " + help=""" + Ignore host and forward all traffic without processing it. In + transparent mode, it is recommended to use an IP address (range), + not the hostname. In regular mode, only SSL traffic is ignored and + the hostname should be used. The supplied value is interpreted as a + regular expression and matched on the ip or the hostname. Can be + passed multiple times. + """ ) group.add_argument( - "-n", + "--tcp", + action="append", type=str, dest="tcp_hosts", default=[], + metavar="HOST", + help=""" + Generic TCP SSL proxy mode for all hosts that match the pattern. + Similar to --ignore, but SSL connections are intercepted. The + communication contents are printed to the event log in verbose mode. + """ + ) + group.add_argument( + "-n", "--no-server", action="store_true", dest="no_server", help="Don't start a proxy server." ) group.add_argument( - "-p", + "-p", "--port", action="store", type=int, dest="port", default=8080, help="Proxy service port." ) group.add_argument( - "-R", - action="store", type=parse_server_spec, dest="reverse_proxy", default=None, - help="Forward all requests to upstream HTTP server: http[s][2http[s]]://host[:port]" + "-R", "--reverse", + action="store", + type=parse_server_spec, + dest="reverse_proxy", + default=None, + help=""" + Forward all requests to upstream HTTP server: + http[s][2http[s]]://host[:port] + """ ) group.add_argument( - "-T", + "--socks", + action="store_true", dest="socks_proxy", default=False, + help="Set SOCKS5 proxy mode." + ) + group.add_argument( + "-T", "--transparent", action="store_true", dest="transparent_proxy", default=False, help="Set transparent proxy mode." ) group.add_argument( - "-U", - action="store", type=parse_server_spec, dest="upstream_proxy", default=None, + "-U", "--upstream", + action="store", + type=parse_server_spec, + dest="upstream_proxy", + default=None, help="Forward all requests to upstream proxy server: http://host[:port]" ) group = parser.add_argument_group( "Advanced Proxy Options", """ - The following options allow a custom adjustment of the proxy behavior. - Normally, you don't want to use these options directly and use the provided wrappers instead (-R, -U, -T). - """.strip() + The following options allow a custom adjustment of the proxy + behavior. Normally, you don't want to use these options directly and + use the provided wrappers instead (-R, -U, -T). + """ ) group.add_argument( "--http-form-in", dest="http_form_in", default=None, @@ -318,38 +371,44 @@ def common_options(parser): group = parser.add_argument_group("Onboarding App") group.add_argument( - "-a", + "-a", "--noapp", action="store_false", dest="app", default=True, help="Disable the mitmproxy onboarding app." ) group.add_argument( "--app-host", action="store", dest="app_host", default=APP_HOST, metavar="host", - help="Domain to serve the onboarding app from. For transparent mode, use an IP when\ - a DNS entry for the app domain is not present. Default: %s" % APP_HOST - + help=""" + Domain to serve the onboarding app from. For transparent mode, use + an IP when a DNS entry for the app domain is not present. Default: + %s + """ % APP_HOST ) group.add_argument( "--app-port", - action="store", dest="app_port", default=APP_PORT, type=int, metavar="80", + action="store", + dest="app_port", + default=APP_PORT, + type=int, + metavar="80", help="Port to serve the onboarding app from." ) group = parser.add_argument_group("Client Replay") group.add_argument( - "-c", + "-c", "--client-replay", action="store", dest="client_replay", default=None, metavar="PATH", help="Replay client requests from a saved file." ) group = parser.add_argument_group("Server Replay") group.add_argument( - "-S", + "-S", "--server-replay", action="store", dest="server_replay", default=None, metavar="PATH", help="Replay server responses from a saved file." ) group.add_argument( - "-k", + "-k", "--kill", action="store_true", dest="kill", default=False, help="Kill extra requests during replay." ) @@ -362,8 +421,10 @@ def common_options(parser): group.add_argument( "--norefresh", action="store_true", dest="norefresh", default=False, - help="Disable response refresh, " - "which updates times in cookies and headers for replayed responses." + help=""" + Disable response refresh, which updates times in cookies and headers + for replayed responses. + """ ) group.add_argument( "--no-pop", @@ -374,14 +435,18 @@ def common_options(parser): group.add_argument( "--replay-ignore-content", action="store_true", dest="replay_ignore_content", default=False, - help="Ignore request's content while searching for a saved flow to replay" + help=""" + Ignore request's content while searching for a saved flow to replay + """ ) group.add_argument( "--replay-ignore-param", action="append", dest="replay_ignore_params", type=str, - help="Request's parameters to be ignored while searching for a saved flow to replay" - "Can be passed multiple times." - ) + help=""" + Request's parameters to be ignored while searching for a saved flow + to replay. Can be passed multiple times. + """ + ) group = parser.add_argument_group( "Replacements", @@ -399,9 +464,12 @@ def common_options(parser): ) group.add_argument( "--replace-from-file", - action="append", type=str, dest="replace_file", default=[], - metavar="PATH", - help="Replacement pattern, where the replacement clause is a path to a file." + action = "append", type=str, dest="replace_file", default=[], + metavar = "PATH", + help = """ + Replacement pattern, where the replacement clause is a path to a + file. + """ ) group = parser.add_argument_group( @@ -437,7 +505,10 @@ def common_options(parser): "--singleuser", action="store", dest="auth_singleuser", type=str, metavar="USER", - help="Allows access to a a single user, specified in the form username:password." + help=""" + Allows access to a a single user, specified in the form + username:password. + """ ) user_specification_group.add_argument( "--htpasswd", @@ -447,3 +518,116 @@ def common_options(parser): ) config.ssl_option_group(parser) + + +def mitmproxy(): + # Don't import libmproxy.console for mitmdump, urwid is not available on all + # platforms. + from .console import palettes + + parser = configargparse.ArgumentParser( + usage="%(prog)s [options]", + args_for_setting_config_path = ["--conf"], + default_config_files = [ + os.path.join(config.CA_DIR, "common.conf"), + os.path.join(config.CA_DIR, "mitmproxy.conf") + ], + add_config_file_help = True, + add_env_var_help = True + ) + common_options(parser) + parser.add_argument( + "--palette", type=str, default="dark", + action="store", dest="palette", + help="Select color palette: " + ", ".join(palettes.palettes.keys()) + ) + parser.add_argument( + "-e", "--eventlog", + action="store_true", dest="eventlog", + help="Show event log." + ) + group = parser.add_argument_group( + "Filters", + "See help in mitmproxy for filter expression syntax." + ) + group.add_argument( + "-i", "--intercept", action="store", + type=str, dest="intercept", default=None, + help="Intercept filter expression." + ) + return parser + + +def mitmdump(): + parser = configargparse.ArgumentParser( + usage="%(prog)s [options] [filter]", + args_for_setting_config_path = ["--conf"], + default_config_files = [ + os.path.join(config.CA_DIR, "common.conf"), + os.path.join(config.CA_DIR, "mitmdump.conf") + ], + add_config_file_help = True, + add_env_var_help = True + ) + + common_options(parser) + parser.add_argument( + "--keepserving", + action= "store_true", dest="keepserving", default=False, + help= """ + Continue serving after client playback or file read. We exit by + default. + """ + ) + parser.add_argument( + "-d", "--detail", + action="count", dest="flow_detail", default=1, + help="Increase flow detail display level. Can be passed multiple times." + ) + parser.add_argument('args', nargs="...") + return parser + + +def mitmweb(): + parser = configargparse.ArgumentParser( + usage="%(prog)s [options]", + args_for_setting_config_path = ["--conf"], + default_config_files = [ + os.path.join(config.CA_DIR, "common.conf"), + os.path.join(config.CA_DIR, "mitmweb.conf") + ], + add_config_file_help = True, + add_env_var_help = True + ) + + group = parser.add_argument_group("Mitmweb") + group.add_argument( + "--wport", + action="store", type=int, dest="wport", default=8081, + metavar="PORT", + help="Mitmweb port." + ) + group.add_argument( + "--wiface", + action="store", dest="wiface", default="127.0.0.1", + metavar="IFACE", + help="Mitmweb interface." + ) + group.add_argument( + "--wdebug", + action="store_true", dest="wdebug", + help="Turn on mitmweb debugging" + ) + + common_options(parser) + group = parser.add_argument_group( + "Filters", + "See help in mitmproxy for filter expression syntax." + ) + group.add_argument( + "-i", "--intercept", action="store", + type=str, dest="intercept", default=None, + help="Intercept filter expression." + ) + return parser + diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 9c4b4827..e6bc9b41 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -129,10 +129,14 @@ class StatusBar(common.WWrap): r.append(":%s in file]"%self.master.server_playback.count()) else: r.append(":%s to go]"%self.master.server_playback.count()) - if self.master.get_ignore(): + if self.master.get_ignore_filter(): r.append("[") r.append(("heading_key", "I")) - r.append("gnore:%d]"%len(self.master.get_ignore())) + r.append("gnore:%d]" % len(self.master.get_ignore_filter())) + if self.master.get_tcp_filter(): + r.append("[") + r.append(("heading_key", "T")) + r.append("CP:%d]" % len(self.master.get_tcp_filter())) if self.master.state.intercept_txt: r.append("[") r.append(("heading_key", "i")) @@ -512,7 +516,8 @@ class ConsoleMaster(flow.FlowMaster): self.start_server_playback( ret, self.killextra, self.rheaders, - False, self.nopop + False, self.nopop, + self.options.replay_ignore_params, self.options.replay_ignore_content ) def spawn_editor(self, data): @@ -798,9 +803,13 @@ class ConsoleMaster(flow.FlowMaster): for command in commands: self.load_script(command) - def edit_ignore(self, ignore): + def edit_ignore_filter(self, ignore): patterns = (x[0] for x in ignore) - self.set_ignore(patterns) + self.set_ignore_filter(patterns) + + def edit_tcp_filter(self, tcp): + patterns = (x[0] for x in tcp) + self.set_tcp_filter(patterns) def loop(self): changed = True @@ -811,7 +820,7 @@ class ConsoleMaster(flow.FlowMaster): self.statusbar.redraw() size = self.drawscreen() changed = self.tick(self.masterq, 0.01) - self.ui.set_input_timeouts(max_wait=0.1) + self.ui.set_input_timeouts(max_wait=0.01) keys = self.ui.get_input() if keys: changed = True @@ -860,10 +869,18 @@ class ConsoleMaster(flow.FlowMaster): ) elif k == "I": self.view_grideditor( - grideditor.IgnoreEditor( + grideditor.HostPatternEditor( + self, + [[x] for x in self.get_ignore_filter()], + self.edit_ignore_filter + ) + ) + elif k == "T": + self.view_grideditor( + grideditor.HostPatternEditor( self, - [[x] for x in self.get_ignore()], - self.edit_ignore + [[x] for x in self.get_tcp_filter()], + self.edit_tcp_filter ) ) elif k == "i": @@ -1033,7 +1050,7 @@ class ConsoleMaster(flow.FlowMaster): self.eventlist[:] = [] def add_event(self, e, level="info"): - needed = dict(error=1, info=1, debug=2).get(level, 1) + needed = dict(error=0, info=1, debug=2).get(level, 1) if self.options.verbosity < needed: return diff --git a/libmproxy/console/flowlist.py b/libmproxy/console/flowlist.py index e0330171..3eb4eb1a 100644 --- a/libmproxy/console/flowlist.py +++ b/libmproxy/console/flowlist.py @@ -120,13 +120,15 @@ class ConnectionItem(common.WWrap): self.master.start_server_playback( [i.copy() for i in self.master.state.view], self.master.killextra, self.master.rheaders, - False, self.master.nopop + False, self.master.nopop, + self.master.options.replay_ignore_params, self.master.options.replay_ignore_content ) elif k == "t": self.master.start_server_playback( [self.flow.copy()], self.master.killextra, self.master.rheaders, - False, self.master.nopop + False, self.master.nopop, + self.master.options.replay_ignore_params, self.master.options.replay_ignore_content ) else: self.master.path_prompt( diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index b2c46147..1ec57a4e 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -574,9 +574,8 @@ class FlowView(common.WWrap): else: if not self.flow.response: self.flow.response = HTTPResponse( - self.flow.request, self.flow.request.httpversion, - 200, "OK", flow.ODictCaseless(), "", None + 200, "OK", flow.ODictCaseless(), "" ) self.flow.response.reply = controller.DummyReply() conn = self.flow.response @@ -749,7 +748,7 @@ class FlowView(common.WWrap): self.master.statusbar.message("") elif key == "m": p = list(contentview.view_prompts) - p.insert(0, ("clear", "c")) + p.insert(0, ("Clear", "C")) self.master.prompt_onekey( "Display mode", p, diff --git a/libmproxy/console/grideditor.py b/libmproxy/console/grideditor.py index d629ec82..438d0ad7 100644 --- a/libmproxy/console/grideditor.py +++ b/libmproxy/console/grideditor.py @@ -123,12 +123,13 @@ class GridWalker(urwid.ListWalker): except ValueError: self.editor.master.statusbar.message("Invalid Python-style string encoding.", 1000) return - errors = self.lst[self.focus][1] emsg = self.editor.is_error(self.focus_col, val) if emsg: self.editor.master.statusbar.message(emsg, 1000) errors.add(self.focus_col) + else: + errors.discard(self.focus_col) row = list(self.lst[self.focus][0]) row[self.focus_col] = val @@ -320,9 +321,11 @@ class GridEditor(common.WWrap): elif key == "d": self.walker.delete_focus() elif key == "r": - self.master.path_prompt("Read file: ", "", self.read_file) + if self.walker.get_current_value() is not None: + self.master.path_prompt("Read file: ", "", self.read_file) elif key == "R": - self.master.path_prompt("Read unescaped file: ", "", self.read_file, True) + if self.walker.get_current_value() is not None: + self.master.path_prompt("Read unescaped file: ", "", self.read_file, True) elif key == "e": o = self.walker.get_current_value() if o is not None: @@ -495,8 +498,8 @@ class ScriptEditor(GridEditor): return str(v) -class IgnoreEditor(GridEditor): - title = "Editing ignore patterns" +class HostPatternEditor(GridEditor): + title = "Editing host patterns" columns = 1 headings = ("Regex (matched on hostname:port / ip:port)",) diff --git a/libmproxy/console/help.py b/libmproxy/console/help.py index bdcf3fd9..27288a36 100644 --- a/libmproxy/console/help.py +++ b/libmproxy/console/help.py @@ -119,6 +119,7 @@ class HelpView(urwid.ListBox): ("s", "add/remove scripts"), ("S", "server replay"), ("t", "set sticky cookie expression"), + ("T", "set tcp proxying pattern"), ("u", "set sticky auth expression"), ] text.extend(common.format_keyvals(keys, key="key", val="text", indent=4)) diff --git a/libmproxy/dump.py b/libmproxy/dump.py index ccb2b5b5..0d9432c9 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -1,10 +1,13 @@ from __future__ import absolute_import -import sys, os +import sys +import os import netlib.utils from . import flow, filt, utils from .protocol import http -class DumpError(Exception): pass + +class DumpError(Exception): + pass class Options(object): @@ -37,6 +40,7 @@ class Options(object): "replay_ignore_content", "replay_ignore_params", ] + def __init__(self, **kwargs): for k, v in kwargs.items(): setattr(self, k, v) @@ -71,7 +75,7 @@ class DumpMaster(flow.FlowMaster): self.anticache = options.anticache self.anticomp = options.anticomp self.showhost = options.showhost - self.replay_ignore_params = options.replay_ignore_params + self.replay_ignore_params = options.replay_ignore_params self.replay_ignore_content = options.replay_ignore_content self.refresh_server_playback = options.refresh_server_playback @@ -88,7 +92,6 @@ class DumpMaster(flow.FlowMaster): if options.stickyauth: self.set_stickyauth(options.stickyauth) - if options.wfile: path = os.path.expanduser(options.wfile) try: @@ -152,7 +155,7 @@ class DumpMaster(flow.FlowMaster): return flows def add_event(self, e, level="info"): - needed = dict(error=1, info=1, debug=2).get(level, 1) + needed = dict(error=0, info=1, debug=2).get(level, 1) if self.o.verbosity >= needed: print >> self.outfile, e self.outfile.flush() @@ -202,7 +205,7 @@ class DumpMaster(flow.FlowMaster): elif self.o.flow_detail >= 3: print >> self.outfile, str_request(f, self.showhost) print >> self.outfile, self.indent(4, f.request.headers) - if utils.isBin(f.request.content): + if f.request.content != http.CONTENT_MISSING and utils.isBin(f.request.content): d = netlib.utils.hexdump(f.request.content) d = "\n".join("%s\t%s %s"%i for i in d) print >> self.outfile, self.indent(4, d) diff --git a/libmproxy/filt.py b/libmproxy/filt.py index 7d2bd737..5d259096 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -343,7 +343,9 @@ bnf = _make() def parse(s): try: - return bnf.parseString(s, parseAll=True)[0] + filt = bnf.parseString(s, parseAll=True)[0] + filt.pattern = s + return filt except pp.ParseException: return None except ValueError: diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 440798bc..a6bf17d8 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -11,7 +11,7 @@ import netlib.http from . import controller, protocol, tnetstring, filt, script, version from .onboarding import app from .protocol import http, handle -from .proxy.config import parse_host_pattern +from .proxy.config import HostMatcher import urlparse ODict = odict.ODict @@ -27,7 +27,12 @@ class AppRegistry: Add a WSGI app to the registry, to be served for requests to the specified domain, on the specified port. """ - self.apps[(domain, port)] = wsgi.WSGIAdaptor(app, domain, port, version.NAMEVERSION) + self.apps[(domain, port)] = wsgi.WSGIAdaptor( + app, + domain, + port, + version.NAMEVERSION + ) def get(self, request): """ @@ -72,7 +77,8 @@ class ReplaceHooks: def get_specs(self): """ - Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) tuples. + Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) + tuples. """ return [i[:3] for i in self.lst] @@ -119,7 +125,8 @@ class SetHeaders: def get_specs(self): """ - Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) tuples. + Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) + tuples. """ return [i[:3] for i in self.lst] @@ -162,6 +169,7 @@ class ClientPlaybackState: def __init__(self, flows, exit): self.flows, self.exit = flows, exit self.current = None + self.testing = False # Disables actual replay for testing. def count(self): return len(self.flows) @@ -179,18 +187,16 @@ class ClientPlaybackState: if flow is self.current: self.current = None - def tick(self, master, testing=False): - """ - testing: Disables actual replay for testing. - """ + def tick(self, master): if self.flows and not self.current: - n = self.flows.pop(0) - n.reply = controller.DummyReply() - self.current = master.handle_request(n) - if not testing and not self.current.response: - master.replay_request(self.current) # pragma: no cover - elif self.current.response: - master.handle_response(self.current) + self.current = self.flows.pop(0).copy() + if not self.testing: + master.replay_request(self.current) + else: + self.current.reply = controller.DummyReply() + master.handle_request(self.current) + if self.current.response: + master.handle_response(self.current) class ServerPlaybackState: @@ -219,9 +225,10 @@ class ServerPlaybackState: queriesArray = urlparse.parse_qsl(query) filtered = [] + ignore_params = self.ignore_params or [] for p in queriesArray: - if p[0] not in self.ignore_params: - filtered.append(p) + if p[0] not in ignore_params: + filtered.append(p) key = [ str(r.host), @@ -339,11 +346,13 @@ class State(object): # These are compiled filt expressions: self._limit = None self.intercept = None - self._limit_txt = None @property def limit_txt(self): - return self._limit_txt + if self._limit: + return self._limit.pattern + else: + return None def flow_count(self): return len(self._flow_list) @@ -362,6 +371,8 @@ class State(object): """ Add a request to the state. Returns the matching flow. """ + if flow in self._flow_list: # catch flow replay + return flow self._flow_list.append(flow) if flow.match(self._limit): self.view.append(flow) @@ -398,10 +409,8 @@ class State(object): if not f: return "Invalid filter expression." self._limit = f - self._limit_txt = txt else: self._limit = None - self._limit_txt = None self.recalculate_view() def set_intercept(self, txt): @@ -465,7 +474,7 @@ class FlowMaster(controller.Master): self.refresh_server_playback = False self.replacehooks = ReplaceHooks() self.setheaders = SetHeaders() - self.replay_ignore_params = False + self.replay_ignore_params = False self.replay_ignore_content = None @@ -515,11 +524,17 @@ class FlowMaster(controller.Master): for script in self.scripts: self.run_single_script_hook(script, name, *args, **kwargs) - def get_ignore(self): - return [i.pattern for i in self.server.config.ignore] + def get_ignore_filter(self): + return self.server.config.check_ignore.patterns + + def set_ignore_filter(self, host_patterns): + self.server.config.check_ignore = HostMatcher(host_patterns) - def set_ignore(self, ignore): - self.server.config.ignore = parse_host_pattern(ignore) + def get_tcp_filter(self): + return self.server.config.check_tcp.patterns + + def set_tcp_filter(self, host_patterns): + self.server.config.check_tcp = HostMatcher(host_patterns) def set_stickycookie(self, txt): if txt: @@ -601,7 +616,7 @@ class FlowMaster(controller.Master): ] if all(e): self.shutdown() - self.client_playback.tick(self, timeout) + self.client_playback.tick(self) return controller.Master.tick(self, q, timeout) @@ -612,6 +627,11 @@ class FlowMaster(controller.Master): """ Loads a flow, and returns a new flow object. """ + + if self.server and self.server.config.mode == "reverse": + f.request.host, f.request.port = self.server.config.mode.dst[2:] + f.request.scheme = "https" if self.server.config.mode.dst[1] else "http" + f.reply = controller.DummyReply() if f.request: self.handle_request(f) @@ -656,6 +676,8 @@ class FlowMaster(controller.Master): """ Returns None if successful, or error message if not. """ + if f.live: + return "Can't replay request which is still live..." if f.intercepting: return "Can't replay while intercepting..." if f.request.content == http.CONTENT_MISSING: @@ -705,7 +727,11 @@ class FlowMaster(controller.Master): if f.live: app = self.apps.get(f.request) if app: - err = app.serve(f, f.client_conn.wfile, **{"mitmproxy.master": self}) + err = app.serve( + f, + f.client_conn.wfile, + **{"mitmproxy.master": self} + ) if err: self.add_event("Error in wsgi app. %s"%err, "error") f.reply(protocol.KILL) @@ -720,8 +746,12 @@ class FlowMaster(controller.Master): def handle_responseheaders(self, f): self.run_script_hook("responseheaders", f) - if self.stream_large_bodies: - self.stream_large_bodies.run(f, False) + try: + if self.stream_large_bodies: + self.stream_large_bodies.run(f, False) + except netlib.http.HttpError: + f.reply(protocol.KILL) + return f.reply() return f @@ -755,7 +785,6 @@ class FlowMaster(controller.Master): self.stream = None - class FlowWriter: def __init__(self, fo): self.fo = fo @@ -787,7 +816,7 @@ class FlowReader: v = ".".join(str(i) for i in data["version"]) raise FlowReadError("Incompatible serialized data version: %s"%v) off = self.fo.tell() - yield handle.protocols[data["conntype"]]["flow"].from_state(data) + yield handle.protocols[data["type"]]["flow"].from_state(data) except ValueError, v: # Error is due to EOF if self.fo.tell() == off and self.fo.read() == '': diff --git a/libmproxy/main.py b/libmproxy/main.py index 2d6a0119..e5b7f56b 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -1,5 +1,4 @@ from __future__ import print_function, absolute_import -import argparse import os import signal import sys @@ -9,27 +8,43 @@ from .proxy import process_proxy_options, ProxyServerError from .proxy.server import DummyServer, ProxyServer +# This file is not included in coverage analysis or tests - anything that can be +# tested should live elsewhere. + def check_versions(): """ - Having installed a wrong version of pyOpenSSL or netlib is unfortunately a very common source of error. - Check before every start that both versions are somewhat okay. + Having installed a wrong version of pyOpenSSL or netlib is unfortunately a + very common source of error. Check before every start that both versions are + somewhat okay. """ - # We don't introduce backward-incompatible changes in patch versions. Only consider major and minor version. + # We don't introduce backward-incompatible changes in patch versions. Only + # consider major and minor version. if netlib.version.IVERSION[:2] != version.IVERSION[:2]: print( "Warning: You are using mitmdump %s with netlib %s. " - "Most likely, that doesn't work - please upgrade!" % (version.VERSION, netlib.version.VERSION), - file=sys.stderr) - import OpenSSL, inspect - + "Most likely, that won't work - please upgrade!" % ( + version.VERSION, netlib.version.VERSION + ), + file=sys.stderr + ) + import OpenSSL + import inspect v = tuple([int(x) for x in OpenSSL.__version__.split(".")][:2]) if v < (0, 14): - print("You are using an outdated version of pyOpenSSL: mitmproxy requires pyOpenSSL 0.14 or greater.", - file=sys.stderr) - # Some users apparently have multiple versions of pyOpenSSL installed. Report which one we got. + print( + "You are using an outdated version of pyOpenSSL:" + " mitmproxy requires pyOpenSSL 0.14 or greater.", + file=sys.stderr + ) + # Some users apparently have multiple versions of pyOpenSSL installed. + # Report which one we got. pyopenssl_path = os.path.dirname(inspect.getfile(OpenSSL)) - print("Your pyOpenSSL %s installation is located at %s" % (OpenSSL.__version__, pyopenssl_path), - file=sys.stderr) + print( + "Your pyOpenSSL %s installation is located at %s" % ( + OpenSSL.__version__, pyopenssl_path + ), + file=sys.stderr + ) sys.exit(1) @@ -38,8 +53,14 @@ def assert_utf8_env(): for i in ["LANG", "LC_CTYPE", "LC_ALL"]: spec += os.environ.get(i, "").lower() if "utf" not in spec: - print("Error: mitmproxy requires a UTF console environment.", file=sys.stderr) - print("Set your LANG enviroment variable to something like en_US.UTF-8", file=sys.stderr) + print( + "Error: mitmproxy requires a UTF console environment.", + file=sys.stderr + ) + print( + "Set your LANG enviroment variable to something like en_US.UTF-8", + file=sys.stderr + ) sys.exit(1) @@ -54,34 +75,13 @@ def get_server(dummy_server, options): sys.exit(1) -def mitmproxy_cmdline(): - # Don't import libmproxy.console for mitmdump, urwid is not available on all platforms. +def mitmproxy(): # pragma: nocover from . import console - from .console import palettes - - parser = argparse.ArgumentParser(usage="%(prog)s [options]") - parser.add_argument('--version', action='version', version=version.NAMEVERSION) - cmdline.common_options(parser) - parser.add_argument( - "--palette", type=str, default="dark", - action="store", dest="palette", - help="Select color palette: " + ", ".join(palettes.palettes.keys()) - ) - parser.add_argument( - "-e", - action="store_true", dest="eventlog", - help="Show event log." - ) - group = parser.add_argument_group( - "Filters", - "See help in mitmproxy for filter expression syntax." - ) - group.add_argument( - "-i", "--intercept", action="store", - type=str, dest="intercept", default=None, - help="Intercept filter expression." - ) + check_versions() + assert_utf8_env() + + parser = cmdline.mitmproxy() options = parser.parse_args() if options.quiet: options.verbose = 0 @@ -92,15 +92,6 @@ def mitmproxy_cmdline(): console_options.eventlog = options.eventlog console_options.intercept = options.intercept - return console_options, proxy_config - - -def mitmproxy(): # pragma: nocover - from . import console - - check_versions() - assert_utf8_env() - console_options, proxy_config = mitmproxy_cmdline() server = get_server(console_options.no_server, proxy_config) m = console.ConsoleMaster(server, console_options) @@ -110,24 +101,12 @@ def mitmproxy(): # pragma: nocover pass -def mitmdump_cmdline(): +def mitmdump(): # pragma: nocover from . import dump - parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]") - parser.add_argument('--version', action='version', version="mitmdump" + " " + version.VERSION) - cmdline.common_options(parser) - parser.add_argument( - "--keepserving", - action="store_true", dest="keepserving", default=False, - help="Continue serving after client playback or file read. We exit by default." - ) - parser.add_argument( - "-d", - action="count", dest="flow_detail", default=1, - help="Increase flow detail display level. Can be passed multiple times." - ) - parser.add_argument('args', nargs=argparse.REMAINDER) + check_versions() + parser = cmdline.mitmdump() options = parser.parse_args() if options.quiet: options.verbose = 0 @@ -139,14 +118,6 @@ def mitmdump_cmdline(): dump_options.keepserving = options.keepserving dump_options.filtstr = " ".join(options.args) if options.args else None - return dump_options, proxy_config - - -def mitmdump(): # pragma: nocover - from . import dump - - check_versions() - dump_options, proxy_config = mitmdump_cmdline() server = get_server(dump_options.no_server, proxy_config) try: @@ -164,44 +135,11 @@ def mitmdump(): # pragma: nocover pass -def mitmweb_cmdline(): +def mitmweb(): # pragma: nocover from . import web - parser = argparse.ArgumentParser(usage="%(prog)s [options]") - parser.add_argument( - '--version', - action='version', - version="mitmweb" + " " + version.VERSION - ) - - group = parser.add_argument_group("Mitmweb") - group.add_argument( - "--wport", - action="store", type=int, dest="wport", default=8081, - metavar="PORT", - help="Mitmweb port." - ) - group.add_argument( - "--wiface", - action="store", dest="wiface", default="127.0.0.1", - metavar="IFACE", - help="Mitmweb interface." - ) - group.add_argument( - "--wdebug", - action="store_true", dest="wdebug", - help="Turn on mitmweb debugging" - ) - - cmdline.common_options(parser) - group = parser.add_argument_group( - "Filters", - "See help in mitmproxy for filter expression syntax." - ) - group.add_argument( - "-i", "--intercept", action="store", - type=str, dest="intercept", default=None, - help="Intercept filter expression." - ) + + check_versions() + parser = cmdline.mitmweb() options = parser.parse_args() if options.quiet: @@ -213,14 +151,7 @@ def mitmweb_cmdline(): web_options.wdebug = options.wdebug web_options.wiface = options.wiface web_options.wport = options.wport - return web_options, proxy_config - -def mitmweb(): # pragma: nocover - from . import web - - check_versions() - web_options, proxy_config = mitmweb_cmdline() server = get_server(web_options.no_server, proxy_config) m = web.WebMaster(server, web_options) diff --git a/libmproxy/onboarding/app.py b/libmproxy/onboarding/app.py index 9b5db38a..4023fae2 100644 --- a/libmproxy/onboarding/app.py +++ b/libmproxy/onboarding/app.py @@ -18,12 +18,12 @@ def index(): @mapp.route("/cert/pem") def certs_pem(): - p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.pem") + p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.pem") return flask.Response(open(p, "rb").read(), mimetype='application/x-x509-ca-cert') @mapp.route("/cert/p12") def certs_p12(): - p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.p12") + p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.p12") return flask.Response(open(p, "rb").read(), mimetype='application/x-pkcs12') diff --git a/libmproxy/onboarding/templates/index.html b/libmproxy/onboarding/templates/index.html index 50cfd5db..65fda5d2 100644 --- a/libmproxy/onboarding/templates/index.html +++ b/libmproxy/onboarding/templates/index.html @@ -1,5 +1,5 @@ {% extends "frame.html" %} -{% block body %} +{% block body %} <center> <h2> Click to install the mitmproxy certificate: </h2> @@ -23,4 +23,13 @@ </div> </div> +<hr/> +<div class="text-center"> + Other mitmproxy users cannot intercept your connection. +</div> +<div class="text-center text-muted"> + This page is served by your local mitmproxy instance. The certificate you are about to install has been uniquely generated on mitmproxy's first run and is not shared + between mitmproxy installations. +</div> + {% endblock %} diff --git a/libmproxy/platform/windows.py b/libmproxy/platform/windows.py index ddbbed52..066a377d 100644 --- a/libmproxy/platform/windows.py +++ b/libmproxy/platform/windows.py @@ -1,4 +1,4 @@ -import argparse +import configargparse import cPickle as pickle from ctypes import byref, windll, Structure from ctypes.wintypes import DWORD @@ -361,7 +361,7 @@ class TransparentProxy(object): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Windows Transparent Proxy") + parser = configargparse.ArgumentParser(description="Windows Transparent Proxy") parser.add_argument('--mode', choices=['forward', 'local', 'both'], default="both", help='redirection operation mode: "forward" to only redirect forwarded packets, ' '"local" to only redirect packets originating from the local machine') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index de5f9950..49f5e8c0 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -18,12 +18,17 @@ HDR_FORM_URLENCODED = "application/x-www-form-urlencoded" CONTENT_MISSING = 0 +class KillSignal(Exception): + pass + + def get_line(fp): """ Get a line, possibly preceded by a blank. """ line = fp.readline() - if line == "\r\n" or line == "\n": # Possible leftover from previous message + if line == "\r\n" or line == "\n": + # Possible leftover from previous message line = fp.readline() if line == "": raise tcp.NetLibDisconnect() @@ -237,25 +242,47 @@ class HTTPRequest(HTTPMessage): is content associated, but not present. CONTENT_MISSING evaluates to False to make checking for the presence of content natural. - form_in: The request form which mitmproxy has received. The following values are possible: - - relative (GET /index.html, OPTIONS *) (covers origin form and asterisk form) - - absolute (GET http://example.com:80/index.html) - - authority-form (CONNECT example.com:443) - Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 + form_in: The request form which mitmproxy has received. The following + values are possible: + + - relative (GET /index.html, OPTIONS *) (covers origin form and + asterisk form) + - absolute (GET http://example.com:80/index.html) + - authority-form (CONNECT example.com:443) + Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 - form_out: The request form which mitmproxy has send out to the destination + form_out: The request form which mitmproxy will send out to the + destination timestamp_start: Timestamp indicating when request transmission started timestamp_end: Timestamp indicating when request transmission ended """ - def __init__(self, form_in, method, scheme, host, port, path, httpversion, headers, - content, timestamp_start=None, timestamp_end=None, form_out=None): + def __init__( + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + content, + timestamp_start=None, + timestamp_end=None, + form_out=None + ): assert isinstance(headers, ODictCaseless) or not headers - HTTPMessage.__init__(self, httpversion, headers, content, timestamp_start, - timestamp_end) - + HTTPMessage.__init__( + self, + httpversion, + headers, + content, + timestamp_start, + timestamp_end + ) self.form_in = form_in self.method = method self.scheme = scheme @@ -308,30 +335,43 @@ class HTTPRequest(HTTPMessage): request_line = get_line(rfile) - if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start + if hasattr(rfile, "first_byte_timestamp"): + # more accurate timestamp_start timestamp_start = rfile.first_byte_timestamp request_line_parts = http.parse_init(request_line) if not request_line_parts: - raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + raise http.HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) method, path, httpversion = request_line_parts if path == '*' or path.startswith("/"): form_in = "relative" if not netlib.utils.isascii(path): - raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + raise http.HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) elif method.upper() == 'CONNECT': form_in = "authority" r = http.parse_init_connect(request_line) if not r: - raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + raise http.HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) host, port, _ = r path = None else: form_in = "absolute" r = http.parse_init_proxy(request_line) if not r: - raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + raise http.HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) _, scheme, host, port, path, _ = r headers = http.read_headers(rfile) @@ -343,50 +383,69 @@ class HTTPRequest(HTTPMessage): method, None, True) timestamp_end = utils.timestamp() - return HTTPRequest(form_in, method, scheme, host, port, path, httpversion, headers, - content, timestamp_start, timestamp_end) + return HTTPRequest( + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + content, + timestamp_start, + timestamp_end + ) def _assemble_first_line(self, form=None): form = form or self.form_out if form == "relative": - path = self.path if self.method != "OPTIONS" else "*" - request_line = '%s %s HTTP/%s.%s' % \ - (self.method, path, self.httpversion[0], self.httpversion[1]) + request_line = '%s %s HTTP/%s.%s' % ( + self.method, self.path, self.httpversion[0], self.httpversion[1] + ) elif form == "authority": - request_line = '%s %s:%s HTTP/%s.%s' % (self.method, self.host, self.port, - self.httpversion[0], self.httpversion[1]) + request_line = '%s %s:%s HTTP/%s.%s' % ( + self.method, self.host, self.port, self.httpversion[0], + self.httpversion[1] + ) elif form == "absolute": - request_line = '%s %s://%s:%s%s HTTP/%s.%s' % \ - (self.method, self.scheme, self.host, self.port, self.path, - self.httpversion[0], self.httpversion[1]) + request_line = '%s %s://%s:%s%s HTTP/%s.%s' % ( + self.method, self.scheme, self.host, + self.port, self.path, self.httpversion[0], + self.httpversion[1] + ) else: raise http.HttpError(400, "Invalid request form") return request_line + # This list is adopted legacy code. + # We probably don't need to strip off keep-alive. + _headers_to_strip_off = ['Proxy-Connection', + 'Keep-Alive', + 'Connection', + 'Transfer-Encoding', + 'Upgrade'] + def _assemble_headers(self): headers = self.headers.copy() - for k in ['Proxy-Connection', - 'Keep-Alive', - 'Connection', - 'Transfer-Encoding']: + for k in self._headers_to_strip_off: del headers[k] - if headers["Upgrade"] == ["h2c"]: # Suppress HTTP2 https://http2.github.io/http2-spec/index.html#discover-http - del headers["Upgrade"] - if not 'host' in headers and self.scheme and self.host and self.port: + if 'host' not in headers and self.scheme and self.host and self.port: headers["Host"] = [utils.hostport(self.scheme, self.host, self.port)] - if self.content: + # If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header. + if self.content or self.content == "": headers["Content-Length"] = [str(len(self.content))] - elif 'Transfer-Encoding' in self.headers: # content-length for e.g. chuncked transfer-encoding with no content - headers["Content-Length"] = ["0"] return str(headers) def _assemble_head(self, form=None): - return "%s\r\n%s\r\n" % (self._assemble_first_line(form), self._assemble_headers()) + return "%s\r\n%s\r\n" % ( + self._assemble_first_line(form), self._assemble_headers() + ) def assemble(self, form=None): """ @@ -396,7 +455,10 @@ class HTTPRequest(HTTPMessage): Raises an Exception if the request cannot be assembled. """ if self.content == CONTENT_MISSING: - raise proxy.ProxyError(502, "Cannot assemble flow with CONTENT_MISSING") + raise proxy.ProxyError( + 502, + "Cannot assemble flow with CONTENT_MISSING" + ) head = self._assemble_head(form) if self.content: return head + self.content @@ -644,7 +706,9 @@ class HTTPResponse(HTTPMessage): return "<HTTPResponse: {code} {msg} ({contenttype}, {size})>".format( code=self.code, msg=self.msg, - contenttype=self.headers.get_first("content-type", "unknown content type"), + contenttype=self.headers.get_first( + "content-type", "unknown content type" + ), size=size ) @@ -665,7 +729,8 @@ class HTTPResponse(HTTPMessage): body_size_limit, include_body=include_body) - if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start + if hasattr(rfile, "first_byte_timestamp"): + # more accurate timestamp_start timestamp_start = rfile.first_byte_timestamp if include_body: @@ -687,26 +752,30 @@ class HTTPResponse(HTTPMessage): return 'HTTP/%s.%s %s %s' % \ (self.httpversion[0], self.httpversion[1], self.code, self.msg) + _headers_to_strip_off = ['Proxy-Connection', + 'Alternate-Protocol', + 'Alt-Svc'] + def _assemble_headers(self, preserve_transfer_encoding=False): headers = self.headers.copy() - for k in ['Proxy-Connection', - 'Alternate-Protocol', - 'Alt-Svc']: + for k in self._headers_to_strip_off: del headers[k] if not preserve_transfer_encoding: del headers['Transfer-Encoding'] - if self.content: + # If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header. + if self.content or self.content == "": headers["Content-Length"] = [str(len(self.content))] - # add content-length for chuncked transfer-encoding with no content - elif not preserve_transfer_encoding and 'Transfer-Encoding' in self.headers: - headers["Content-Length"] = ["0"] return str(headers) def _assemble_head(self, preserve_transfer_encoding=False): return '%s\r\n%s\r\n' % ( - self._assemble_first_line(), self._assemble_headers(preserve_transfer_encoding=preserve_transfer_encoding)) + self._assemble_first_line(), + self._assemble_headers( + preserve_transfer_encoding=preserve_transfer_encoding + ) + ) def assemble(self): """ @@ -716,7 +785,10 @@ class HTTPResponse(HTTPMessage): Raises an Exception if the request cannot be assembled. """ if self.content == CONTENT_MISSING: - raise proxy.ProxyError(502, "Cannot assemble flow with CONTENT_MISSING") + raise proxy.ProxyError( + 502, + "Cannot assemble flow with CONTENT_MISSING" + ) head = self._assemble_head() if self.content: return head + self.content @@ -783,8 +855,9 @@ class HTTPResponse(HTTPMessage): pairs = [pair.partition("=") for pair in header.split(';')] cookie_name = pairs[0][0] # the key of the first key/value pairs cookie_value = pairs[0][2] # the value of the first key/value pairs - cookie_parameters = {key.strip().lower(): value.strip() for key, sep, value in - pairs[1:]} + cookie_parameters = { + key.strip().lower(): value.strip() for key, sep, value in pairs[1:] + } cookies.append((cookie_name, (cookie_value, cookie_parameters))) return dict(cookies) @@ -817,7 +890,8 @@ class HTTPFlow(Flow): self.response = None """@type: HTTPResponse""" - self.intercepting = False # FIXME: Should that rather be an attribute of Flow? + # FIXME: Should that rather be an attribute of Flow? + self.intercepting = False _stateobject_attributes = Flow._stateobject_attributes.copy() _stateobject_attributes.update( @@ -905,7 +979,9 @@ class HTTPFlow(Flow): class HttpAuthenticationError(Exception): def __init__(self, auth_headers=None): - super(HttpAuthenticationError, self).__init__("Proxy Authentication Required") + super(HttpAuthenticationError, self).__init__( + "Proxy Authentication Required" + ) self.headers = auth_headers self.code = 407 @@ -937,16 +1013,23 @@ class HTTPHandler(ProtocolHandler): try: self.c.server_conn.send(request_raw) # Only get the headers at first... - flow.response = HTTPResponse.from_stream(self.c.server_conn.rfile, flow.request.method, - body_size_limit=self.c.config.body_size_limit, - include_body=False) + flow.response = HTTPResponse.from_stream( + self.c.server_conn.rfile, flow.request.method, + body_size_limit=self.c.config.body_size_limit, + include_body=False + ) break except (tcp.NetLibDisconnect, http.HttpErrorConnClosed), v: - self.c.log("error in server communication: %s" % repr(v), level="debug") + self.c.log( + "error in server communication: %s" % repr(v), + level="debug" + ) if attempt == 0: - # In any case, we try to reconnect at least once. - # This is necessary because it might be possible that we already initiated an upstream connection - # after clientconnect that has already been expired, e.g consider the following event log: + # In any case, we try to reconnect at least once. This is + # necessary because it might be possible that we already + # initiated an upstream connection after clientconnect that + # has already been expired, e.g consider the following event + # log: # > clientconnect (transparent mode destination known) # > serverconnect # > read n% of large request @@ -959,19 +1042,21 @@ class HTTPHandler(ProtocolHandler): # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True - self.c.channel.ask("responseheaders", flow) - - # now get the rest of the request body, if body still needs to be read - # but not streaming this response - if flow.response.stream: - flow.response.content = CONTENT_MISSING + flow = self.c.channel.ask("responseheaders", flow) + if flow is None or flow == KILL: + raise KillSignal() else: - flow.response.content = http.read_http_body( - self.c.server_conn.rfile, flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, flow.response.code, False - ) - flow.response.timestamp_end = utils.timestamp() + # now get the rest of the request body, if body still needs to be + # read but not streaming this response + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = http.read_http_body( + self.c.server_conn.rfile, flow.response.headers, + self.c.config.body_size_limit, + flow.request.method, flow.response.code, False + ) + flow.response.timestamp_end = utils.timestamp() def handle_flow(self): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live) @@ -1001,10 +1086,10 @@ class HTTPHandler(ProtocolHandler): # sent through to the Master. flow.request = req request_reply = self.c.channel.ask("request", flow) - self.process_server_address(flow) # The inline script may have changed request.host - if request_reply is None or request_reply == KILL: - return False + raise KillSignal() + + self.process_server_address(flow) # The inline script may have changed request.host if isinstance(request_reply, HTTPResponse): flow.response = request_reply @@ -1018,7 +1103,7 @@ class HTTPHandler(ProtocolHandler): self.c.log("response", "debug", [flow.response._assemble_first_line()]) response_reply = self.c.channel.ask("response", flow) if response_reply is None or response_reply == KILL: - return False + raise KillSignal() self.send_response_to_client(flow) @@ -1050,15 +1135,27 @@ class HTTPHandler(ProtocolHandler): flow.live.restore_server() return True # Next flow please. - except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: + except ( + HttpAuthenticationError, + http.HttpError, + proxy.ProxyError, + tcp.NetLibError, + ), e: self.handle_error(e, flow) + except KillSignal: + self.c.log("Connection killed", "info") finally: flow.live = None # Connection is not live anymore. return False def handle_server_reconnect(self, state): if state["state"] == "connect": - send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False) + send_connect_request( + self.c.server_conn, + state["host"], + state["port"], + update_state=False + ) else: # pragma: nocover raise RuntimeError("Unknown State: %s" % state["state"]) @@ -1079,14 +1176,14 @@ class HTTPHandler(ProtocolHandler): if message: self.c.log(message, level="info") if message_debug: - self.c.log(message, level="debug") + self.c.log(message_debug, level="debug") if flow: - # TODO: no flows without request or with both request and response at the moment. + # TODO: no flows without request or with both request and response + # at the moment. if flow.request and not flow.response: flow.error = Error(message or message_debug) self.c.channel.ask("error", flow) - try: code = getattr(error, "code", 502) headers = getattr(error, "headers", None) @@ -1100,12 +1197,22 @@ class HTTPHandler(ProtocolHandler): def send_error(self, code, message, headers): response = http_status.RESPONSES.get(code, "Unknown") - html_content = '<html><head>\n<title>%d %s</title>\n</head>\n<body>\n%s\n</body>\n</html>' % \ - (code, response, message) + html_content = """ + <html> + <head> + <title>%d %s</title> + </head> + <body>%s</body> + </html> + """ % (code, response, message) self.c.client_conn.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response)) - self.c.client_conn.wfile.write("Server: %s\r\n" % self.c.config.server_version) + self.c.client_conn.wfile.write( + "Server: %s\r\n" % self.c.config.server_version + ) self.c.client_conn.wfile.write("Content-type: text/html\r\n") - self.c.client_conn.wfile.write("Content-Length: %d\r\n" % len(html_content)) + self.c.client_conn.wfile.write( + "Content-Length: %d\r\n" % len(html_content) + ) if headers: for key, value in headers.items(): self.c.client_conn.wfile.write("%s: %s\r\n" % (key, value)) @@ -1145,11 +1252,15 @@ class HTTPHandler(ProtocolHandler): # Now we can process the request. if request.form_in == "authority": if self.c.client_conn.ssl_established: - raise http.HttpError(400, "Must not CONNECT on already encrypted connection") + raise http.HttpError( + 400, + "Must not CONNECT on already encrypted connection" + ) if self.c.config.mode == "regular": self.c.set_server_address((request.host, request.port)) - flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow + # Update server_conn attribute on the flow + flow.server_conn = self.c.server_conn self.c.establish_server_connection() self.c.client_conn.send( 'HTTP/1.1 200 Connection established\r\n' + @@ -1161,7 +1272,9 @@ class HTTPHandler(ProtocolHandler): elif self.c.config.mode == "upstream": return None else: - pass # CONNECT should never occur if we don't expect absolute-form requests + # CONNECT should never occur if we don't expect absolute-form + # requests + pass elif request.form_in == self.expected_form_in: @@ -1169,61 +1282,77 @@ class HTTPHandler(ProtocolHandler): if request.form_in == "absolute": if request.scheme != "http": - raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme) + raise http.HttpError( + 400, + "Invalid request scheme: %s" % request.scheme + ) if self.c.config.mode == "regular": - # Update info so that an inline script sees the correct value at flow.server_conn + # Update info so that an inline script sees the correct + # value at flow.server_conn self.c.set_server_address((request.host, request.port)) flow.server_conn = self.c.server_conn return None - - raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" % - (self.expected_form_in, request.form_in)) + raise http.HttpError( + 400, "Invalid HTTP request form (expected: %s, got: %s)" % ( + self.expected_form_in, request.form_in + ) + ) def process_server_address(self, flow): # Depending on the proxy mode, server handling is entirely different - # We provide a mostly unified API to the user, which needs to be unfiddled here + # We provide a mostly unified API to the user, which needs to be + # unfiddled here # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 ) address = netlib.tcp.Address((flow.request.host, flow.request.port)) ssl = (flow.request.scheme == "https") if self.c.config.mode == "upstream": - - # The connection to the upstream proxy may have a state we may need to take into account. + # The connection to the upstream proxy may have a state we may need + # to take into account. connected_to = None for s in flow.server_conn.state: if s[0] == "http" and s[1]["state"] == "connect": connected_to = tcp.Address((s[1]["host"], s[1]["port"])) - # We need to reconnect if the current flow either requires a (possibly impossible) - # change to the connection state, e.g. the host has changed but we already CONNECTed somewhere else. + # We need to reconnect if the current flow either requires a + # (possibly impossible) change to the connection state, e.g. the + # host has changed but we already CONNECTed somewhere else. needs_server_change = ( ssl != self.c.server_conn.ssl_established or - (connected_to and address != connected_to) # HTTP proxying is "stateless", CONNECT isn't. + # HTTP proxying is "stateless", CONNECT isn't. + (connected_to and address != connected_to) ) if needs_server_change: # force create new connection to the proxy server to reset state self.live.change_server(self.c.server_conn.address, force=True) if ssl: - send_connect_request(self.c.server_conn, address.host, address.port) + send_connect_request( + self.c.server_conn, + address.host, + address.port + ) self.c.establish_ssl(server=True) else: - # If we're not in upstream mode, we just want to update the host and possibly establish TLS. - self.live.change_server(address, ssl=ssl) # this is a no op if the addresses match. + # If we're not in upstream mode, we just want to update the host and + # possibly establish TLS. This is a no op if the addresses match. + self.live.change_server(address, ssl=ssl) flow.server_conn = self.c.server_conn def send_response_to_client(self, flow): if not flow.response.stream: # no streaming: - # we already received the full response from the server and can send it to the client straight away. + # we already received the full response from the server and can send + # it to the client straight away. self.c.client_conn.send(flow.response.assemble()) else: # streaming: - # First send the headers and then transfer the response incrementally: + # First send the headers and then transfer the response + # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, @@ -1237,7 +1366,8 @@ class HTTPHandler(ProtocolHandler): def check_close_connection(self, flow): """ - Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so. + Checks if the connection should be closed depending on the HTTP + semantics. Returns True, if so. """ close_connection = ( http.connection_close(flow.request.httpversion, flow.request.headers) or @@ -1260,20 +1390,39 @@ class HTTPHandler(ProtocolHandler): Returns False, if the connection should be closed immediately. """ address = tcp.Address.wrap(address) - if self.c.check_ignore_address(address): + if self.c.config.check_ignore(address): self.c.log("Ignore host: %s:%s" % address(), "info") - TCPHandler(self.c).handle_messages() + TCPHandler(self.c, log=False).handle_messages() return False else: self.expected_form_in = "relative" self.expected_form_out = "relative" self.skip_authentication = True - if address.port in self.c.config.ssl_ports: + # In practice, nobody issues a CONNECT request to send unencrypted HTTP requests afterwards. + # If we don't delegate to TCP mode, we should always negotiate a SSL connection. + # + # FIXME: + # Turns out the previous statement isn't entirely true. Chrome on Windows CONNECTs to :80 + # if an explicit proxy is configured and a websocket connection should be established. + # We don't support websocket at the moment, so it fails anyway, but we should come up with + # a better solution to this if we start to support WebSockets. + should_establish_ssl = ( + address.port in self.c.config.ssl_ports + or + not self.c.config.check_tcp(address) + ) + + if should_establish_ssl: self.c.log("Received CONNECT request to SSL port. Upgrading to SSL...", "debug") self.c.establish_ssl(server=True, client=True) self.c.log("Upgrade to SSL completed.", "debug") + if self.c.config.check_tcp(address): + self.c.log("Generic TCP mode for host: %s:%s" % address(), "info") + TCPHandler(self.c).handle_messages() + return False + return True def authenticate(self, request): @@ -1297,31 +1446,43 @@ class RequestReplayThread(threading.Thread): r = self.flow.request form_out_backup = r.form_out try: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - server_address = self.config.mode.get_upstream_server(self.flow.client_conn)[2:] - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - send_connect_request(server, r.host, r.port) - server.establish_ssl(self.config.clientcerts, sni=r.host) - r.form_out = "relative" - else: - r.form_out = "absolute" + self.flow.response = None + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == KILL: + raise KillSignal() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - server.establish_ssl(self.config.clientcerts, sni=r.host) - r.form_out = "relative" - - server.send(r.assemble()) - self.flow.response = HTTPResponse.from_stream(server.rfile, r.method, - body_size_limit=self.config.body_size_limit) - self.channel.ask("response", self.flow) - except (proxy.ProxyError, http.HttpError, tcp.NetLibError), v: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + server_address = self.config.mode.get_upstream_server(self.flow.client_conn)[2:] + server = ServerConnection(server_address) + server.connect() + if r.scheme == "https": + send_connect_request(server, r.host, r.port) + server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + if r.scheme == "https": + server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni) + r.form_out = "relative" + + server.send(r.assemble()) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_stream(server.rfile, r.method, + body_size_limit=self.config.body_size_limit) + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == KILL: + raise KillSignal() + except (proxy.ProxyError, http.HttpError, tcp.NetLibError) as v: self.flow.error = Error(repr(v)) self.channel.ask("error", self.flow) + except KillSignal: + self.channel.tell("log", proxy.Log("Connection killed", "info")) finally: r.form_out = form_out_backup diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py index 519693db..3be1cc45 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/protocol/primitives.py @@ -59,8 +59,8 @@ class Flow(stateobject.StateObject): A Flow is a collection of objects representing a single transaction. This class is usually subclassed for each protocol, e.g. HTTPFlow. """ - def __init__(self, conntype, client_conn, server_conn, live=None): - self.conntype = conntype + def __init__(self, type, client_conn, server_conn, live=None): + self.type = type self.id = str(uuid.uuid4()) self.client_conn = client_conn """@type: ClientConnection""" @@ -78,7 +78,7 @@ class Flow(stateobject.StateObject): error=Error, client_conn=ClientConnection, server_conn=ServerConnection, - conntype=str + type=str ) def get_state(self, short=False): @@ -174,7 +174,7 @@ class LiveConnection(object): self._backup_server_conn = None """@type: libmproxy.proxy.connection.ServerConnection""" - def change_server(self, address, ssl=None, force=False, persistent_change=False): + def change_server(self, address, ssl=None, sni=None, force=False, persistent_change=False): """ Change the server connection to the specified address. @returns: @@ -183,7 +183,14 @@ class LiveConnection(object): """ address = netlib.tcp.Address.wrap(address) - ssl_mismatch = (ssl is not None and ssl != self.c.server_conn.ssl_established) + ssl_mismatch = ( + ssl is not None and + ( + ssl != self.c.server_conn.ssl_established + or + (sni is not None and sni != self.c.sni) + ) + ) address_mismatch = (address != self.c.server_conn.address) if persistent_change: @@ -212,6 +219,8 @@ class LiveConnection(object): self.c.set_server_address(address) self.c.establish_server_connection(ask=False) + if sni: + self.c.sni = sni if ssl: self.c.establish_ssl(server=True) return True diff --git a/libmproxy/protocol/tcp.py b/libmproxy/protocol/tcp.py index a56bf07b..da0c9087 100644 --- a/libmproxy/protocol/tcp.py +++ b/libmproxy/protocol/tcp.py @@ -13,6 +13,10 @@ class TCPHandler(ProtocolHandler): chunk_size = 4096 + def __init__(self, c, log=True): + super(TCPHandler, self).__init__(c) + self.log = log + def handle_messages(self): self.c.establish_server_connection() @@ -63,26 +67,25 @@ class TCPHandler(ProtocolHandler): # if one of the peers is over SSL, we need to send # bytes/strings if not src.ssl_established: - # only ssl to dst, i.e. we revc'd into buf but need - # bytes/string now. + # we revc'd into buf but need bytes/string now. contents = buf[:size].tobytes() - self.c.log( - "%s %s\r\n%s" % ( - direction, dst_str, cleanBin(contents) - ), - "debug" - ) + if self.log: + self.c.log( + "%s %s\r\n%s" % ( + direction, dst_str, cleanBin(contents) + ), + "info" + ) dst.connection.send(contents) else: # socket.socket.send supports raw bytearrays/memoryviews - self.c.log( - "%s %s\r\n%s" % ( - direction, - dst_str, - cleanBin(buf.tobytes()) - ), - "debug" - ) + if self.log: + self.c.log( + "%s %s\r\n%s" % ( + direction, dst_str, cleanBin(buf.tobytes()) + ), + "info" + ) dst.connection.send(buf[:size]) except socket.error as e: self.c.log("TCP connection closed unexpectedly.", "debug") diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 62104a24..3d373a28 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,26 +1,54 @@ from __future__ import absolute_import import os import re -from netlib import http_auth, certutils +from netlib import http_auth, certutils, tcp from .. import utils, platform, version -from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode +from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" -CONF_DIR = "~/.mitmproxy" +CA_DIR = "~/.mitmproxy" -def parse_host_pattern(patterns): - return [re.compile(p, re.IGNORECASE) for p in patterns] +class HostMatcher(object): + def __init__(self, patterns=[]): + self.patterns = list(patterns) + self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] + + def __call__(self, address): + address = tcp.Address.wrap(address) + host = "%s:%s" % (address.host, address.port) + if any(rex.search(host) for rex in self.regexes): + return True + else: + return False + + def __nonzero__(self): + return bool(self.patterns) class ProxyConfig: - def __init__(self, host='', port=8080, server_version=version.NAMEVERSION, - confdir=CONF_DIR, ca_file=None, clientcerts=None, - no_upstream_cert=False, body_size_limit=None, - mode=None, upstream_server=None, http_form_in=None, http_form_out=None, - authenticator=None, ignore=[], - ciphers=None, certs=[], certforward=False, ssl_ports=TRANSPARENT_SSL_PORTS): + def __init__( + self, + host='', + port=8080, + server_version=version.NAMEVERSION, + cadir=CA_DIR, + clientcerts=None, + no_upstream_cert=False, + body_size_limit=None, + mode=None, + upstream_server=None, + http_form_in=None, + http_form_out=None, + authenticator=None, + ignore_hosts=[], + tcp_hosts=[], + ciphers=None, + certs=[], + certforward=False, + ssl_ports=TRANSPARENT_SSL_PORTS + ): self.host = host self.port = port self.server_version = server_version @@ -30,7 +58,9 @@ class ProxyConfig: self.body_size_limit = body_size_limit if mode == "transparent": - self.mode = TransparentProxyMode(platform.resolver(), TRANSPARENT_SSL_PORTS) + self.mode = TransparentProxyMode(platform.resolver(), ssl_ports) + elif mode == "socks5": + self.mode = Socks5ProxyMode(ssl_ports) elif mode == "reverse": self.mode = ReverseProxyMode(upstream_server) elif mode == "upstream": @@ -42,11 +72,11 @@ class ProxyConfig: self.mode.http_form_in = http_form_in or self.mode.http_form_in self.mode.http_form_out = http_form_out or self.mode.http_form_out - self.ignore = parse_host_pattern(ignore) + self.check_ignore = HostMatcher(ignore_hosts) + self.check_tcp = HostMatcher(tcp_hosts) self.authenticator = authenticator - self.confdir = os.path.expanduser(confdir) - self.ca_file = ca_file or os.path.join(self.confdir, CONF_BASENAME + "-ca.pem") - self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) + self.cadir = os.path.expanduser(cadir) + self.certstore = certutils.CertStore.from_store(self.cadir, CONF_BASENAME) for spec, cert in certs: self.certstore.add_cert_file(spec, cert) self.certforward = certforward @@ -63,6 +93,9 @@ def process_proxy_options(parser, options): if not platform.resolver: return parser.error("Transparent mode not supported on this platform.") mode = "transparent" + if options.socks_proxy: + c += 1 + mode = "socks5" if options.reverse_proxy: c += 1 mode = "reverse" @@ -72,7 +105,7 @@ def process_proxy_options(parser, options): mode = "upstream" upstream_server = options.upstream_proxy if c > 1: - return parser.error("Transparent mode, reverse mode and upstream proxy mode " + return parser.error("Transparent, SOCKS5, reverse and upstream proxy mode " "are mutually exclusive.") if options.clientcerts: @@ -109,10 +142,16 @@ def process_proxy_options(parser, options): parser.error("Certificate file does not exist: %s" % parts[1]) certs.append(parts) + ssl_ports = options.ssl_ports + if options.ssl_ports != TRANSPARENT_SSL_PORTS: + # arparse appends to default value by default, strip that off. + # see http://bugs.python.org/issue16399 + ssl_ports = ssl_ports[len(TRANSPARENT_SSL_PORTS):] + return ProxyConfig( host=options.addr, port=options.port, - confdir=options.confdir, + cadir=options.cadir, clientcerts=options.clientcerts, no_upstream_cert=options.no_upstream_cert, body_size_limit=body_size_limit, @@ -120,11 +159,13 @@ def process_proxy_options(parser, options): upstream_server=upstream_server, http_form_in=options.http_form_in, http_form_out=options.http_form_out, - ignore=options.ignore, + ignore_hosts=options.ignore_hosts, + tcp_hosts=options.tcp_hosts, authenticator=authenticator, ciphers=options.ciphers, certs=certs, certforward=options.certforward, + ssl_ports=ssl_ports ) @@ -133,10 +174,12 @@ def ssl_option_group(parser): group.add_argument( "--cert", dest='certs', default=[], type=str, metavar="SPEC", action="append", - help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' \ - 'The domain may include a wildcard, and is equal to "*" if not specified. ' \ - 'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' \ - 'it is used, else the default key in the conf dir is used. Can be passed multiple times.' + help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' + 'The domain may include a wildcard, and is equal to "*" if not specified. ' + 'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' + 'it is used, else the default key in the conf dir is used. ' + 'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. ' + 'Can be passed multiple times.' ) group.add_argument( "--client-certs", action="store", @@ -159,7 +202,7 @@ def ssl_option_group(parser): help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--ssl-port", action="append", type=int, dest="ssl_ports", default=TRANSPARENT_SSL_PORTS, + "--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS), metavar="PORT", help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " "Defaults to %s." % str(TRANSPARENT_SSL_PORTS) diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index 23d089d3..c0ae424d 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -1,5 +1,5 @@ from __future__ import absolute_import - +from netlib import socks class ProxyError(Exception): def __init__(self, code, message, headers=None): @@ -15,7 +15,7 @@ class ProxyMode(object): http_form_in = None http_form_out = None - def get_upstream_server(self, conn): + def get_upstream_server(self, client_conn): """ Returns the address of the server to connect to. Returns None if the address needs to be determined on the protocol level (regular proxy mode) @@ -46,7 +46,7 @@ class RegularProxyMode(ProxyMode): http_form_in = "absolute" http_form_out = "relative" - def get_upstream_server(self, conn): + def get_upstream_server(self, client_conn): return None @@ -58,9 +58,9 @@ class TransparentProxyMode(ProxyMode): self.resolver = resolver self.sslports = sslports - def get_upstream_server(self, conn): + def get_upstream_server(self, client_conn): try: - dst = self.resolver.original_addr(conn) + dst = self.resolver.original_addr(client_conn.connection) except Exception, e: raise ProxyError(502, "Transparent mode failure: %s" % str(e)) @@ -71,11 +71,80 @@ class TransparentProxyMode(ProxyMode): return [ssl, ssl] + list(dst) +class Socks5ProxyMode(ProxyMode): + http_form_in = "relative" + http_form_out = "relative" + + def __init__(self, sslports): + self.sslports = sslports + + @staticmethod + def _assert_socks5(msg): + if msg.ver != socks.VERSION.SOCKS5: + if msg.ver == ord("G") and len(msg.methods) == ord("E"): + guess = "Probably not a SOCKS request but a regular HTTP request. " + else: + guess = "" + raise socks.SocksError( + socks.REP.GENERAL_SOCKS_SERVER_FAILURE, + guess + "Invalid SOCKS version. Expected 0x05, got 0x%x" % msg.ver) + + def get_upstream_server(self, client_conn): + try: + # Parse Client Greeting + client_greet = socks.ClientGreeting.from_file(client_conn.rfile) + self._assert_socks5(client_greet) + if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: + raise socks.SocksError( + socks.METHOD.NO_ACCEPTABLE_METHODS, + "mitmproxy only supports SOCKS without authentication" + ) + + # Send Server Greeting + server_greet = socks.ServerGreeting( + socks.VERSION.SOCKS5, + socks.METHOD.NO_AUTHENTICATION_REQUIRED + ) + server_greet.to_file(client_conn.wfile) + client_conn.wfile.flush() + + # Parse Connect Request + connect_request = socks.Message.from_file(client_conn.rfile) + self._assert_socks5(connect_request) + if connect_request.msg != socks.CMD.CONNECT: + raise socks.SocksError( + socks.REP.COMMAND_NOT_SUPPORTED, + "mitmproxy only supports SOCKS5 CONNECT." + ) + + # We do not connect here yet, as the clientconnect event has not been handled yet. + + connect_reply = socks.Message( + socks.VERSION.SOCKS5, + socks.REP.SUCCEEDED, + socks.ATYP.DOMAINNAME, + client_conn.address # dummy value, we don't have an upstream connection yet. + ) + connect_reply.to_file(client_conn.wfile) + client_conn.wfile.flush() + + ssl = bool(connect_request.addr.port in self.sslports) + return ssl, ssl, connect_request.addr.host, connect_request.addr.port + + except socks.SocksError as e: + msg = socks.Message(5, e.code, socks.ATYP.DOMAINNAME, repr(e)) + try: + msg.to_file(client_conn.wfile) + except: + pass + raise ProxyError(502, "SOCKS5 mode failure: %s" % str(e)) + + class _ConstDestinationProxyMode(ProxyMode): def __init__(self, dst): self.dst = dst - def get_upstream_server(self, conn): + def get_upstream_server(self, client_conn): return self.dst diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 307a4bcd..55e2b30e 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -70,13 +70,15 @@ class ConnectionHandler: # Can we already identify the target server and connect to it? client_ssl, server_ssl = False, False - upstream_info = self.config.mode.get_upstream_server(self.client_conn.connection) + conn_kwargs = dict() + upstream_info = self.config.mode.get_upstream_server(self.client_conn) if upstream_info: self.set_server_address(upstream_info[2:]) client_ssl, server_ssl = upstream_info[:2] - if self.check_ignore_address(self.server_conn.address): + if self.config.check_ignore(self.server_conn.address): self.log("Ignore host: %s:%s" % self.server_conn.address(), "info") self.conntype = "tcp" + conn_kwargs["log"] = False client_ssl, server_ssl = False, False else: pass # No upstream info from the metadata: upstream info in the protocol (e.g. HTTP absolute-form) @@ -90,15 +92,18 @@ class ConnectionHandler: if client_ssl or server_ssl: self.establish_ssl(client=client_ssl, server=server_ssl) + if self.config.check_tcp(self.server_conn.address): + self.log("Generic TCP mode for host: %s:%s" % self.server_conn.address(), "info") + self.conntype = "tcp" + # Delegate handling to the protocol handler - protocol_handler(self.conntype)(self).handle_messages() + protocol_handler(self.conntype)(self, **conn_kwargs).handle_messages() - self.del_server_connection() self.log("clientdisconnect", "info") self.channel.tell("clientdisconnect", self) except ProxyError as e: - protocol_handler(self.conntype)(self).handle_error(e) + protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e) except Exception: import traceback, sys @@ -106,6 +111,10 @@ class ConnectionHandler: print >> sys.stderr, traceback.format_exc() print >> sys.stderr, "mitmproxy has crashed!" print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy" + finally: + # Make sure that we close the server connection in any case. + # The client connection is closed by the ProxyServer and does not have be handled here. + self.del_server_connection() def del_server_connection(self): """ @@ -113,20 +122,13 @@ class ConnectionHandler: """ if self.server_conn and self.server_conn.connection: self.server_conn.finish() + self.server_conn.close() self.log("serverdisconnect", "debug", ["%s:%s" % (self.server_conn.address.host, self.server_conn.address.port)]) self.channel.tell("serverdisconnect", self) self.server_conn = None self.sni = None - def check_ignore_address(self, address): - address = tcp.Address.wrap(address) - host = "%s:%s" % (address.host, address.port) - if host and any(rex.search(host) for rex in self.config.ignore): - return True - else: - return False - def set_server_address(self, address): """ Sets a new server address with the given priority. @@ -190,14 +192,14 @@ class ConnectionHandler: if client: if self.client_conn.ssl_established: raise ProxyError(502, "SSL to Client already established.") - cert, key = self.find_cert() + cert, key, chain_file = self.find_cert() try: self.client_conn.convert_to_ssl( cert, key, handle_sni=self.handle_sni, cipher_list=self.config.ciphers, dhparams=self.config.certstore.dhparams, - ca_file=self.config.ca_file + chain_file=chain_file ) except tcp.NetLibError as v: raise ProxyError(400, repr(v)) @@ -234,7 +236,7 @@ class ConnectionHandler: def find_cert(self): if self.config.certforward and self.server_conn.ssl_established: - return self.server_conn.cert, self.config.certstore.gen_pkey(self.server_conn.cert) + return self.server_conn.cert, self.config.certstore.gen_pkey(self.server_conn.cert), None else: host = self.server_conn.address.host sans = [] @@ -264,17 +266,17 @@ class ConnectionHandler: self.log("SNI received: %s" % self.sni, "debug") self.server_reconnect() # reconnect to upstream server with SNI # Now, change client context to reflect changed certificate: - cert, key = self.find_cert() + cert, key, chain_file = self.find_cert() new_context = self.client_conn._create_ssl_context( cert, key, method=SSL.TLSv1_METHOD, cipher_list=self.config.ciphers, dhparams=self.config.certstore.dhparams, - ca_file=self.config.ca_file + chain_file=chain_file ) connection.set_context(new_context) # An unhandled exception in this method will core dump PyOpenSSL, so # make dang sure it doesn't happen. - except Exception: # pragma: no cover + except: # pragma: no cover import traceback - self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error")
\ No newline at end of file + self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") diff --git a/libmproxy/version.py b/libmproxy/version.py index 3483625d..8dcaecc8 100644 --- a/libmproxy/version.py +++ b/libmproxy/version.py @@ -1,5 +1,9 @@ -IVERSION = (0, 11) +IVERSION = (0, 11, 1) VERSION = ".".join(str(i) for i in IVERSION) MINORVERSION = ".".join(str(i) for i in IVERSION[:2]) NAME = "mitmproxy" NAMEVERSION = NAME + " " + VERSION + +NEXT_MINORVERSION = list(IVERSION) +NEXT_MINORVERSION[1] += 1 +NEXT_MINORVERSION = ".".join(str(i) for i in NEXT_MINORVERSION[:2])
\ No newline at end of file diff --git a/libmproxy/web/static/flows.json b/libmproxy/web/static/flows.json index a0358db0..35accd38 100644 --- a/libmproxy/web/static/flows.json +++ b/libmproxy/web/static/flows.json @@ -93,7 +93,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -259,7 +259,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -425,7 +425,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -595,7 +595,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -765,7 +765,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -919,7 +919,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1057,7 +1057,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1195,7 +1195,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1329,7 +1329,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1483,7 +1483,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1633,7 +1633,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1767,7 +1767,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1901,7 +1901,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -2027,7 +2027,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 diff --git a/release/osx-binaries b/release/osx-binaries index 4be85800..9945e471 100755 --- a/release/osx-binaries +++ b/release/osx-binaries @@ -10,7 +10,8 @@ # answer is to touch the __init__.py file in the zope directory. On my system: # touch /Library/Python/2.7/site-packages/zope/__init__.py -# To run, change into the pyinstaller directory, and then run this script. +# To run, first install netlib and mitmproxy, then change into the pyinstaller +# directory, and then run this script. DST=/tmp/osx-mitmproxy MITMPROXY=~/mitmproxy/mitmproxy diff --git a/release/release-checklist b/release/release-checklist index d0bf8aad..c0d8d82d 100644 --- a/release/release-checklist +++ b/release/release-checklist @@ -2,8 +2,6 @@ - Bump the version number: mitmproxy/libmproxy/version.py - mitmproxy/requirements.txt - mitmproxy/test/requirements.txt netlib/netlib/version.py netlib/requirements.txt netlib/test/requirements.txt @@ -26,3 +24,22 @@ - tar -xzvf pkgfile.tgz - virtualenv venv +- Build the OSX binaries + - Follow instructions in osxbinaries + - Package: + cp -r ./doc /tmp/osx-mitmproxy/ + mv /tmp/osx-mitmproxy /tmp/osx-mitmproxy-VERSION + tar -czvf /tmp/osx-mitmproxy-VERSION.tar.gz /tmp/osx-mitmproxy-VERSION + mv /tmp/osx-mitmproxy-VERSION.tar.gz ~/mitmproxy/www.mitmproxy.org/src/download + +- Build the sources for each project: + python ./setup.py sdist + mv ./dist/FILE ~/mitmproxy/www.mitmproxy.org/src/download + + +- Adjust links on www.mitmproxy.org + +- Upload to pypi for each project: + + python ./setup.py sdist upload + diff --git a/release/test-release b/release/test-release index 8b53c14c..8cbcea8c 100755 --- a/release/test-release +++ b/release/test-release @@ -23,15 +23,16 @@ python ./setup.py -q sdist --dist-dir $DST echo "Creating virtualenv for test install..." virtualenv -q $DST/venv +cd $DST echo "Installing netlib..." -$DST/venv/bin/pip -q install --download-cache ~/.pipcache $DST/netlib* +./venv/bin/pip -q install --download-cache ~/.pipcache ./netlib* echo "Installing pathod..." -$DST/venv/bin/pip -q install --download-cache ~/.pipcache $DST/pathod* +./venv/bin/pip -q install --download-cache ~/.pipcache ./pathod* echo "Installing mitmproxy..." -$DST/venv/bin/pip -q install --download-cache ~/.pipcache $DST/mitmproxy* +./venv/bin/pip -q install --download-cache ~/.pipcache ./mitmproxy* echo "Running binaries..." -$DST/venv/bin/mitmproxy --version -$DST/venv/bin/mitmdump --version -$DST/venv/bin/pathod --version -$DST/venv/bin/pathoc --version +./venv/bin/mitmproxy --version +./venv/bin/mitmdump --version +./venv/bin/pathod --version +./venv/bin/pathoc --version diff --git a/requirements.txt b/requirements.txt index d84347b7..946e5ffe 100644 --- a/requirements.txt +++ b/requirements.txt @@ -1,3 +1,3 @@ -e git+https://github.com/mitmproxy/netlib.git#egg=netlib -e git+https://github.com/mitmproxy/pathod.git#egg=pathod --e .[dev]
\ No newline at end of file +-e .[dev,examples]
\ No newline at end of file @@ -16,13 +16,12 @@ if os.name != "nt": scripts.append("mitmproxy") deps = { - "netlib>=%s" % version.MINORVERSION, + "netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION), "pyasn1>0.1.2", - "requests>=2.4.0", "pyOpenSSL>=0.14", "Flask>=0.10.1", "tornado>=4.0.2", - "sortedcontainers>=0.9.1" + "configargparse>=0.9.3" } script_deps = { "mitmproxy": { @@ -37,10 +36,6 @@ for script in scripts: if os.name == "nt": deps.add("pydivert>=0.0.4") # Transparent proxying on Windows -console_scripts = [ - "%s = libmproxy.main:%s" % (s, s) for s in scripts -] - setup( name="mitmproxy", @@ -67,14 +62,9 @@ setup( "Topic :: Internet :: Proxy Servers", "Topic :: Software Development :: Testing" ], - packages=find_packages(), include_package_data=True, - - entry_points={ - 'console_scripts': console_scripts - }, - + scripts = scripts, install_requires=list(deps), extras_require={ 'dev': [ @@ -82,12 +72,19 @@ setup( "nose>=1.3.0", "nose-cov>=1.6", "coveralls>=0.4.1", - "pathod>=%s" % version.MINORVERSION + "pathod>=%s, <%s" % ( + version.MINORVERSION, version.NEXT_MINORVERSION + ) ], 'contentviews': [ "pyamf>=0.6.1", "protobuf>=2.5.0", "cssutils>=1.0" + ], + 'examples': [ + "pytz", + "harparser", + "beautifulsoup4" ] } -)
\ No newline at end of file +) diff --git a/test/fuzzing/.env b/test/fuzzing/.env new file mode 100644 index 00000000..82ae6a8d --- /dev/null +++ b/test/fuzzing/.env @@ -0,0 +1,6 @@ + +MITMDUMP=../../mitmdump +PATHOD=../../../pathod/pathod +PATHOC=../../../pathod/pathoc +FUZZ_SETTINGS=-remTt 1 -n 0 + diff --git a/test/fuzzing/README b/test/fuzzing/README new file mode 100644 index 00000000..2760506f --- /dev/null +++ b/test/fuzzing/README @@ -0,0 +1,14 @@ + +A fuzzing architecture for mitmproxy +==================================== + +Quick start: + + honcho -f ./straight_stream start + + +Notes: + + - Processes are managed using honcho (pip install honcho) + - Paths and common settings live in .env + diff --git a/test/fuzzing/client_patterns b/test/fuzzing/client_patterns new file mode 100644 index 00000000..83457b6f --- /dev/null +++ b/test/fuzzing/client_patterns @@ -0,0 +1,4 @@ +get:'http://localhost:9999/p/200':ir,"\n" +get:'http://localhost:9999/p/200':ir,"\0" +get:'http://localhost:9999/p/200':ir,@5 +get:'http://localhost:9999/p/200':dr diff --git a/test/fuzzing/go_proxy b/test/fuzzing/go_proxy index c9b6aef6..ea29400f 100755 --- a/test/fuzzing/go_proxy +++ b/test/fuzzing/go_proxy @@ -3,20 +3,27 @@ # mitmproxy/mitmdump is running on port 8080 in straight proxy mode. # pathod is running on port 9999 -BASE_HTTP="/Users/aldo/git/public/pathod/pathoc -Tt 1 -eo -I 200,400,405,502 -p 8080 localhost " +BASE="../../../" +BASE_HTTP=$BASE"/pathod/pathoc -Tt 1 -e -I 200,400,405,502 -p 8080 localhost " +BASE_HTTPS=$BASE"/pathod/pathoc -sc localhost:9999 -Tt 1 -eo -I 200,400,404,405,502,800 -p 8080 localhost " + #$BASE_HTTP -n 10000 "get:'http://localhost:9999':ir,@1" #$BASE_HTTP -n 100 "get:'http://localhost:9999':dr" -#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200:ir,@300.0 +#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200':ir,@300" + +#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200:ir,@1'" +#$BASE_HTTP -n 100 "get:'http://localhost:9999/p/200:dr'" +#$BASE_HTTP -n 10000 "get:'http://localhost:9999/p/200:ir,@100'" # Assuming: # mitmproxy/mitmdump is running on port 8080 in straight proxy mode. # pathod with SSL enabled is running on port 9999 -BASE_HTTPS="/Users/aldo/git/public/pathod/pathoc -sc localhost:9999 -Tt 1 -eo -I 200,400,404,405,502,800 -p 8080 localhost " -$BASE_HTTPS -en 10000 "get:'/p/200:b@10:ir,@1'" +#$BASE_HTTPS -en 10000 "get:'/p/200:b@100:ir,@1'" #$BASE_HTTPS -en 10000 "get:'/p/200:ir,@1'" #$BASE_HTTPS -n 100 "get:'/p/200:dr'" #$BASE_HTTPS -n 10000 "get:'/p/200:ir,@3000'" #$BASE_HTTPS -n 10000 "get:'/p/200:ir,\"\\n\"'" + diff --git a/test/fuzzing/reverse_patterns b/test/fuzzing/reverse_patterns new file mode 100644 index 00000000..8d1d76a2 --- /dev/null +++ b/test/fuzzing/reverse_patterns @@ -0,0 +1,9 @@ +get:'/p/200':b@10:ir,"\n" +get:'/p/200':b@10:ir,"\r\n" +get:'/p/200':b@10:ir,"\0" +get:'/p/200':b@10:ir,@5 +get:'/p/200':b@10:dr + +get:'/p/200:b@10:ir,@1' +get:'/p/200:b@10:dr' +get:'/p/200:b@10:ir,@100' diff --git a/test/fuzzing/straight_stream b/test/fuzzing/straight_stream new file mode 100644 index 00000000..41e2a6e1 --- /dev/null +++ b/test/fuzzing/straight_stream @@ -0,0 +1,6 @@ + +mitmdump: $MITMDUMP +pathod: $PATHOD +pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 ./straight_stream_patterns +#pathoc: sleep 2 && $PATHOC localhost:8080 /tmp/err + diff --git a/test/fuzzing/straight_stream_patterns b/test/fuzzing/straight_stream_patterns new file mode 100644 index 00000000..93a066e6 --- /dev/null +++ b/test/fuzzing/straight_stream_patterns @@ -0,0 +1,17 @@ +get:'http://localhost:9999/p/':s'200:b"foo"':ir,'\n' +get:'http://localhost:9999/p/':s'200:b"foo"':ir,'a' +get:'http://localhost:9999/p/':s'200:b"foo"':ir,'9' +get:'http://localhost:9999/p/':s'200:b"foo"':ir,':' +get:'http://localhost:9999/p/':s'200:b"foo"':ir,'"' +get:'http://localhost:9999/p/':s'200:b"foo"':ir,'-' + +get:'http://localhost:9999/p/':s'200:b"foo":ir,"\n"' +get:'http://localhost:9999/p/':s'200:b"foo":ir,"a"' +get:'http://localhost:9999/p/':s'200:b"foo":ir,"9"' +get:'http://localhost:9999/p/':s'200:b"foo":ir,":"' +get:'http://localhost:9999/p/':s"200:b'foo':ir,'\"'" +get:'http://localhost:9999/p/':s'200:b"foo":ir,"-"' +get:'http://localhost:9999/p/':s'200:b"foo":dr' + +get:'http://localhost:9999/p/':s'200:b"foo"':ir,@2 +get:'http://localhost:9999/p/':s'200:b"foo":ir,@2' diff --git a/test/fuzzing/straight_stream_ssl b/test/fuzzing/straight_stream_ssl new file mode 100644 index 00000000..708ff0b3 --- /dev/null +++ b/test/fuzzing/straight_stream_ssl @@ -0,0 +1,6 @@ + +mitmdump: $MITMDUMP -q --stream 1 +pathod: $PATHOD +pathoc: sleep 2 && $PATHOC $FUZZ_SETTINGS localhost:8080 ./straight_stream_patterns +#pathoc: sleep 2 && $PATHOC localhost:8080 /tmp/err + diff --git a/test/test_cmdline.py b/test/test_cmdline.py index 12e8aa89..476fc620 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -1,7 +1,6 @@ import argparse from libmproxy import cmdline import tutils -import os.path def test_parse_replace_hook(): @@ -51,6 +50,7 @@ def test_parse_setheaders(): x = cmdline.parse_setheader("/foo/bar/voing") assert x == ("foo", "bar", "voing") + def test_common(): parser = argparse.ArgumentParser() cmdline.common_options(parser) @@ -108,3 +108,19 @@ def test_common(): assert len(v) == 1 assert v[0][2].strip() == "replacecontents" + +def test_mitmproxy(): + ap = cmdline.mitmproxy() + assert ap + + +def test_mitmdump(): + ap = cmdline.mitmdump() + assert ap + + +def test_mitmweb(): + ap = cmdline.mitmweb() + assert ap + + diff --git a/test/test_dump.py b/test/test_dump.py index 2e58e073..e9cb4d33 100644 --- a/test/test_dump.py +++ b/test/test_dump.py @@ -1,10 +1,12 @@ import os from cStringIO import StringIO -from libmproxy import dump, flow, proxy +from libmproxy import dump, flow +from libmproxy.protocol import http from libmproxy.proxy.primitives import Log import tutils import mock + def test_strfuncs(): t = tutils.tresp() t.is_replay = True @@ -58,6 +60,18 @@ class TestDumpMaster: assert m.handle_error(f) assert "error" in cs.getvalue() + def test_missing_content(self): + cs = StringIO() + o = dump.Options(flow_detail=3) + m = dump.DumpMaster(None, o, outfile=cs) + f = tutils.tflow() + f.request.content = http.CONTENT_MISSING + m.handle_request(f) + f.response = tutils.tresp() + f.response.content = http.CONTENT_MISSING + m.handle_response(f) + assert "content missing" in cs.getvalue() + def test_replay(self): cs = StringIO() diff --git a/test/test_examples.py b/test/test_examples.py index fd42e6f0..deb97b49 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -1,10 +1,8 @@ import glob -import mock from libmproxy import utils, script from libmproxy.proxy import config import tservers -@mock.patch.dict("sys.modules", {"bs4": mock.Mock()}) def test_load_scripts(): example_dir = utils.Data("libmproxy").path("../examples") scripts = glob.glob("%s/*.py" % example_dir) @@ -12,8 +10,11 @@ def test_load_scripts(): tmaster = tservers.TestMaster(config.ProxyConfig()) for f in scripts: + if "har_extractor" in f: + f += " foo" if "iframe_injector" in f: f += " foo" # one argument required if "modify_response_body" in f: f += " foo bar" # two arguments required - script.Script(f, tmaster) # Loads the script file.
\ No newline at end of file + s = script.Script(f, tmaster) # Loads the script file. + s.unload()
\ No newline at end of file diff --git a/test/test_flow.py b/test/test_flow.py index b74119dd..22abb4d4 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -5,8 +5,10 @@ import mock from libmproxy import filt, protocol, controller, utils, tnetstring, flow from libmproxy.protocol.primitives import Error, Flow from libmproxy.protocol.http import decoded, CONTENT_MISSING -from libmproxy.proxy.connection import ClientConnection, ServerConnection -from netlib import tcp +from libmproxy.proxy.config import HostMatcher +from libmproxy.proxy import ProxyConfig +from libmproxy.proxy.server import DummyServer +from libmproxy.proxy.connection import ClientConnection import tutils @@ -84,19 +86,20 @@ class TestClientPlaybackState: fm = flow.FlowMaster(None, s) fm.start_client_playback([first, tutils.tflow()], True) c = fm.client_playback + c.testing = True assert not c.done() assert not s.flow_count() assert c.count() == 2 - c.tick(fm, testing=True) + c.tick(fm) assert s.flow_count() assert c.count() == 1 - c.tick(fm, testing=True) + c.tick(fm) assert c.count() == 1 c.clear(c.current) - c.tick(fm, testing=True) + c.tick(fm) assert c.count() == 0 c.clear(c.current) assert c.done() @@ -531,6 +534,14 @@ class TestSerialize: fm.load_flows(r) assert len(s._flow_list) == 6 + def test_load_flows_reverse(self): + r = self._treader() + s = flow.State() + conf = ProxyConfig(mode="reverse", upstream_server=[True,True,"use-this-domain",80]) + fm = flow.FlowMaster(DummyServer(conf), s) + fm.load_flows(r) + assert s._flow_list[0].request.host == "use-this-domain" + def test_filter(self): sio = StringIO() fl = filt.parse("~c 200") @@ -584,11 +595,11 @@ class TestFlowMaster: def test_getset_ignore(self): p = mock.Mock() - p.config.ignore = [] + p.config.check_ignore = HostMatcher() fm = flow.FlowMaster(p, flow.State()) - assert not fm.get_ignore() - fm.set_ignore(["^apple\.com:", ":443$"]) - assert fm.get_ignore() + assert not fm.get_ignore_filter() + fm.set_ignore_filter(["^apple\.com:", ":443$"]) + assert fm.get_ignore_filter() def test_replay(self): s = flow.State() @@ -600,6 +611,9 @@ class TestFlowMaster: f.intercepting = True assert "intercepting" in fm.replay_request(f) + f.live = True + assert "live" in fm.replay_request(f) + def test_script_reqerr(self): s = flow.State() fm = flow.FlowMaster(None, s) @@ -679,9 +693,11 @@ class TestFlowMaster: f = tutils.tflow(resp=True) pb = [tutils.tflow(resp=True), f] - fm = flow.FlowMaster(None, s) + + fm = flow.FlowMaster(DummyServer(ProxyConfig()), s) assert not fm.start_server_playback(pb, False, [], False, False, None, False) assert not fm.start_client_playback(pb, False) + fm.client_playback.testing = True q = Queue.Queue() assert not fm.state.flow_count() diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index ea6cf3fd..16870777 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -23,7 +23,7 @@ def test_stripped_chunked_encoding_no_content(): class TestHTTPRequest: - def test_asterisk_form(self): + def test_asterisk_form_in(self): s = StringIO("OPTIONS * HTTP/1.1") f = tutils.tflow(req=None) f.request = HTTPRequest.from_stream(s) @@ -31,9 +31,11 @@ class TestHTTPRequest: f.request.host = f.server_conn.address.host f.request.port = f.server_conn.address.port f.request.scheme = "http" - assert f.request.assemble() == "OPTIONS * HTTP/1.1\r\nHost: address:22\r\n\r\n" + assert f.request.assemble() == ("OPTIONS * HTTP/1.1\r\n" + "Host: address:22\r\n" + "Content-Length: 0\r\n\r\n") - def test_origin_form(self): + def test_relative_form_in(self): s = StringIO("GET /foo\xff HTTP/1.1") tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s) s = StringIO("GET /foo HTTP/1.1\r\nConnection: Upgrade\r\nUpgrade: h2c") @@ -52,22 +54,47 @@ class TestHTTPRequest: r.update_host_header() assert "Host" in r.headers - - def test_authority_form(self): + def test_authority_form_in(self): s = StringIO("CONNECT oops-no-port.com HTTP/1.1") tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s) s = StringIO("CONNECT address:22 HTTP/1.1") r = HTTPRequest.from_stream(s) r.scheme, r.host, r.port = "http", "address", 22 - assert r.assemble() == "CONNECT address:22 HTTP/1.1\r\nHost: address:22\r\n\r\n" + assert r.assemble() == ("CONNECT address:22 HTTP/1.1\r\n" + "Host: address:22\r\n" + "Content-Length: 0\r\n\r\n") assert r.pretty_url(False) == "address:22" - def test_absolute_form(self): + def test_absolute_form_in(self): s = StringIO("GET oops-no-protocol.com HTTP/1.1") tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s) s = StringIO("GET http://address:22/ HTTP/1.1") r = HTTPRequest.from_stream(s) - assert r.assemble() == "GET http://address:22/ HTTP/1.1\r\nHost: address:22\r\n\r\n" + assert r.assemble() == "GET http://address:22/ HTTP/1.1\r\nHost: address:22\r\nContent-Length: 0\r\n\r\n" + + def test_http_options_relative_form_in(self): + """ + Exercises fix for Issue #392. + """ + s = StringIO("OPTIONS /secret/resource HTTP/1.1") + r = HTTPRequest.from_stream(s) + r.host = 'address' + r.port = 80 + r.scheme = "http" + assert r.assemble() == ("OPTIONS /secret/resource HTTP/1.1\r\n" + "Host: address\r\n" + "Content-Length: 0\r\n\r\n") + + def test_http_options_absolute_form_in(self): + s = StringIO("OPTIONS http://address/secret/resource HTTP/1.1") + r = HTTPRequest.from_stream(s) + r.host = 'address' + r.port = 80 + r.scheme = "http" + assert r.assemble() == ("OPTIONS http://address:80/secret/resource HTTP/1.1\r\n" + "Host: address\r\n" + "Content-Length: 0\r\n\r\n") + def test_assemble_unknown_form(self): r = tutils.treq() @@ -133,4 +160,4 @@ class TestInvalidRequests(tservers.HTTPProxTest): p.connect() r = p.request("get:/p/200") assert r.status_code == 400 - assert "Invalid HTTP request form" in r.content
\ No newline at end of file + assert "Invalid HTTP request form" in r.content diff --git a/test/test_proxy.py b/test/test_proxy.py index c396183b..641b4f47 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -70,9 +70,9 @@ class TestProcessProxyOptions: def test_simple(self): assert self.p() - def test_confdir(self): - with tutils.tmpdir() as confdir: - self.assert_noerr("--confdir", confdir) + def test_cadir(self): + with tutils.tmpdir() as cadir: + self.assert_noerr("--cadir", cadir) @mock.patch("libmproxy.platform.resolver", None) def test_no_transparent(self): @@ -94,12 +94,12 @@ class TestProcessProxyOptions: self.assert_err("mutually exclusive", "-R", "http://localhost", "-T") def test_client_certs(self): - with tutils.tmpdir() as confdir: - self.assert_noerr("--client-certs", confdir) + with tutils.tmpdir() as cadir: + self.assert_noerr("--client-certs", cadir) self.assert_err("directory does not exist", "--client-certs", "nonexistent") def test_certs(self): - with tutils.tmpdir() as confdir: + with tutils.tmpdir() as cadir: self.assert_noerr("--cert", tutils.test_data.path("data/testkey.pem")) self.assert_err("does not exist", "--cert", "nonexistent") diff --git a/test/test_server.py b/test/test_server.py index 0ce5d056..c81eab2b 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -1,5 +1,5 @@ import socket, time -from libmproxy.proxy.config import parse_host_pattern +from libmproxy.proxy.config import HostMatcher from netlib import tcp, http_auth, http from libpathod import pathoc, pathod from netlib.certutils import SSLCert @@ -19,6 +19,17 @@ class CommonMixin: def test_large(self): assert len(self.pathod("200:b@50k").content) == 1024*50 + @staticmethod + def wait_until_not_live(flow): + """ + Race condition: We don't want to replay the flow while it is still live. + """ + s = time.time() + while flow.live: + time.sleep(0.001) + if time.time() - s > 5: + raise RuntimeError("Flow is live for too long.") + def test_replay(self): assert self.pathod("304").status_code == 304 if isinstance(self, tservers.HTTPUpstreamProxTest) and self.ssl: @@ -28,6 +39,7 @@ class CommonMixin: l = self.master.state.view[-1] assert l.response.code == 304 l.request.path = "/p/305" + self.wait_until_not_live(l) rt = self.master.replay_request(l, block=True) assert l.response.code == 305 @@ -79,11 +91,14 @@ class CommonMixin: class TcpMixin: def _ignore_on(self): - ignore = parse_host_pattern([".+:%s" % self.server.port])[0] - self.config.ignore.append(ignore) + assert not hasattr(self, "_ignore_backup") + self._ignore_backup = self.config.check_ignore + self.config.check_ignore = HostMatcher([".+:%s" % self.server.port] + self.config.check_ignore.patterns) def _ignore_off(self): - self.config.ignore.pop() + assert hasattr(self, "_ignore_backup") + self.config.check_ignore = self._ignore_backup + del self._ignore_backup def test_ignore(self): spec = '304:h"Alternate-Protocol"="mitmproxy-will-remove-this"' @@ -114,6 +129,40 @@ class TcpMixin: tutils.raises("invalid server response", self.pathod, spec) # pathoc tries to parse answer as HTTP self._ignore_off() + def _tcpproxy_on(self): + assert not hasattr(self, "_tcpproxy_backup") + self._tcpproxy_backup = self.config.check_tcp + self.config.check_tcp = HostMatcher([".+:%s" % self.server.port] + self.config.check_tcp.patterns) + + def _tcpproxy_off(self): + assert hasattr(self, "_tcpproxy_backup") + self.config.check_ignore = self._tcpproxy_backup + del self._tcpproxy_backup + + + def test_tcp(self): + spec = '304:h"Alternate-Protocol"="mitmproxy-will-remove-this"' + n = self.pathod(spec) + self._tcpproxy_on() + i = self.pathod(spec) + i2 = self.pathod(spec) + self._tcpproxy_off() + + assert i.status_code == i2.status_code == n.status_code == 304 + assert "Alternate-Protocol" in i.headers + assert "Alternate-Protocol" in i2.headers + assert "Alternate-Protocol" not in n.headers + + # Test that we get the original SSL cert + if self.ssl: + i_cert = SSLCert(i.sslinfo.certchain[0]) + i2_cert = SSLCert(i2.sslinfo.certchain[0]) + n_cert = SSLCert(n.sslinfo.certchain[0]) + + assert i_cert == i2_cert == n_cert + + # Make sure that TCP messages are in the event log. + assert any("mitmproxy-will-remove-this" in m for m in self.master.log) class AppMixin: def test_app(self): @@ -579,16 +628,50 @@ class TestUpstreamProxy(tservers.HTTPUpstreamProxTest, CommonMixin, AppMixin): class TestUpstreamProxySSL(tservers.HTTPUpstreamProxTest, CommonMixin, TcpMixin): ssl = True + def _host_pattern_on(self, attr): + """ + Updates config.check_tcp or check_ignore, depending on attr. + """ + assert not hasattr(self, "_ignore_%s_backup" % attr) + backup = [] + for proxy in self.chain: + old_matcher = getattr(proxy.tmaster.server.config, "check_%s" % attr) + backup.append(old_matcher) + setattr( + proxy.tmaster.server.config, + "check_%s" % attr, + HostMatcher([".+:%s" % self.server.port] + old_matcher.patterns) + ) + + setattr(self, "_ignore_%s_backup" % attr, backup) + + def _host_pattern_off(self, attr): + backup = getattr(self, "_ignore_%s_backup" % attr) + for proxy in reversed(self.chain): + setattr( + proxy.tmaster.server.config, + "check_%s" % attr, + backup.pop() + ) + + assert not backup + delattr(self, "_ignore_%s_backup" % attr) + def _ignore_on(self): super(TestUpstreamProxySSL, self)._ignore_on() - ignore = parse_host_pattern([".+:%s" % self.server.port])[0] - for proxy in self.chain: - proxy.tmaster.server.config.ignore.append(ignore) + self._host_pattern_on("ignore") def _ignore_off(self): super(TestUpstreamProxySSL, self)._ignore_off() - for proxy in self.chain: - proxy.tmaster.server.config.ignore.pop() + self._host_pattern_off("ignore") + + def _tcpproxy_on(self): + super(TestUpstreamProxySSL, self)._tcpproxy_on() + self._host_pattern_on("tcp") + + def _tcpproxy_off(self): + super(TestUpstreamProxySSL, self)._tcpproxy_off() + self._host_pattern_off("tcp") def test_simple(self): p = self.pathoc() diff --git a/test/tools/passive_close.py b/test/tools/passive_close.py new file mode 100644 index 00000000..d0b36e7f --- /dev/null +++ b/test/tools/passive_close.py @@ -0,0 +1,21 @@ +import SocketServer +from threading import Thread +from time import sleep + +class service(SocketServer.BaseRequestHandler): + def handle(self): + data = 'dummy' + print "Client connected with ", self.client_address + while True: + self.request.send("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 7\r\n\r\ncontent") + data = self.request.recv(1024) + if not len(data): + print "Connection closed by remote: ", self.client_address + sleep(3600) + + +class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): + pass + +server = ThreadedTCPServer(('',1520), service) +server.serve_forever() diff --git a/test/tservers.py b/test/tservers.py index 93c8a80a..12154ba7 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -99,7 +99,7 @@ class ProxTestBase(object): @classmethod def teardownAll(cls): - shutil.rmtree(cls.confdir) + shutil.rmtree(cls.cadir) cls.proxy.shutdown() cls.server.shutdown() cls.server2.shutdown() @@ -116,10 +116,10 @@ class ProxTestBase(object): @classmethod def get_proxy_config(cls): - cls.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy") + cls.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy") return dict( no_upstream_cert = cls.no_upstream_cert, - confdir = cls.confdir, + cadir = cls.cadir, authenticator = cls.authenticator, certforward = cls.certforward, ssl_ports=([cls.server.port, cls.server2.port] if cls.ssl else []), |
