diff options
| -rwxr-xr-x | dev | 5 | ||||
| -rw-r--r-- | dev.bat | 3 | ||||
| -rw-r--r-- | examples/README | 1 | ||||
| -rw-r--r-- | examples/flowwriter.py | 20 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 33 | ||||
| -rw-r--r-- | test/test_examples.py | 2 | ||||
| -rw-r--r-- | test/test_protocol_http.py | 22 |
7 files changed, 79 insertions, 7 deletions
@@ -1,7 +1,8 @@ -#!/bin/sh +#!/bin/bash +set -e VENV=../venv.mitmproxy -virtualenv $VENV +python2 -m virtualenv $VENV source $VENV/bin/activate pip install --src .. -r requirements.txt @@ -2,8 +2,11 @@ set VENV=..\venv.mitmproxy virtualenv %VENV% +if %errorlevel% neq 0 exit /b %errorlevel% call %VENV%\Scripts\activate.bat +if %errorlevel% neq 0 exit /b %errorlevel% pip install --src .. -r requirements.txt +if %errorlevel% neq 0 exit /b %errorlevel% echo. echo * Created virtualenv environment in %VENV%. diff --git a/examples/README b/examples/README index 4a5c268d..adfcd0f2 100644 --- a/examples/README +++ b/examples/README @@ -4,6 +4,7 @@ change_upstream_proxy.py Dynamically change the upstream proxy dns_spoofing.py Use mitmproxy in a DNS spoofing scenario. dup_and_replay.py Duplicates each request, changes it, and then replays the modified request. filt.py Use mitmproxy's filter expressions in your script. +flowwriter.py Only write selected flows into a mitmproxy dumpfile. iframe_injector.py Inject configurable iframe into pages. modify_form.py Modify all form submissions to add a parameter. modify_querystring.py Modify all query strings to add a parameters. diff --git a/examples/flowwriter.py b/examples/flowwriter.py new file mode 100644 index 00000000..f411ec45 --- /dev/null +++ b/examples/flowwriter.py @@ -0,0 +1,20 @@ +import random +import sys + +from libmproxy.flow import FlowWriter + + +def start(context, argv): + if len(argv) != 2: + raise ValueError('Usage: -s "flowriter.py filename"') + + if argv[1] == "-": + f = sys.stdout + else: + f = open(argv[1], "wb") + context.flow_writer = FlowWriter(f) + + +def response(context, flow): + if random.choice([True, False]): + context.flow_writer.add(flow)
\ No newline at end of file diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 496b71cc..94077e42 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -328,9 +328,22 @@ class HTTPRequest(HTTPMessage): ) @classmethod - def from_stream(cls, rfile, include_body=True, body_size_limit=None): + def from_stream(cls, rfile, include_body=True, body_size_limit=None, wfile=None): """ Parse an HTTP request from a file stream + + Args: + rfile (file): Input file to read from + include_body (bool): Read response body as well + body_size_limit (bool): Maximum body size + wfile (file): If specified, HTTP Expect headers are handled automatically. + by writing a HTTP 100 CONTINUE response to the stream. + + Returns: + HTTPRequest: The HTTP request + + Raises: + HttpError: If the input is invalid. """ httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end = ( None, None, None, None, None, None, None, None, None, None) @@ -385,6 +398,15 @@ class HTTPRequest(HTTPMessage): if headers is None: raise http.HttpError(400, "Invalid headers") + expect_header = headers.get_first("expect") + if expect_header and expect_header.lower() == "100-continue" and httpversion >= (1, 1): + wfile.write( + 'HTTP/1.1 100 Continue\r\n' + '\r\n' + ) + wfile.flush() + del headers['expect'] + if include_body: content = http.read_http_body(rfile, headers, body_size_limit, method, None, True) @@ -609,8 +631,10 @@ class HTTPRequest(HTTPMessage): host = self.headers.get_first("host") if not host: host = self.host - host = host.encode("idna") - return host + if host: + return host.encode("idna") + else: + return None def pretty_url(self, hostheader): if self.form_out == "authority": # upstream proxy mode @@ -1062,7 +1086,8 @@ class HTTPHandler(ProtocolHandler): try: req = HTTPRequest.from_stream( self.c.client_conn.rfile, - body_size_limit=self.c.config.body_size_limit + body_size_limit=self.c.config.body_size_limit, + wfile=self.c.client_conn.wfile ) except tcp.NetLibError: # don't throw an error for disconnects that happen diff --git a/test/test_examples.py b/test/test_examples.py index daf4b902..fd901b5d 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -11,7 +11,7 @@ def test_load_scripts(): tmaster = tservers.TestMaster(config.ProxyConfig()) for f in scripts: - if "har_extractor" in f: + if "har_extractor" in f or "flowwriter" in f: f += " -" if "iframe_injector" in f: f += " foo" # one argument required diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index 08ed114c..6b949ce3 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -59,6 +59,14 @@ class TestHTTPRequest: r.update_host_header() assert "Host" in r.headers + def test_expect_header(self): + s = StringIO("GET / HTTP/1.1\r\nContent-Length: 3\r\nExpect: 100-continue\r\n\r\nfoobar") + w = StringIO() + r = HTTPRequest.from_stream(s, wfile=w) + assert w.getvalue() == "HTTP/1.1 100 Continue\r\n\r\n" + assert r.content == "foo" + assert s.read(3) == "bar" + def test_authority_form_in(self): s = StringIO("CONNECT oops-no-port.com HTTP/1.1") tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s) @@ -117,6 +125,20 @@ class TestHTTPRequest: r = tutils.treq() assert repr(r) + def test_pretty_host(self): + r = tutils.treq() + assert r.pretty_host(True) == "address" + assert r.pretty_host(False) == "address" + r.headers["host"] = ["other"] + assert r.pretty_host(True) == "other" + assert r.pretty_host(False) == "address" + r.host = None + assert r.pretty_host(True) == "other" + assert r.pretty_host(False) is None + del r.headers["host"] + assert r.pretty_host(True) is None + assert r.pretty_host(False) is None + def test_get_form_for_urlencoded(self): r = tutils.treq() r.headers.add("content-type", "application/x-www-form-urlencoded") |