From 4bd992d63c3fd37a921ff48f06382bab2003a4e8 Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Mon, 17 Dec 2018 00:08:27 +0530 Subject: external-viewer-fix --- mitmproxy/tools/console/master.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index dd15a2f5..6ab9ba5a 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -120,7 +120,7 @@ class ConsoleMaster(master.Master): with open(fd, "w" if text else "wb") as f: f.write(data) # if no EDITOR is set, assume 'vi' - c = os.environ.get("EDITOR") or "vi" + c = os.environ.get("MITMPROXY_EDITOR") or os.environ.get("EDITOR") or "vi" cmd = shlex.split(c) cmd.append(name) with self.uistopped(): @@ -159,7 +159,7 @@ class ConsoleMaster(master.Master): shell = True if not cmd: # hm which one should get priority? - c = os.environ.get("PAGER") or os.environ.get("EDITOR") + c = os.environ.get("MITMPROXY_EDITOR") or os.environ.get("PAGER") or os.environ.get("EDITOR") if not c: c = "less" cmd = shlex.split(c) -- cgit v1.2.3 From 1ee54f7d739afb34063712b71db2bf892d1dd663 Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Mon, 7 Jan 2019 23:17:11 +0530 Subject: option added --- mitmproxy/tools/console/consoleaddons.py | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index a40cdeaa..ba14eeb6 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -113,6 +113,11 @@ class ConsoleAddon: "console_mouse", bool, True, "Console mouse interaction." ) + loader.add_option( + "console_external_viewer_default", str, "vi", + "External viewer for flow body. Set environment variable $MITMPROXY_EDITOR " + "to change default." + ) @command.command("console.layout.options") def layout_options(self) -> typing.Sequence[str]: -- cgit v1.2.3 From d027891cec67e190403fc4fa73f17d7a74f02720 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Sun, 13 Jan 2019 23:27:43 -0300 Subject: Fix command injection when exporting to curl The command generated by `export.clip curl @focus` or `export.file curl @focus /path/to/file` wasn't being properly escaped so it could contain a malicious command instead of just a simple curl. --- mitmproxy/addons/export.py | 25 ++++++++++++++++--------- test/mitmproxy/addons/test_export.py | 28 +++++++++++++++++++++++----- 2 files changed, 39 insertions(+), 14 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 90e95d3e..271fc49d 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -1,4 +1,5 @@ import typing +import shlex from mitmproxy import ctx from mitmproxy import command @@ -18,20 +19,26 @@ def raise_if_missing_request(f: flow.Flow) -> None: def curl_command(f: flow.Flow) -> str: raise_if_missing_request(f) - data = "curl " + args = ["curl"] request = f.request.copy() # type: ignore request.decode(strict=False) for k, v in request.headers.items(multi=True): - data += "-H '%s:%s' " % (k, v) + args += ["-H", shlex.quote("%s:%s" % (k, v))] if request.method != "GET": - data += "-X %s " % request.method - data += "'%s'" % request.url + args += ["-X", shlex.quote(request.method)] + args.append(shlex.quote(request.url)) if request.content: - data += " --data-binary '%s'" % strutils.bytes_to_escaped_str( - request.content, - escape_single_quotes=True - ) - return data + try: + content = strutils.always_str(request.content) + except UnicodeDecodeError: + # shlex.quote doesn't support a bytes object + # see https://github.com/python/cpython/pull/10871 + raise exceptions.CommandError("Request content must be valid unicode") + args += [ + "--data-binary", + shlex.quote(strutils.always_str(request.content)) + ] + return ' '.join(args) def httpie_command(f: flow.Flow) -> str: diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index f4bb0f64..5c365135 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -1,4 +1,5 @@ import os +import shlex import pytest import pyperclip @@ -47,23 +48,40 @@ def tcp_flow(): class TestExportCurlCommand: def test_get(self, get_request): - result = """curl -H 'header:qvalue' -H 'content-length:0' 'http://address:22/path?a=foo&a=bar&b=baz'""" + result = """curl -H header:qvalue -H content-length:0 'http://address:22/path?a=foo&a=bar&b=baz'""" assert export.curl_command(get_request) == result def test_post(self, post_request): - result = "curl -H 'content-length:256' -X POST 'http://address:22/path' --data-binary '{}'".format( - str(bytes(range(256)))[2:-1] - ) + post_request.request.content = b'nobinarysupport' + result = "curl -H content-length:15 -X POST http://address:22/path --data-binary nobinarysupport" assert export.curl_command(post_request) == result + def test_fails_with_binary_data(self, post_request): + # shlex.quote doesn't support a bytes object + # see https://github.com/python/cpython/pull/10871 + with pytest.raises(exceptions.CommandError): + export.curl_command(post_request) + def test_patch(self, patch_request): - result = """curl -H 'header:qvalue' -H 'content-length:7' -X PATCH 'http://address:22/path?query=param' --data-binary 'content'""" + result = """curl -H header:qvalue -H content-length:7 -X PATCH 'http://address:22/path?query=param' --data-binary content""" assert export.curl_command(patch_request) == result def test_tcp(self, tcp_flow): with pytest.raises(exceptions.CommandError): export.curl_command(tcp_flow) + def test_escape_single_quotes_in_body(self): + request = tflow.tflow( + req=tutils.treq( + method=b'POST', + headers=(), + content=b"'&#" + ) + ) + command = export.curl_command(request) + assert shlex.split(command)[-2] == '--data-binary' + assert shlex.split(command)[-1] == "'&#" + class TestExportHttpieCommand: def test_get(self, get_request): -- cgit v1.2.3 From eab4174b87c7ba0b7dab2c8d7e0b13253833abe8 Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Sun, 13 Jan 2019 23:45:28 -0300 Subject: Fix command injection when exporting to httpie The command generated by `export.clip httpie @focus` or `export.file httpie @focus /path/to/file` wasn't being properly escaped so it could contain a malicious command instead of just a simple httpie call. --- mitmproxy/addons/export.py | 20 ++++++++++++-------- test/mitmproxy/addons/test_export.py | 27 ++++++++++++++++++++++----- 2 files changed, 34 insertions(+), 13 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 271fc49d..761b3915 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -44,17 +44,21 @@ def curl_command(f: flow.Flow) -> str: def httpie_command(f: flow.Flow) -> str: raise_if_missing_request(f) request = f.request.copy() # type: ignore - data = "http %s " % request.method + args = ["http"] + args.append(shlex.quote(request.method)) request.decode(strict=False) - data += "%s" % request.url + args.append(shlex.quote(request.url)) for k, v in request.headers.items(multi=True): - data += " '%s:%s'" % (k, v) + args.append(shlex.quote("%s:%s" % (k, v))) if request.content: - data += " <<< '%s'" % strutils.bytes_to_escaped_str( - request.content, - escape_single_quotes=True - ) - return data + try: + content = strutils.always_str(request.content) + except UnicodeDecodeError: + # shlex.quote doesn't support a bytes object + # see https://github.com/python/cpython/pull/10871 + raise exceptions.CommandError("Request content must be valid unicode") + args += ["<<<", shlex.quote(content)] + return ' '.join(args) def raw(f: flow.Flow) -> bytes: diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index 5c365135..67d0fc99 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -85,23 +85,40 @@ class TestExportCurlCommand: class TestExportHttpieCommand: def test_get(self, get_request): - result = """http GET http://address:22/path?a=foo&a=bar&b=baz 'header:qvalue' 'content-length:0'""" + result = """http GET 'http://address:22/path?a=foo&a=bar&b=baz' header:qvalue content-length:0""" assert export.httpie_command(get_request) == result def test_post(self, post_request): - result = "http POST http://address:22/path 'content-length:256' <<< '{}'".format( - str(bytes(range(256)))[2:-1] - ) + post_request.request.content = b'nobinarysupport' + result = "http POST http://address:22/path content-length:15 <<< nobinarysupport" assert export.httpie_command(post_request) == result + def test_fails_with_binary_data(self, post_request): + # shlex.quote doesn't support a bytes object + # see https://github.com/python/cpython/pull/10871 + with pytest.raises(exceptions.CommandError): + export.httpie_command(post_request) + def test_patch(self, patch_request): - result = """http PATCH http://address:22/path?query=param 'header:qvalue' 'content-length:7' <<< 'content'""" + result = """http PATCH 'http://address:22/path?query=param' header:qvalue content-length:7 <<< content""" assert export.httpie_command(patch_request) == result def test_tcp(self, tcp_flow): with pytest.raises(exceptions.CommandError): export.httpie_command(tcp_flow) + def test_escape_single_quotes_in_body(self): + request = tflow.tflow( + req=tutils.treq( + method=b'POST', + headers=(), + content=b"'&#" + ) + ) + command = export.httpie_command(request) + assert shlex.split(command)[-2] == '<<<' + assert shlex.split(command)[-1] == "'&#" + class TestRaw: def test_get(self, get_request): -- cgit v1.2.3 From 674f92a7c164c538047c970fc42194211856276a Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Sun, 13 Jan 2019 23:53:51 -0300 Subject: Refactor curl_command and httpie_command To avoid calling to shlex.quote many times --- mitmproxy/addons/export.py | 21 ++++++++++----------- 1 file changed, 10 insertions(+), 11 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 761b3915..04c0349a 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -23,10 +23,10 @@ def curl_command(f: flow.Flow) -> str: request = f.request.copy() # type: ignore request.decode(strict=False) for k, v in request.headers.items(multi=True): - args += ["-H", shlex.quote("%s:%s" % (k, v))] + args += ["-H", "%s:%s" % (k, v)] if request.method != "GET": - args += ["-X", shlex.quote(request.method)] - args.append(shlex.quote(request.url)) + args += ["-X", request.method] + args.append(request.url) if request.content: try: content = strutils.always_str(request.content) @@ -36,20 +36,19 @@ def curl_command(f: flow.Flow) -> str: raise exceptions.CommandError("Request content must be valid unicode") args += [ "--data-binary", - shlex.quote(strutils.always_str(request.content)) + strutils.always_str(request.content) ] - return ' '.join(args) + return ' '.join(shlex.quote(arg) for arg in args) def httpie_command(f: flow.Flow) -> str: raise_if_missing_request(f) request = f.request.copy() # type: ignore - args = ["http"] - args.append(shlex.quote(request.method)) request.decode(strict=False) - args.append(shlex.quote(request.url)) + args = ["http", request.method, request.url] for k, v in request.headers.items(multi=True): - args.append(shlex.quote("%s:%s" % (k, v))) + args.append("%s:%s" % (k, v)) + cmd = ' '.join(shlex.quote(arg) for arg in args) if request.content: try: content = strutils.always_str(request.content) @@ -57,8 +56,8 @@ def httpie_command(f: flow.Flow) -> str: # shlex.quote doesn't support a bytes object # see https://github.com/python/cpython/pull/10871 raise exceptions.CommandError("Request content must be valid unicode") - args += ["<<<", shlex.quote(content)] - return ' '.join(args) + cmd += " <<< " + shlex.quote(content) + return cmd def raw(f: flow.Flow) -> bytes: -- cgit v1.2.3 From d852f292c9a45de7f45cc8537f2aef217259017e Mon Sep 17 00:00:00 2001 From: =?UTF-8?q?Mat=C3=ADas=20Lang?= Date: Mon, 14 Jan 2019 02:02:07 -0300 Subject: Fix flake8 warning in curl_command There was an unused variable (I redefined its value below). It didn't affect the behavior of the function --- mitmproxy/addons/export.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 04c0349a..a7152ae4 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -34,10 +34,7 @@ def curl_command(f: flow.Flow) -> str: # shlex.quote doesn't support a bytes object # see https://github.com/python/cpython/pull/10871 raise exceptions.CommandError("Request content must be valid unicode") - args += [ - "--data-binary", - strutils.always_str(request.content) - ] + args += ["--data-binary", content] return ' '.join(shlex.quote(arg) for arg in args) -- cgit v1.2.3 From 4df325335b4abcdea6d59314ebfc96e7465a3979 Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Fri, 14 Dec 2018 21:31:34 +0530 Subject: multipart-fix --- mitmproxy/net/http/request.py | 3 ++- mitmproxy/tools/console/grideditor/editors.py | 12 ++++++++++-- test/mitmproxy/net/http/test_request.py | 5 +++-- 3 files changed, 15 insertions(+), 5 deletions(-) diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 959fdd33..218699e0 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -468,7 +468,8 @@ class Request(message.Message): return () def _set_multipart_form(self, value): - raise NotImplementedError() + self.headers["content-type"] = "multipart/form-data" + self.content = mitmproxy.net.http.url.encode(value, self.get_text(strict=False)).encode() @property def multipart_form(self): diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index 61fcf6b4..21cc8159 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -54,16 +54,24 @@ class ResponseHeaderEditor(HeaderEditor): class RequestFormEditor(base.FocusEditor): - title = "Edit URL-encoded Form" + title = "Edit Form" columns = [ col_text.Column("Key"), col_text.Column("Value") ] def get_data(self, flow): - return flow.request.urlencoded_form.items(multi=True) + + if "application/x-www-form-urlencoded" in flow.request.headers['Content-Type']: + return flow.request.urlencoded_form.items(multi=True) + + return flow.request.multipart_form.items(multi=True) def set_data(self, vals, flow): + + if "multipart/form-data" in flow.request.headers['Content-Type']: + flow.request.multipart_form = vals + flow.request.urlencoded_form = vals diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py index ef581a91..6ef73389 100644 --- a/test/mitmproxy/net/http/test_request.py +++ b/test/mitmproxy/net/http/test_request.py @@ -372,5 +372,6 @@ class TestRequestUtils: def test_set_multipart_form(self): request = treq(content=b"foobar") - with pytest.raises(NotImplementedError): - request.multipart_form = "foobar" + request.multipart_form = [("filename", "shell.jpg"), ("file_size", "1000")] + assert request.headers['Content-Type'] == "multipart/form-data" + assert request.content -- cgit v1.2.3 From d08d2185eab0d58eef7a2b32d557475e51acb61a Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Tue, 29 Jan 2019 22:35:01 +0530 Subject: multipart encoder and tests --- mitmproxy/net/http/multipart.py | 55 +++++++++++++++++++++++++------ mitmproxy/net/http/request.py | 2 +- test/mitmproxy/net/http/test_multipart.py | 18 ++++++++++ test/mitmproxy/net/http/test_request.py | 8 ++--- 4 files changed, 68 insertions(+), 15 deletions(-) diff --git a/mitmproxy/net/http/multipart.py b/mitmproxy/net/http/multipart.py index a854d47f..4edf76ac 100644 --- a/mitmproxy/net/http/multipart.py +++ b/mitmproxy/net/http/multipart.py @@ -1,8 +1,43 @@ import re - +import mimetypes +from urllib.parse import quote from mitmproxy.net.http import headers +def encode(head, l): + + k = head.get("content-type") + if k: + k = headers.parse_content_type(k) + if k is not None: + try: + boundary = k[2]["boundary"].encode("ascii") + boundary = quote(boundary) + except (KeyError, UnicodeError): + return b"" + hdrs = [] + for key, value in l: + file_type = mimetypes.guess_type(str(key))[0] or "text/plain; charset=utf-8" + + if key: + hdrs.append(b"--%b" % boundary.encode('utf-8')) + disposition = b'form-data; name="%b"' % key + hdrs.append(b"Content-Disposition: %b" % disposition) + hdrs.append(b"Content-Type: %b" % file_type.encode('utf-8')) + hdrs.append(b'') + hdrs.append(value) + hdrs.append(b'') + + if value is not None: + # If boundary is found in value then raise ValueError + if re.search(rb"^--%b$" % re.escape(boundary.encode('utf-8')), value): + raise ValueError(b"boundary found in encoded string") + + hdrs.append(b"--%b--\r\n" % boundary.encode('utf-8')) + temp = b"\r\n".join(hdrs) + return temp + + def decode(hdrs, content): """ Takes a multipart boundary encoded string and returns list of (key, value) tuples. @@ -19,14 +54,14 @@ def decode(hdrs, content): rx = re.compile(br'\bname="([^"]+)"') r = [] - - for i in content.split(b"--" + boundary): - parts = i.splitlines() - if len(parts) > 1 and parts[0][0:2] != b"--": - match = rx.search(parts[1]) - if match: - key = match.group(1) - value = b"".join(parts[3 + parts[2:].index(b""):]) - r.append((key, value)) + if content is not None: + for i in content.split(b"--" + boundary): + parts = i.splitlines() + if len(parts) > 1 and parts[0][0:2] != b"--": + match = rx.search(parts[1]) + if match: + key = match.group(1) + value = b"".join(parts[3 + parts[2:].index(b""):]) + r.append((key, value)) return r return [] diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 218699e0..783fd5ff 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -468,8 +468,8 @@ class Request(message.Message): return () def _set_multipart_form(self, value): + self.content = mitmproxy.net.http.multipart.encode(self.headers, value) self.headers["content-type"] = "multipart/form-data" - self.content = mitmproxy.net.http.url.encode(value, self.get_text(strict=False)).encode() @property def multipart_form(self): diff --git a/test/mitmproxy/net/http/test_multipart.py b/test/mitmproxy/net/http/test_multipart.py index 68ae6bbd..ce7dee5a 100644 --- a/test/mitmproxy/net/http/test_multipart.py +++ b/test/mitmproxy/net/http/test_multipart.py @@ -1,5 +1,6 @@ from mitmproxy.net.http import Headers from mitmproxy.net.http import multipart +import pytest def test_decode(): @@ -22,3 +23,20 @@ def test_decode(): assert len(form) == 2 assert form[0] == (b"field1", b"value1") assert form[1] == (b"field2", b"value2") + + +def test_encode(): + data = [("file".encode('utf-8'), "shell.jpg".encode('utf-8')), + ("file_size".encode('utf-8'), "1000".encode('utf-8'))] + headers = Headers( + content_type='multipart/form-data; boundary=127824672498' + ) + content = multipart.encode(headers, data) + + assert b'Content-Disposition: form-data; name="file"' in content + assert b'Content-Type: text/plain; charset=utf-8\r\n\r\nshell.jpg\r\n\r\n--127824672498\r\n' in content + assert b'1000\r\n\r\n--127824672498--\r\n' + assert len(content) == 252 + + with pytest.raises(ValueError, match=r"boundary found in encoded string"): + multipart.encode(headers, [("key".encode('utf-8'), "--127824672498".encode('utf-8'))]) diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py index 6ef73389..71d5c7a1 100644 --- a/test/mitmproxy/net/http/test_request.py +++ b/test/mitmproxy/net/http/test_request.py @@ -371,7 +371,7 @@ class TestRequestUtils: assert list(request.multipart_form.items()) == [] def test_set_multipart_form(self): - request = treq(content=b"foobar") - request.multipart_form = [("filename", "shell.jpg"), ("file_size", "1000")] - assert request.headers['Content-Type'] == "multipart/form-data" - assert request.content + request = treq() + request.multipart_form = [("file", "shell.jpg"), ("file_size", "1000")] + assert request.headers["Content-Type"] == 'multipart/form-data' + assert request.content is None -- cgit v1.2.3 From 8948703470fdb4b46adf901b2b0918fd74c4df1f Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Wed, 30 Jan 2019 16:22:16 +0530 Subject: separate editors --- mitmproxy/tools/console/consoleaddons.py | 9 ++++++--- mitmproxy/tools/console/grideditor/editors.py | 22 +++++++++++++++------- mitmproxy/tools/console/window.py | 3 ++- 3 files changed, 23 insertions(+), 11 deletions(-) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index a40cdeaa..58f236c0 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -368,7 +368,8 @@ class ConsoleAddon: """ return [ "cookies", - "form", + "urlencoded form", + "multipart form", "path", "method", "query", @@ -403,8 +404,10 @@ class ConsoleAddon: flow.response = http.HTTPResponse.make() if part == "cookies": self.master.switch_view("edit_focus_cookies") - elif part == "form": - self.master.switch_view("edit_focus_form") + elif part == "urlencoded form": + self.master.switch_view("edit_focus_urlencoded_form") + elif part == "multipart form": + self.master.switch_view("edit_focus_multipart_form") elif part == "path": self.master.switch_view("edit_focus_path") elif part == "query": diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index 21cc8159..09666d58 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -53,8 +53,8 @@ class ResponseHeaderEditor(HeaderEditor): flow.response.headers = Headers(vals) -class RequestFormEditor(base.FocusEditor): - title = "Edit Form" +class RequestMultipartEditor(base.FocusEditor): + title = "Edit Multipart Form" columns = [ col_text.Column("Key"), col_text.Column("Value") @@ -62,16 +62,24 @@ class RequestFormEditor(base.FocusEditor): def get_data(self, flow): - if "application/x-www-form-urlencoded" in flow.request.headers['Content-Type']: - return flow.request.urlencoded_form.items(multi=True) - return flow.request.multipart_form.items(multi=True) def set_data(self, vals, flow): + flow.request.multipart_form = vals + + +class RequestUrlEncodedEditor(base.FocusEditor): + title = "Edit UrlEncoded Form" + columns = [ + col_text.Column("Key"), + col_text.Column("Value") + ] - if "multipart/form-data" in flow.request.headers['Content-Type']: - flow.request.multipart_form = vals + def get_data(self, flow): + return flow.request.urlencoded_form.items(multi=True) + + def set_data(self, vals, flow): flow.request.urlencoded_form = vals diff --git a/mitmproxy/tools/console/window.py b/mitmproxy/tools/console/window.py index 7669299c..fb2e8c1e 100644 --- a/mitmproxy/tools/console/window.py +++ b/mitmproxy/tools/console/window.py @@ -64,7 +64,8 @@ class WindowStack: edit_focus_cookies = grideditor.CookieEditor(master), edit_focus_setcookies = grideditor.SetCookieEditor(master), edit_focus_setcookie_attrs = grideditor.CookieAttributeEditor(master), - edit_focus_form = grideditor.RequestFormEditor(master), + edit_focus_multipart_form=grideditor.RequestMultipartEditor(master), + edit_focus_urlencoded_form=grideditor.RequestUrlEncodedEditor(master), edit_focus_path = grideditor.PathEditor(master), edit_focus_request_headers = grideditor.RequestHeaderEditor(master), edit_focus_response_headers = grideditor.ResponseHeaderEditor(master), -- cgit v1.2.3 From 580ba356adf4e11241725005eb79d47f3468e092 Mon Sep 17 00:00:00 2001 From: rjt-gupta Date: Wed, 6 Feb 2019 03:35:36 +0530 Subject: test coverage improved --- test/mitmproxy/net/http/test_multipart.py | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/test/mitmproxy/net/http/test_multipart.py b/test/mitmproxy/net/http/test_multipart.py index ce7dee5a..6d2e5017 100644 --- a/test/mitmproxy/net/http/test_multipart.py +++ b/test/mitmproxy/net/http/test_multipart.py @@ -24,6 +24,18 @@ def test_decode(): assert form[0] == (b"field1", b"value1") assert form[1] == (b"field2", b"value2") + boundary = 'boundary茅莽' + headers = Headers( + content_type='multipart/form-data; boundary=' + boundary + ) + result = multipart.decode(headers, content) + assert result == [] + + headers = Headers( + content_type='' + ) + assert multipart.decode(headers, content) == [] + def test_encode(): data = [("file".encode('utf-8'), "shell.jpg".encode('utf-8')), @@ -40,3 +52,10 @@ def test_encode(): with pytest.raises(ValueError, match=r"boundary found in encoded string"): multipart.encode(headers, [("key".encode('utf-8'), "--127824672498".encode('utf-8'))]) + + boundary = 'boundary茅莽' + headers = Headers( + content_type='multipart/form-data; boundary=' + boundary + ) + result = multipart.encode(headers, data) + assert result == b'' -- cgit v1.2.3 From 141ade5ae10d7a6c1ae687f9c320fdee8df035fa Mon Sep 17 00:00:00 2001 From: peter-way <2475625231@qq.com> Date: Mon, 15 Apr 2019 08:43:52 +0800 Subject: Use 'host_header' instead of 'host', when calculating 'HTTPRequest' hash in transparent mode. --- mitmproxy/addons/serverplayback.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 51ba60b4..29f2ef5d 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -128,7 +128,10 @@ class ServerPlayback: key.append(str(r.raw_content)) if not ctx.options.server_replay_ignore_host: - key.append(r.host) + if ctx.options.mode == "transparent": + key.append(r.host_header) + else: + key.append(r.host) filtered = [] ignore_params = ctx.options.server_replay_ignore_params or [] -- cgit v1.2.3 From 3eebfed79f4d54840a054c2dc5061e155c416d3e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 17 Jul 2019 22:46:39 +0200 Subject: Update serverplayback.py --- mitmproxy/addons/serverplayback.py | 5 +---- 1 file changed, 1 insertion(+), 4 deletions(-) diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 29f2ef5d..e3192a4c 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -128,10 +128,7 @@ class ServerPlayback: key.append(str(r.raw_content)) if not ctx.options.server_replay_ignore_host: - if ctx.options.mode == "transparent": - key.append(r.host_header) - else: - key.append(r.host) + key.append(r.pretty_host) filtered = [] ignore_params = ctx.options.server_replay_ignore_params or [] -- cgit v1.2.3 From 0d042c6d8eb89691b7f731ccfd3d4bbaa27e3d48 Mon Sep 17 00:00:00 2001 From: cs Date: Fri, 9 Aug 2019 14:19:10 +0800 Subject: fix duplicate error response --- mitmproxy/proxy/protocol/http.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index 2ae656b3..4c20617b 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -263,7 +263,7 @@ class HttpLayer(base.Layer): else: msg = "Unexpected CONNECT request." self.send_error_response(400, msg) - raise exceptions.ProtocolException(msg) + return False validate_request_form(self.mode, request) self.channel.ask("requestheaders", f) @@ -289,9 +289,12 @@ class HttpLayer(base.Layer): f.request = None f.error = flow.Error(str(e)) self.channel.ask("error", f) - raise exceptions.ProtocolException( - "HTTP protocol error in client request: {}".format(e) - ) from e + self.log( + "request", + "warn", + ["HTTP protocol error in client request: {}".format(e)] + ) + return False self.log("request", "debug", [repr(request)]) @@ -448,8 +451,8 @@ class HttpLayer(base.Layer): return False # should never be reached except (exceptions.ProtocolException, exceptions.NetlibException) as e: - self.send_error_response(502, repr(e)) if not f.response: + self.send_error_response(502, repr(e)) f.error = flow.Error(str(e)) self.channel.ask("error", f) return False -- cgit v1.2.3 From 2239c49e18c62331a21aa75fa5e74e6a7ea88334 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 15 Nov 2019 02:27:33 +0100 Subject: improve flowfilter --- mitmproxy/flowfilter.py | 112 +++++++++++++++++++++--------------------------- 1 file changed, 48 insertions(+), 64 deletions(-) diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index 3f5afb48..b222d2a8 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -32,19 +32,17 @@ rex Equivalent to ~u rex """ +import functools import re import sys -import functools +from typing import Callable, ClassVar, Optional, Sequence, Type + +import pyparsing as pp +from mitmproxy import flow from mitmproxy import http -from mitmproxy import websocket from mitmproxy import tcp -from mitmproxy import flow - -from mitmproxy.utils import strutils - -import pyparsing as pp -from typing import Callable, Sequence, Type, Optional, ClassVar +from mitmproxy import websocket def only(*types): @@ -54,7 +52,9 @@ def only(*types): if isinstance(flow, types): return fn(self, flow) return False + return filter_types + return decorator @@ -146,10 +146,10 @@ class _Rex(_Action): def __init__(self, expr): self.expr = expr if self.is_binary: - expr = strutils.escaped_str_to_bytes(expr) + expr = expr.encode() try: self.re = re.compile(expr, self.flags) - except: + except Exception: raise ValueError("Cannot compile expression.") @@ -336,6 +336,7 @@ class FUrl(_Rex): code = "u" help = "URL" is_binary = False + # FUrl is special, because it can be "naked". @classmethod @@ -469,68 +470,51 @@ def _make(): # Order is important - multi-char expressions need to come before narrow # ones. parts = [] - for klass in filter_unary: - f = pp.Literal("~%s" % klass.code) + pp.WordEnd() - f.setParseAction(klass.make) + for cls in filter_unary: + f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + f.setParseAction(cls.make) parts.append(f) - simplerex = "".join(c for c in pp.printables if c not in "()~'\"") - alphdevanagari = pp.pyparsing_unicode.Devanagari.alphas - alphcyrillic = pp.pyparsing_unicode.Cyrillic.alphas - alphgreek = pp.pyparsing_unicode.Greek.alphas - alphchinese = pp.pyparsing_unicode.Chinese.alphas - alpharabic = pp.pyparsing_unicode.Arabic.alphas - alphhebrew = pp.pyparsing_unicode.Hebrew.alphas - alphjapanese = pp.pyparsing_unicode.Japanese.alphas - alphkorean = pp.pyparsing_unicode.Korean.alphas - alphlatin1 = pp.pyparsing_unicode.Latin1.alphas - alphlatinA = pp.pyparsing_unicode.LatinA.alphas - alphlatinB = pp.pyparsing_unicode.LatinB.alphas - - rex = pp.Word(simplerex) |\ - pp.Word(alphcyrillic) |\ - pp.Word(alphgreek) |\ - pp.Word(alphchinese) |\ - pp.Word(alpharabic) |\ - pp.Word(alphdevanagari) |\ - pp.Word(alphhebrew) |\ - pp.Word(alphjapanese) |\ - pp.Word(alphkorean) |\ - pp.Word(alphlatin1) |\ - pp.Word(alphlatinA) |\ - pp.Word(alphlatinB) |\ - pp.QuotedString("\"", escChar='\\') |\ - pp.QuotedString("'", escChar='\\') - for klass in filter_rex: - f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + rex.copy() - f.setParseAction(klass.make) + # This is a bit of a hack to simulate Word(pyparsing_unicode.printables), + # which has a horrible performance with len(pyparsing.pyparsing_unicode.printables) == 1114060 + unicode_words = pp.CharsNotIn("()~'\"" + pp.ParserElement.DEFAULT_WHITE_CHARS) + unicode_words.skipWhitespace = True + regex = ( + unicode_words + | pp.QuotedString('"', escChar='\\') + | pp.QuotedString("'", escChar='\\') + ) + for cls in filter_rex: + f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + regex.copy() + f.setParseAction(cls.make) parts.append(f) - for klass in filter_int: - f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + pp.Word(pp.nums) - f.setParseAction(klass.make) + for cls in filter_int: + f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + pp.Word(pp.nums) + f.setParseAction(cls.make) parts.append(f) # A naked rex is a URL rex: - f = rex.copy() + f = regex.copy() f.setParseAction(FUrl.make) parts.append(f) atom = pp.MatchFirst(parts) - expr = pp.operatorPrecedence(atom, - [(pp.Literal("!").suppress(), - 1, - pp.opAssoc.RIGHT, - lambda x: FNot(*x)), - (pp.Literal("&").suppress(), - 2, - pp.opAssoc.LEFT, - lambda x: FAnd(*x)), - (pp.Literal("|").suppress(), - 2, - pp.opAssoc.LEFT, - lambda x: FOr(*x)), - ]) + expr = pp.infixNotation( + atom, + [(pp.Literal("!").suppress(), + 1, + pp.opAssoc.RIGHT, + lambda x: FNot(*x)), + (pp.Literal("&").suppress(), + 2, + pp.opAssoc.LEFT, + lambda x: FAnd(*x)), + (pp.Literal("|").suppress(), + 2, + pp.opAssoc.LEFT, + lambda x: FOr(*x)), + ]) expr = pp.OneOrMore(expr) return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x) @@ -570,15 +554,15 @@ def match(flt, flow): help = [] for a in filter_unary: help.append( - ("~%s" % a.code, a.help) + (f"~{a.code}", a.help) ) for b in filter_rex: help.append( - ("~%s regex" % b.code, b.help) + (f"~{b.code} regex", b.help) ) for c in filter_int: help.append( - ("~%s int" % c.code, c.help) + (f"~{c.code} int", c.help) ) help.sort() help.extend( -- cgit v1.2.3 From b3dac1318410c72ea61d9c2c92bb19e68db377fd Mon Sep 17 00:00:00 2001 From: Jesson Soto Ventura Date: Thu, 31 Oct 2019 09:51:50 -0400 Subject: ctrl-c should prompt for an exit --- mitmproxy/tools/_main.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py index a00a3e98..11a87327 100644 --- a/mitmproxy/tools/_main.py +++ b/mitmproxy/tools/_main.py @@ -11,6 +11,7 @@ import signal import typing from mitmproxy.tools import cmdline +from mitmproxy.tools.console.master import ConsoleMaster from mitmproxy import exceptions, master from mitmproxy import options from mitmproxy import optmanager @@ -114,7 +115,10 @@ def run( loop = asyncio.get_event_loop() for signame in ('SIGINT', 'SIGTERM'): try: - loop.add_signal_handler(getattr(signal, signame), master.shutdown) + if isinstance(master, ConsoleMaster): + loop.add_signal_handler(getattr(signal, signame), master.prompt_for_exit) + else: + loop.add_signal_handler(getattr(signal, signame), master.shutdown) except NotImplementedError: # Not supported on Windows pass -- cgit v1.2.3 From be46008b5ef0d71b65b8f4dab7528b4bcc7205e6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 15 Nov 2019 15:56:55 +0100 Subject: disable overly strict indentation checks --- setup.cfg | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.cfg b/setup.cfg index 0a34b805..e7643b08 100644 --- a/setup.cfg +++ b/setup.cfg @@ -1,7 +1,7 @@ [flake8] max-line-length = 140 max-complexity = 25 -ignore = E251,E252,C901,W292,W503,W504,W605,E722,E741 +ignore = E251,E252,C901,W292,W503,W504,W605,E722,E741,E126 exclude = mitmproxy/contrib/*,test/mitmproxy/data/*,release/build/*,mitmproxy/io/proto/* addons = file,open,basestring,xrange,unicode,long,cmp -- cgit v1.2.3 From a5412ab1366205cb1a2000993cb6b2c9dca28246 Mon Sep 17 00:00:00 2001 From: Jurriaan Bremer Date: Sun, 29 Apr 2018 16:53:11 +0200 Subject: allow server replay functionality to run on a different port By providing the "server_replay_ignore_port" configuration value we're able to run mitmproxy in server replay mode on a different port than the web server in the flows was originally running. This is also useful in case multiple ports are present in a flow (I suspect). --- mitmproxy/addons/serverplayback.py | 11 ++++++++++- 1 file changed, 10 insertions(+), 1 deletion(-) diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 51ba60b4..0818696f 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -68,6 +68,13 @@ class ServerPlayback: to replay. """ ) + loader.add_option( + "server_replay_ignore_port", bool, False, + """ + Ignore request's destination port while searching for a saved flow + to replay. + """ + ) @command.command("replay.server") def load_flows(self, flows: typing.Sequence[flow.Flow]) -> None: @@ -110,7 +117,7 @@ class ServerPlayback: _, _, path, _, query, _ = urllib.parse.urlparse(r.url) queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) - key: typing.List[typing.Any] = [str(r.port), str(r.scheme), str(r.method), str(path)] + key: typing.List[typing.Any] = [str(r.scheme), str(r.method), str(path)] if not ctx.options.server_replay_ignore_content: if ctx.options.server_replay_ignore_payload_params and r.multipart_form: key.extend( @@ -129,6 +136,8 @@ class ServerPlayback: if not ctx.options.server_replay_ignore_host: key.append(r.host) + if not ctx.options.server_replay_ignore_port: + key.append(r.port) filtered = [] ignore_params = ctx.options.server_replay_ignore_params or [] -- cgit v1.2.3 From f8926170a5be481ff4cb2dd181ead95b0ec081cb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 15 Nov 2019 17:22:41 +0100 Subject: remove superfluous option --- mitmproxy/tools/console/consoleaddons.py | 5 ----- 1 file changed, 5 deletions(-) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index ba14eeb6..a40cdeaa 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -113,11 +113,6 @@ class ConsoleAddon: "console_mouse", bool, True, "Console mouse interaction." ) - loader.add_option( - "console_external_viewer_default", str, "vi", - "External viewer for flow body. Set environment variable $MITMPROXY_EDITOR " - "to change default." - ) @command.command("console.layout.options") def layout_options(self) -> typing.Sequence[str]: -- cgit v1.2.3 From 01ddda75e8aaab542d2c11b9bc9218a94a342f10 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 15 Nov 2019 19:02:59 +0100 Subject: improve curl/httpie export --- mitmproxy/addons/export.py | 73 ++++++++++++++++++++---------------- test/mitmproxy/addons/test_export.py | 16 ++++---- 2 files changed, 49 insertions(+), 40 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 20dac655..638a1e2e 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -1,47 +1,60 @@ -import typing import shlex +import typing + +import pyperclip -from mitmproxy import ctx +import mitmproxy.types from mitmproxy import command -from mitmproxy import flow +from mitmproxy import ctx, http from mitmproxy import exceptions -from mitmproxy.utils import strutils +from mitmproxy import flow from mitmproxy.net.http.http1 import assemble -import mitmproxy.types - -import pyperclip +from mitmproxy.utils import strutils -def cleanup_request(f: flow.Flow): +def cleanup_request(f: flow.Flow) -> http.HTTPRequest: if not hasattr(f, "request"): raise exceptions.CommandError("Can't export flow with no request.") - request = f.request.copy() # type: ignore + assert isinstance(f, http.HTTPFlow) + request = f.request.copy() request.decode(strict=False) - # a bit of clean-up - if request.method == 'GET' and request.headers.get("content-length", None) == "0": - request.headers.pop('content-length') - request.headers.pop(':authority', None) + # a bit of clean-up - these headers should be automatically set by curl/httpie + request.headers.pop('content-length') + if request.headers.get("host", "") == request.host: + request.headers.pop("host") + if request.headers.get(":authority", "") == request.host: + request.headers.pop(":authority") return request +def request_content_for_console(request: http.HTTPRequest) -> str: + try: + text = request.get_text(strict=True) + except ValueError: + # shlex.quote doesn't support a bytes object + # see https://github.com/python/cpython/pull/10871 + raise exceptions.CommandError("Request content must be valid unicode") + escape_control_chars = {chr(i): f"\\x{i:02x}" for i in range(32)} + return "".join( + escape_control_chars.get(x, x) + for x in text + ) + + def curl_command(f: flow.Flow) -> str: request = cleanup_request(f) args = ["curl"] for k, v in request.headers.items(multi=True): - args += ["--compressed "] if k == 'accept-encoding' else [] - args += ["-H", "%s:%s" % (k, v)] + if k.lower() == "accept-encoding": + args.append("--compressed") + else: + args += ["-H", f"{k}: {v}"] if request.method != "GET": args += ["-X", request.method] args.append(request.url) if request.content: - try: - content = strutils.always_str(request.content) - except UnicodeDecodeError: - # shlex.quote doesn't support a bytes object - # see https://github.com/python/cpython/pull/10871 - raise exceptions.CommandError("Request content must be valid unicode") - args += ["--data-binary", content] + args += ["-d", request_content_for_console(request)] return ' '.join(shlex.quote(arg) for arg in args) @@ -49,16 +62,10 @@ def httpie_command(f: flow.Flow) -> str: request = cleanup_request(f) args = ["http", request.method, request.url] for k, v in request.headers.items(multi=True): - args.append("%s:%s" % (k, v)) + args.append(f"{k}: {v}") cmd = ' '.join(shlex.quote(arg) for arg in args) if request.content: - try: - content = strutils.always_str(request.content) - except UnicodeDecodeError: - # shlex.quote doesn't support a bytes object - # see https://github.com/python/cpython/pull/10871 - raise exceptions.CommandError("Request content must be valid unicode") - cmd += " <<< " + shlex.quote(content) + cmd += " <<< " + shlex.quote(request_content_for_console(request)) return cmd @@ -67,9 +74,9 @@ def raw(f: flow.Flow) -> bytes: formats = dict( - curl = curl_command, - httpie = httpie_command, - raw = raw, + curl=curl_command, + httpie=httpie_command, + raw=raw, ) diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index a4c10c1a..1c16afb2 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -42,22 +42,23 @@ def tcp_flow(): class TestExportCurlCommand: def test_get(self, get_request): - result = """curl -H header:qvalue 'http://address:22/path?a=foo&a=bar&b=baz'""" + result = """curl -H 'header: qvalue' 'http://address:22/path?a=foo&a=bar&b=baz'""" assert export.curl_command(get_request) == result def test_post(self, post_request): post_request.request.content = b'nobinarysupport' - result = "curl -H content-length:15 -X POST http://address:22/path --data-binary nobinarysupport" + result = "curl -X POST http://address:22/path -d nobinarysupport" assert export.curl_command(post_request) == result def test_fails_with_binary_data(self, post_request): # shlex.quote doesn't support a bytes object # see https://github.com/python/cpython/pull/10871 + post_request.request.headers["Content-Type"] = "application/json; charset=utf-8" with pytest.raises(exceptions.CommandError): export.curl_command(post_request) def test_patch(self, patch_request): - result = """curl -H header:qvalue -H content-length:7 -X PATCH 'http://address:22/path?query=param' --data-binary content""" + result = """curl -H 'header: qvalue' -X PATCH 'http://address:22/path?query=param' -d content""" assert export.curl_command(patch_request) == result def test_tcp(self, tcp_flow): @@ -73,28 +74,29 @@ class TestExportCurlCommand: ) ) command = export.curl_command(request) - assert shlex.split(command)[-2] == '--data-binary' + assert shlex.split(command)[-2] == '-d' assert shlex.split(command)[-1] == "'&#" class TestExportHttpieCommand: def test_get(self, get_request): - result = """http GET 'http://address:22/path?a=foo&a=bar&b=baz' header:qvalue""" + result = """http GET 'http://address:22/path?a=foo&a=bar&b=baz' 'header: qvalue'""" assert export.httpie_command(get_request) == result def test_post(self, post_request): post_request.request.content = b'nobinarysupport' - result = "http POST http://address:22/path content-length:15 <<< nobinarysupport" + result = "http POST http://address:22/path <<< nobinarysupport" assert export.httpie_command(post_request) == result def test_fails_with_binary_data(self, post_request): # shlex.quote doesn't support a bytes object # see https://github.com/python/cpython/pull/10871 + post_request.request.headers["Content-Type"] = "application/json; charset=utf-8" with pytest.raises(exceptions.CommandError): export.httpie_command(post_request) def test_patch(self, patch_request): - result = """http PATCH 'http://address:22/path?query=param' header:qvalue content-length:7 <<< content""" + result = """http PATCH 'http://address:22/path?query=param' 'header: qvalue' <<< content""" assert export.httpie_command(patch_request) == result def test_tcp(self, tcp_flow): -- cgit v1.2.3 From 484e099eb10897b89d83d4b0441b4455faab9422 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 15 Nov 2019 20:57:03 +0100 Subject: test coverage++ --- mitmproxy/addons/export.py | 1 + test/mitmproxy/addons/test_export.py | 8 ++++++++ 2 files changed, 9 insertions(+) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 638a1e2e..80413ac9 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -30,6 +30,7 @@ def cleanup_request(f: flow.Flow) -> http.HTTPRequest: def request_content_for_console(request: http.HTTPRequest) -> str: try: text = request.get_text(strict=True) + assert text except ValueError: # shlex.quote doesn't support a bytes object # see https://github.com/python/cpython/pull/10871 diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index 1c16afb2..4b905722 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -77,6 +77,14 @@ class TestExportCurlCommand: assert shlex.split(command)[-2] == '-d' assert shlex.split(command)[-1] == "'&#" + def test_strip_unnecessary(self, get_request): + get_request.request.headers.clear() + get_request.request.headers["host"] = "address" + get_request.request.headers[":authority"] = "address" + get_request.request.headers["accept-encoding"] = "br" + result = """curl --compressed 'http://address:22/path?a=foo&a=bar&b=baz'""" + assert export.curl_command(get_request) == result + class TestExportHttpieCommand: def test_get(self, get_request): -- cgit v1.2.3 From 8f2cee722592ac100a4b159d2945044331afcfbc Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 15 Nov 2019 21:00:47 +0100 Subject: fix #3469 --- mitmproxy/addons/clientplayback.py | 20 +++++++------- mitmproxy/addons/serverplayback.py | 55 +++++++++++++++++++++++++------------- 2 files changed, 46 insertions(+), 29 deletions(-) diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 7bdaeb33..7adefd7a 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -1,23 +1,23 @@ import queue import threading -import typing import time +import typing -from mitmproxy import log +import mitmproxy.types +from mitmproxy import command +from mitmproxy import connections from mitmproxy import controller +from mitmproxy import ctx from mitmproxy import exceptions -from mitmproxy import http from mitmproxy import flow +from mitmproxy import http +from mitmproxy import io +from mitmproxy import log from mitmproxy import options -from mitmproxy import connections +from mitmproxy.coretypes import basethread from mitmproxy.net import server_spec, tls from mitmproxy.net.http import http1 -from mitmproxy.coretypes import basethread from mitmproxy.utils import human -from mitmproxy import ctx -from mitmproxy import io -from mitmproxy import command -import mitmproxy.types class RequestReplayThread(basethread.BaseThread): @@ -117,7 +117,7 @@ class RequestReplayThread(basethread.BaseThread): finally: r.first_line_format = first_line_format_backup f.live = False - if server.connected(): + if server and server.connected(): server.finish() server.close() diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 0818696f..f2e381b2 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -1,16 +1,19 @@ import hashlib -import urllib import typing +import urllib -from mitmproxy import ctx -from mitmproxy import flow +import mitmproxy.types +from mitmproxy import command +from mitmproxy import ctx, http from mitmproxy import exceptions +from mitmproxy import flow from mitmproxy import io -from mitmproxy import command -import mitmproxy.types class ServerPlayback: + flowmap: typing.Dict[typing.Hashable, typing.List[http.HTTPFlow]] + configured: bool + def __init__(self): self.flowmap = {} self.configured = False @@ -82,10 +85,10 @@ class ServerPlayback: Replay server responses from flows. """ self.flowmap = {} - for i in flows: - if i.response: # type: ignore - l = self.flowmap.setdefault(self._hash(i), []) - l.append(i) + for f in flows: + if isinstance(f, http.HTTPFlow): + lst = self.flowmap.setdefault(self._hash(f), []) + lst.append(f) ctx.master.addons.trigger("update", []) @command.command("replay.server.file") @@ -108,12 +111,11 @@ class ServerPlayback: def count(self) -> int: return sum([len(i) for i in self.flowmap.values()]) - def _hash(self, flow): + def _hash(self, flow: http.HTTPFlow) -> typing.Hashable: """ Calculates a loose hash of the flow request. """ r = flow.request - _, _, path, _, query, _ = urllib.parse.urlparse(r.url) queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) @@ -158,20 +160,33 @@ class ServerPlayback: repr(key).encode("utf8", "surrogateescape") ).digest() - def next_flow(self, request): + def next_flow(self, flow: http.HTTPFlow) -> typing.Optional[http.HTTPFlow]: """ Returns the next flow object, or None if no matching flow was found. """ - hsh = self._hash(request) - if hsh in self.flowmap: + request = flow.request + hash = self._hash(flow) + if hash in self.flowmap: if ctx.options.server_replay_nopop: - return self.flowmap[hsh][0] + return next(( + flow + for flow in self.flowmap[hash] + if flow.response + ), None) else: - ret = self.flowmap[hsh].pop(0) - if not self.flowmap[hsh]: - del self.flowmap[hsh] + ret = self.flowmap[hash].pop(0) + while not ret.response: + if self.flowmap[hash]: + ret = self.flowmap[hash].pop(0) + else: + del self.flowmap[hash] + return None + if not self.flowmap[hash]: + del self.flowmap[hash] return ret + else: + return None def configure(self, updated): if not self.configured and ctx.options.server_replay: @@ -182,10 +197,11 @@ class ServerPlayback: raise exceptions.OptionsError(str(e)) self.load_flows(flows) - def request(self, f): + def request(self, f: http.HTTPFlow) -> None: if self.flowmap: rflow = self.next_flow(f) if rflow: + assert rflow.response response = rflow.response.copy() response.is_replay = True if ctx.options.server_replay_refresh: @@ -197,4 +213,5 @@ class ServerPlayback: f.request.url ) ) + assert f.reply f.reply.kill() -- cgit v1.2.3 From 248034c528c158c95e87f5a888176ab14e11a6dc Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 15 Nov 2019 21:17:29 +0100 Subject: tests++ --- mitmproxy/addons/serverplayback.py | 1 - test/mitmproxy/addons/test_serverplayback.py | 34 +++++++++++++++++++++++----- 2 files changed, 28 insertions(+), 7 deletions(-) diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index f2e381b2..0e1c2309 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -165,7 +165,6 @@ class ServerPlayback: Returns the next flow object, or None if no matching flow was found. """ - request = flow.request hash = self._hash(flow) if hash in self.flowmap: if ctx.options.server_replay_nopop: diff --git a/test/mitmproxy/addons/test_serverplayback.py b/test/mitmproxy/addons/test_serverplayback.py index c6a0c1f4..2e42fa03 100644 --- a/test/mitmproxy/addons/test_serverplayback.py +++ b/test/mitmproxy/addons/test_serverplayback.py @@ -1,13 +1,13 @@ import urllib -import pytest -from mitmproxy.test import taddons -from mitmproxy.test import tflow +import pytest import mitmproxy.test.tutils -from mitmproxy.addons import serverplayback from mitmproxy import exceptions from mitmproxy import io +from mitmproxy.addons import serverplayback +from mitmproxy.test import taddons +from mitmproxy.test import tflow def tdump(path, flows): @@ -321,7 +321,7 @@ def test_server_playback_full(): with taddons.context(s) as tctx: tctx.configure( s, - server_replay_refresh = True, + server_replay_refresh=True, ) f = tflow.tflow() @@ -345,7 +345,7 @@ def test_server_playback_kill(): with taddons.context(s) as tctx: tctx.configure( s, - server_replay_refresh = True, + server_replay_refresh=True, server_replay_kill_extra=True ) @@ -357,3 +357,25 @@ def test_server_playback_kill(): f.request.host = "nonexistent" tctx.cycle(s, f) assert f.reply.value == exceptions.Kill + + +def test_server_playback_response_deleted(): + """ + The server playback addon holds references to flows that can be modified by the user in the meantime. + One thing that can happen is that users remove the response object. This happens for example when doing a client + replay at the same time. + """ + sp = serverplayback.ServerPlayback() + with taddons.context(sp) as tctx: + tctx.configure(sp) + f1 = tflow.tflow(resp=True) + f2 = tflow.tflow(resp=True) + + assert not sp.flowmap + + sp.load_flows([f1, f2]) + assert sp.flowmap + + f1.response = f2.response = None + assert not sp.next_flow(f1) + assert not sp.flowmap -- cgit v1.2.3 From 5cb1746ef667ff4a4c1426c959d6ce6a34d74540 Mon Sep 17 00:00:00 2001 From: Jesson Soto Ventura Date: Fri, 15 Nov 2019 23:22:38 -0500 Subject: used getattr to select exit --- mitmproxy/tools/_main.py | 6 +----- 1 file changed, 1 insertion(+), 5 deletions(-) diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py index 11a87327..0163e8d3 100644 --- a/mitmproxy/tools/_main.py +++ b/mitmproxy/tools/_main.py @@ -11,7 +11,6 @@ import signal import typing from mitmproxy.tools import cmdline -from mitmproxy.tools.console.master import ConsoleMaster from mitmproxy import exceptions, master from mitmproxy import options from mitmproxy import optmanager @@ -115,10 +114,7 @@ def run( loop = asyncio.get_event_loop() for signame in ('SIGINT', 'SIGTERM'): try: - if isinstance(master, ConsoleMaster): - loop.add_signal_handler(getattr(signal, signame), master.prompt_for_exit) - else: - loop.add_signal_handler(getattr(signal, signame), master.shutdown) + loop.add_signal_handler(getattr(signal, signame), getattr(master, "prompt_for_exit", master.shutdown)) except NotImplementedError: # Not supported on Windows pass -- cgit v1.2.3