From 33707403614642ae63d7ed24f36c80a3f58ce7ac Mon Sep 17 00:00:00 2001 From: Yoann L Date: Mon, 28 Oct 2019 17:51:59 +0100 Subject: several fixes on command exports has several problems: #3676 * authority can usually rely on actual URL. as `:authority` headers will break curl command. (advise if it's better to change them to Host, or if it should be reported on curl side) * `content-length`: 0 is added for each request. if it's found in the curl argument list, it'll try to fetch an empty body (and crash). also trying to guess on accept-encoding header to add the `--compress` option when fetching potentially compressed content. * ditto for httpie --- mitmproxy/addons/export.py | 15 +++++++++++++++ test/mitmproxy/addons/test_export.py | 24 +++++++++--------------- 2 files changed, 24 insertions(+), 15 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 90e95d3e..0c10f352 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -11,6 +11,15 @@ import mitmproxy.types import pyperclip +def cleanup_for_commands(request): + if ('content-length' in request.headers.keys() and + request.headers['content-length'] == '0' and + request.method == 'GET'): + request.headers.pop('content-length') + request.headers.pop(':authority', None) + return request + + def raise_if_missing_request(f: flow.Flow) -> None: if not hasattr(f, "request"): raise exceptions.CommandError("Can't export flow with no request.") @@ -21,10 +30,15 @@ def curl_command(f: flow.Flow) -> str: data = "curl " request = f.request.copy() # type: ignore request.decode(strict=False) + request = cleanup_for_commands(request) for k, v in request.headers.items(multi=True): data += "-H '%s:%s' " % (k, v) if request.method != "GET": data += "-X %s " % request.method + for header in request.headers.keys(): + if ('accept-encoding' == header.lower() and + any(comp in request.headers[header] for comp in ['gzip', 'deflate'])): + data += "--compressed " data += "'%s'" % request.url if request.content: data += " --data-binary '%s'" % strutils.bytes_to_escaped_str( @@ -39,6 +53,7 @@ def httpie_command(f: flow.Flow) -> str: request = f.request.copy() # type: ignore data = "http %s " % request.method request.decode(strict=False) + request = cleanup_for_commands(request) data += "%s" % request.url for k, v in request.headers.items(multi=True): data += " '%s:%s'" % (k, v) diff --git a/test/mitmproxy/addons/test_export.py b/test/mitmproxy/addons/test_export.py index f4bb0f64..c86e0c7d 100644 --- a/test/mitmproxy/addons/test_export.py +++ b/test/mitmproxy/addons/test_export.py @@ -14,29 +14,23 @@ from unittest import mock @pytest.fixture def get_request(): return tflow.tflow( - req=tutils.treq( - method=b'GET', - content=b'', - path=b"/path?a=foo&a=bar&b=baz" - ) - ) + req=tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz")) @pytest.fixture def post_request(): return tflow.tflow( - req=tutils.treq( - method=b'POST', - headers=(), - content=bytes(range(256)) - ) - ) + req=tutils.treq(method=b'POST', headers=(), content=bytes(range(256)))) @pytest.fixture def patch_request(): return tflow.tflow( - req=tutils.treq(method=b'PATCH', path=b"/path?query=param") + req=tutils.treq( + method=b'PATCH', + content=b'content', + path=b"/path?query=param" + ) ) @@ -47,7 +41,7 @@ def tcp_flow(): class TestExportCurlCommand: def test_get(self, get_request): - result = """curl -H 'header:qvalue' -H 'content-length:0' 'http://address:22/path?a=foo&a=bar&b=baz'""" + result = """curl -H 'header:qvalue' 'http://address:22/path?a=foo&a=bar&b=baz'""" assert export.curl_command(get_request) == result def test_post(self, post_request): @@ -67,7 +61,7 @@ class TestExportCurlCommand: class TestExportHttpieCommand: def test_get(self, get_request): - result = """http GET http://address:22/path?a=foo&a=bar&b=baz 'header:qvalue' 'content-length:0'""" + result = """http GET http://address:22/path?a=foo&a=bar&b=baz 'header:qvalue'""" assert export.httpie_command(get_request) == result def test_post(self, post_request): -- cgit v1.2.3 From 4383122b7b6930612ba865c6d027eb375ecce058 Mon Sep 17 00:00:00 2001 From: Yoann L Date: Tue, 29 Oct 2019 12:24:07 +0100 Subject: several fixes on command exports has several problems: #3676 * authority can usually rely on actual URL. as `:authority` headers will break curl command. (advise if it's better to change them to Host, or if it should be reported on curl side) * `content-length`: 0 is added for each request. if it's found in the curl argument list, it'll try to fetch an empty body (and crash). also trying to guess on accept-encoding header to add the `--compress` option when fetching potentially compressed content. * ditto for httpie * factorize it into raise_if_missing_request (and rename it) --- mitmproxy/addons/export.py | 33 +++++++++++---------------------- 1 file changed, 11 insertions(+), 22 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 0c10f352..744db0fc 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -11,7 +11,12 @@ import mitmproxy.types import pyperclip -def cleanup_for_commands(request): +def cleanup_request(f: flow.Flow): + if not hasattr(f, "request"): + raise exceptions.CommandError("Can't export flow with no request.") + request = f.request.copy() + request.decode(strict=False) + # a bit of clean-up if ('content-length' in request.headers.keys() and request.headers['content-length'] == '0' and request.method == 'GET'): @@ -20,25 +25,14 @@ def cleanup_for_commands(request): return request -def raise_if_missing_request(f: flow.Flow) -> None: - if not hasattr(f, "request"): - raise exceptions.CommandError("Can't export flow with no request.") - - def curl_command(f: flow.Flow) -> str: - raise_if_missing_request(f) data = "curl " - request = f.request.copy() # type: ignore - request.decode(strict=False) - request = cleanup_for_commands(request) + request = cleanup_request(f) for k, v in request.headers.items(multi=True): + data += "--compressed " if k == 'accept-encoding' else "" data += "-H '%s:%s' " % (k, v) if request.method != "GET": data += "-X %s " % request.method - for header in request.headers.keys(): - if ('accept-encoding' == header.lower() and - any(comp in request.headers[header] for comp in ['gzip', 'deflate'])): - data += "--compressed " data += "'%s'" % request.url if request.content: data += " --data-binary '%s'" % strutils.bytes_to_escaped_str( @@ -49,12 +43,8 @@ def curl_command(f: flow.Flow) -> str: def httpie_command(f: flow.Flow) -> str: - raise_if_missing_request(f) - request = f.request.copy() # type: ignore - data = "http %s " % request.method - request.decode(strict=False) - request = cleanup_for_commands(request) - data += "%s" % request.url + request = cleanup_request(f) + data = "http %s %s" % (request.method, request.url) for k, v in request.headers.items(multi=True): data += " '%s:%s'" % (k, v) if request.content: @@ -66,8 +56,7 @@ def httpie_command(f: flow.Flow) -> str: def raw(f: flow.Flow) -> bytes: - raise_if_missing_request(f) - return assemble.assemble_request(f.request) # type: ignore + return assemble.assemble_request(cleanup_request(f)) # type: ignore formats = dict( -- cgit v1.2.3 From 6da1db750c7fc105d1d312fe6520e1b1f9d32567 Mon Sep 17 00:00:00 2001 From: Yoann L Date: Tue, 29 Oct 2019 12:38:11 +0100 Subject: missed a type ignore, for mypy compliance --- mitmproxy/addons/export.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index 744db0fc..cc698f9c 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -14,7 +14,7 @@ import pyperclip def cleanup_request(f: flow.Flow): if not hasattr(f, "request"): raise exceptions.CommandError("Can't export flow with no request.") - request = f.request.copy() + request = f.request.copy() # type: ignore request.decode(strict=False) # a bit of clean-up if ('content-length' in request.headers.keys() and -- cgit v1.2.3 From fcccab684d61d9c428fd9b862db14dd9da5a0e38 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 29 Oct 2019 16:15:07 +0100 Subject: simplify condition --- mitmproxy/addons/export.py | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py index cc698f9c..2776118a 100644 --- a/mitmproxy/addons/export.py +++ b/mitmproxy/addons/export.py @@ -17,9 +17,7 @@ def cleanup_request(f: flow.Flow): request = f.request.copy() # type: ignore request.decode(strict=False) # a bit of clean-up - if ('content-length' in request.headers.keys() and - request.headers['content-length'] == '0' and - request.method == 'GET'): + if request.method == 'GET' and request.headers.get("content-length", None) == "0": request.headers.pop('content-length') request.headers.pop(':authority', None) return request -- cgit v1.2.3