diff options
| -rw-r--r-- | docs/certinstall.rst | 2 | ||||
| -rw-r--r-- | mitmproxy/addons/stickycookie.py | 39 | ||||
| -rw-r--r-- | mitmproxy/export.py | 29 | ||||
| -rw-r--r-- | test/mitmproxy/addons/test_dumper.py | 1 | ||||
| -rw-r--r-- | test/mitmproxy/addons/test_stickycookie.py | 4 | ||||
| -rw-r--r-- | test/mitmproxy/data/test_flow_export/locust_task_post.py | 2 | ||||
| -rw-r--r-- | test/mitmproxy/data/test_flow_export/python_post.py | 13 | ||||
| -rw-r--r-- | test/mitmproxy/test_export.py | 145 | ||||
| -rw-r--r-- | web/src/js/__tests__/flow/utilsSpec.js | 69 | ||||
| -rw-r--r-- | web/src/js/__tests__/utilsSpec.js | 95 |
10 files changed, 295 insertions, 104 deletions
diff --git a/docs/certinstall.rst b/docs/certinstall.rst index 1bd6df99..14d66d30 100644 --- a/docs/certinstall.rst +++ b/docs/certinstall.rst @@ -132,7 +132,7 @@ mitmproxy-ca-cert.cer Same file as .pem, but with an extension expected by some Using a custom certificate -------------------------- -You can use your own certificate by passing the ``--cert [domain=]path_to_certificate`` option to +You can use your own (leaf) certificate by passing the ``--cert [domain=]path_to_certificate`` option to mitmproxy. Mitmproxy then uses the provided certificate for interception of the specified domain instead of generating a certificate signed by its own CA. diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index 04d99975..e58e0a58 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -1,14 +1,14 @@ import collections from http import cookiejar +from typing import List, Tuple, Dict, Optional # noqa +from mitmproxy import http, flowfilter, ctx, exceptions from mitmproxy.net.http import cookies -from mitmproxy import exceptions -from mitmproxy import flowfilter -from mitmproxy import ctx +TOrigin = Tuple[str, int, str] -def ckey(attrs, f): +def ckey(attrs: Dict[str, str], f: http.HTTPFlow) -> TOrigin: """ Returns a (domain, port, path) tuple. """ @@ -21,18 +21,18 @@ def ckey(attrs, f): return (domain, f.request.port, path) -def domain_match(a, b): - if cookiejar.domain_match(a, b): +def domain_match(a: str, b: str) -> bool: + if cookiejar.domain_match(a, b): # type: ignore return True - elif cookiejar.domain_match(a, b.strip(".")): + elif cookiejar.domain_match(a, b.strip(".")): # type: ignore return True return False class StickyCookie: def __init__(self): - self.jar = collections.defaultdict(dict) - self.flt = None + self.jar = collections.defaultdict(dict) # type: Dict[TOrigin, Dict[str, str]] + self.flt = None # type: Optional[flowfilter.TFilter] def configure(self, updated): if "stickycookie" in updated: @@ -46,7 +46,7 @@ class StickyCookie: else: self.flt = None - def response(self, flow): + def response(self, flow: http.HTTPFlow): if self.flt: for name, (value, attrs) in flow.response.cookies.items(multi=True): # FIXME: We now know that Cookie.py screws up some cookies with @@ -63,24 +63,21 @@ class StickyCookie: if not self.jar[dom_port_path]: self.jar.pop(dom_port_path, None) else: - b = attrs.copy() - b.insert(0, name, value) - self.jar[dom_port_path][name] = b + self.jar[dom_port_path][name] = value - def request(self, flow): + def request(self, flow: http.HTTPFlow): if self.flt: - l = [] + cookie_list = [] # type: List[Tuple[str,str]] if flowfilter.match(self.flt, flow): - for domain, port, path in self.jar.keys(): + for (domain, port, path), c in self.jar.items(): match = [ domain_match(flow.request.host, domain), flow.request.port == port, flow.request.path.startswith(path) ] if all(match): - c = self.jar[(domain, port, path)] - l.extend([cookies.format_cookie_header(c[name].items(multi=True)) for name in c.keys()]) - if l: + cookie_list.extend(c.items()) + if cookie_list: # FIXME: we need to formalise this... - flow.request.stickycookie = True - flow.request.headers["cookie"] = "; ".join(l) + flow.metadata["stickycookie"] = True + flow.request.headers["cookie"] = cookies.format_cookie_header(cookie_list) diff --git a/mitmproxy/export.py b/mitmproxy/export.py index 235e754a..efa08874 100644 --- a/mitmproxy/export.py +++ b/mitmproxy/export.py @@ -6,19 +6,7 @@ import textwrap from typing import Any from mitmproxy import http - - -def _native(s): - if isinstance(s, bytes): - return s.decode() - return s - - -def dictstr(items, indent: str) -> str: - lines = [] - for k, v in items: - lines.append(indent + "%s: %s,\n" % (repr(_native(k)), repr(_native(v)))) - return "{\n%s}\n" % "".join(lines) +from mitmproxy.utils import strutils def curl_command(flow: http.HTTPFlow) -> str: @@ -36,7 +24,10 @@ def curl_command(flow: http.HTTPFlow) -> str: data += "'%s'" % request.url if request.content: - data += " --data-binary '%s'" % _native(request.content) + data += " --data-binary '%s'" % strutils.bytes_to_escaped_str( + request.content, + escape_single_quotes=True + ) return data @@ -127,10 +118,14 @@ def locust_code(flow): args = "" headers = "" + + def conv(x): + return strutils.bytes_to_escaped_str(x, escape_single_quotes=True) + if flow.request.headers: lines = [ - (_native(k), _native(v)) for k, v in flow.request.headers.fields - if _native(k).lower() not in [":authority", "host", "cookie"] + (conv(k), conv(v)) for k, v in flow.request.headers.fields + if conv(k).lower() not in [":authority", "host", "cookie"] ] lines = [" '%s': '%s',\n" % (k, v) for k, v in lines] headers += "\n headers = {\n%s }\n" % "".join(lines) @@ -148,7 +143,7 @@ def locust_code(flow): data = "" if flow.request.content: - data = "\n data = '''%s'''\n" % _native(flow.request.content) + data = "\n data = '''%s'''\n" % conv(flow.request.content) args += "\n data=data," code = code.format( diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index d2cefe79..d8aa593b 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -68,7 +68,6 @@ def test_simple(): ctx.configure(d, flow_detail=4) flow = tflow.tflow() flow.request = tutils.treq() - flow.request.stickycookie = True flow.client_conn = mock.MagicMock() flow.client_conn.address[0] = "foo" flow.response = tutils.tresp(content=None) diff --git a/test/mitmproxy/addons/test_stickycookie.py b/test/mitmproxy/addons/test_stickycookie.py index 9092e09b..f77d019d 100644 --- a/test/mitmproxy/addons/test_stickycookie.py +++ b/test/mitmproxy/addons/test_stickycookie.py @@ -110,8 +110,8 @@ class TestStickyCookie: f.response.headers["Set-Cookie"] = c2 sc.response(f) googlekey = list(sc.jar.keys())[0] - assert len(sc.jar[googlekey].keys()) == 1 - assert list(sc.jar[googlekey]["somecookie"].items())[0][1] == "newvalue" + assert len(sc.jar[googlekey]) == 1 + assert sc.jar[googlekey]["somecookie"] == "newvalue" def test_response_delete(self): sc = stickycookie.StickyCookie() diff --git a/test/mitmproxy/data/test_flow_export/locust_task_post.py b/test/mitmproxy/data/test_flow_export/locust_task_post.py index 989df455..a5f307ee 100644 --- a/test/mitmproxy/data/test_flow_export/locust_task_post.py +++ b/test/mitmproxy/data/test_flow_export/locust_task_post.py @@ -2,7 +2,7 @@ def path(self): url = self.locust.host + '/path' - data = '''content''' + data = '''\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff''' self.response = self.client.request( method='POST', diff --git a/test/mitmproxy/data/test_flow_export/python_post.py b/test/mitmproxy/data/test_flow_export/python_post.py index 6254adfb..42f1af9a 100644 --- a/test/mitmproxy/data/test_flow_export/python_post.py +++ b/test/mitmproxy/data/test_flow_export/python_post.py @@ -2,7 +2,16 @@ import requests response = requests.post( 'http://address:22/path', - data=b'content' + data=(b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13' + b'\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./01234567' + b'89:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f' + b'\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f' + b'\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f' + b'\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf' + b'\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf' + b'\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf' + b'\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf' + b'\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef' + b'\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff') ) - print(response.text) diff --git a/test/mitmproxy/test_export.py b/test/mitmproxy/test_export.py index 457d8836..b789e6b5 100644 --- a/test/mitmproxy/test_export.py +++ b/test/mitmproxy/test_export.py @@ -1,13 +1,15 @@ -from mitmproxy.test import tflow import re -from mitmproxy.net.http import Headers +import pytest + from mitmproxy import export # heh +from mitmproxy.net.http import Headers +from mitmproxy.test import tflow from mitmproxy.test import tutils def clean_blanks(s): - return re.sub(r"^(\s+)$", "", s, flags=re.MULTILINE) + return re.sub(r"^\s+", "", s, flags=re.MULTILINE) def python_equals(testdata, text): @@ -19,85 +21,110 @@ def python_equals(testdata, text): assert clean_blanks(text).rstrip() == clean_blanks(d).rstrip() -def req_get(): - return tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz") +@pytest.fixture +def get_request(): + return tflow.tflow( + req=tutils.treq( + method=b'GET', + content=b'', + path=b"/path?a=foo&a=bar&b=baz" + ) + ) + + +@pytest.fixture +def post_request(): + return tflow.tflow( + req=tutils.treq( + method=b'POST', + headers=(), + content=bytes(range(256)) + ) + ) + + +@pytest.fixture +def patch_request(): + return tflow.tflow( + req=tutils.treq(method=b'PATCH', path=b"/path?query=param") + ) -def req_post(): - return tutils.treq(method=b'POST', headers=()) +class TExport: + def test_get(self, get_request): + raise NotImplementedError() + def test_post(self, post_request): + raise NotImplementedError() -def req_patch(): - return tutils.treq(method=b'PATCH', path=b"/path?query=param") + def test_patch(self, patch_request): + raise NotImplementedError() -class TestExportCurlCommand: - def test_get(self): - flow = tflow.tflow(req=req_get()) +class TestExportCurlCommand(TExport): + def test_get(self, get_request): result = """curl -H 'header:qvalue' -H 'content-length:7' 'http://address:22/path?a=foo&a=bar&b=baz'""" - assert export.curl_command(flow) == result + assert export.curl_command(get_request) == result - def test_post(self): - flow = tflow.tflow(req=req_post()) - result = """curl -X POST 'http://address:22/path' --data-binary 'content'""" - assert export.curl_command(flow) == result + def test_post(self, post_request): + result = "curl -X POST 'http://address:22/path' --data-binary '{}'".format( + str(bytes(range(256)))[2:-1] + ) + assert export.curl_command(post_request) == result - def test_patch(self): - flow = tflow.tflow(req=req_patch()) + def test_patch(self, patch_request): result = """curl -H 'header:qvalue' -H 'content-length:7' -X PATCH 'http://address:22/path?query=param' --data-binary 'content'""" - assert export.curl_command(flow) == result + assert export.curl_command(patch_request) == result -class TestExportPythonCode: - def test_get(self): - flow = tflow.tflow(req=req_get()) - python_equals("mitmproxy/data/test_flow_export/python_get.py", export.python_code(flow)) +class TestExportPythonCode(TExport): + def test_get(self, get_request): + python_equals("mitmproxy/data/test_flow_export/python_get.py", + export.python_code(get_request)) - def test_post(self): - flow = tflow.tflow(req=req_post()) - python_equals("mitmproxy/data/test_flow_export/python_post.py", export.python_code(flow)) + def test_post(self, post_request): + python_equals("mitmproxy/data/test_flow_export/python_post.py", + export.python_code(post_request)) - def test_post_json(self): - p = req_post() - p.content = b'{"name": "example", "email": "example@example.com"}' - p.headers = Headers(content_type="application/json") - flow = tflow.tflow(req=p) - python_equals("mitmproxy/data/test_flow_export/python_post_json.py", export.python_code(flow)) + def test_post_json(self, post_request): + post_request.request.content = b'{"name": "example", "email": "example@example.com"}' + post_request.request.headers = Headers(content_type="application/json") + python_equals("mitmproxy/data/test_flow_export/python_post_json.py", + export.python_code(post_request)) - def test_patch(self): - flow = tflow.tflow(req=req_patch()) - python_equals("mitmproxy/data/test_flow_export/python_patch.py", export.python_code(flow)) + def test_patch(self, patch_request): + python_equals("mitmproxy/data/test_flow_export/python_patch.py", + export.python_code(patch_request)) -class TestExportLocustCode: - def test_get(self): - flow = tflow.tflow(req=req_get()) - python_equals("mitmproxy/data/test_flow_export/locust_get.py", export.locust_code(flow)) +class TestExportLocustCode(TExport): + def test_get(self, get_request): + python_equals("mitmproxy/data/test_flow_export/locust_get.py", + export.locust_code(get_request)) - def test_post(self): - p = req_post() - p.content = b'content' - p.headers = '' - flow = tflow.tflow(req=p) - python_equals("mitmproxy/data/test_flow_export/locust_post.py", export.locust_code(flow)) + def test_post(self, post_request): + post_request.request.content = b'content' + post_request.request.headers.clear() + python_equals("mitmproxy/data/test_flow_export/locust_post.py", + export.locust_code(post_request)) - def test_patch(self): - flow = tflow.tflow(req=req_patch()) - python_equals("mitmproxy/data/test_flow_export/locust_patch.py", export.locust_code(flow)) + def test_patch(self, patch_request): + python_equals("mitmproxy/data/test_flow_export/locust_patch.py", + export.locust_code(patch_request)) -class TestExportLocustTask: - def test_get(self): - flow = tflow.tflow(req=req_get()) - python_equals("mitmproxy/data/test_flow_export/locust_task_get.py", export.locust_task(flow)) +class TestExportLocustTask(TExport): + def test_get(self, get_request): + python_equals("mitmproxy/data/test_flow_export/locust_task_get.py", + export.locust_task(get_request)) - def test_post(self): - flow = tflow.tflow(req=req_post()) - python_equals("mitmproxy/data/test_flow_export/locust_task_post.py", export.locust_task(flow)) + def test_post(self, post_request): + python_equals("mitmproxy/data/test_flow_export/locust_task_post.py", + export.locust_task(post_request)) - def test_patch(self): - flow = tflow.tflow(req=req_patch()) - python_equals("mitmproxy/data/test_flow_export/locust_task_patch.py", export.locust_task(flow)) + def test_patch(self, patch_request): + python_equals("mitmproxy/data/test_flow_export/locust_task_patch.py", + export.locust_task(patch_request)) class TestURL: diff --git a/web/src/js/__tests__/flow/utilsSpec.js b/web/src/js/__tests__/flow/utilsSpec.js new file mode 100644 index 00000000..2d8f0456 --- /dev/null +++ b/web/src/js/__tests__/flow/utilsSpec.js @@ -0,0 +1,69 @@ +import * as utils from '../../flow/utils' + +describe('MessageUtils', () => { + it('should be possible to get first header', () => { + let msg = { headers: [["foo", "bar"]]} + expect(utils.MessageUtils.get_first_header(msg, "foo")).toEqual("bar") + expect(utils.MessageUtils.get_first_header(msg, "123")).toEqual(undefined) + }) + + it('should be possible to get Content-Type', () => { + let type = "text/html", + msg = { headers: [["Content-Type", type]]} + expect(utils.MessageUtils.getContentType(msg)).toEqual(type) + }) + + it('should be possible to match header', () => { + let h1 = ["foo", "bar"], + msg = {headers : [h1]} + expect(utils.MessageUtils.match_header(msg, /foo/i)).toEqual(h1) + expect(utils.MessageUtils.match_header(msg, /123/i)).toBeFalsy() + }) + + it('should be possible to get content URL', () => { + // request + let msg = "foo", view = "bar", + flow = { request: msg, id: 1} + expect(utils.MessageUtils.getContentURL(flow, msg, view)).toEqual( + "/flows/1/request/content/bar" + ) + expect(utils.MessageUtils.getContentURL(flow, msg, '')).toEqual( + "/flows/1/request/content" + ) + // response + flow = {response: msg, id: 2} + expect(utils.MessageUtils.getContentURL(flow, msg, view)).toEqual( + "/flows/2/response/content/bar" + ) + }) +}) + +describe('RequestUtils', () => { + it('should be possible prettify url', () => { + let request = {port: 4444, scheme: "http", pretty_host: "foo", path: "/bar"} + expect(utils.RequestUtils.pretty_url(request)).toEqual( + "http://foo:4444/bar" + ) + }) +}) + +describe('parseUrl', () => { + it('should be possible to parse url', () => { + let url = "http://foo:4444/bar" + expect(utils.parseUrl(url)).toEqual({ + port: 4444, + scheme: 'http', + host: 'foo', + path: '/bar' + }) + + expect(utils.parseUrl("foo:foo")).toBeFalsy() + }) +}) + +describe('isValidHttpVersion', () => { + it('should be possible to validate http version', () => { + expect(utils.isValidHttpVersion("HTTP/1.1")).toBeTruthy() + expect(utils.isValidHttpVersion("HTTP//1")).toBeFalsy() + }) +}) diff --git a/web/src/js/__tests__/utilsSpec.js b/web/src/js/__tests__/utilsSpec.js new file mode 100644 index 00000000..9a1a0750 --- /dev/null +++ b/web/src/js/__tests__/utilsSpec.js @@ -0,0 +1,95 @@ +import * as utils from '../utils' + +global.fetch = jest.fn() + +describe('formatSize', () => { + it('should return 0 when 0 byte', () => { + expect(utils.formatSize(0)).toEqual('0') + }) + + it('should return formatted size', () => { + expect(utils.formatSize(27104011)).toEqual("25.8mb") + expect(utils.formatSize(1023)).toEqual("1023b") + }) +}) + +describe('formatTimeDelta', () => { + it('should return formatted time', () => { + expect(utils.formatTimeDelta(3600100)).toEqual("1h") + }) +}) + +describe('formatTimeSTamp', () => { + it('should return formatted time', () => { + expect(utils.formatTimeStamp(1483228800)).toEqual("2017-01-01 00:00:00.000") + }) +}) + +describe('reverseString', () => { + it('should return reversed string', () => { + let str1 = "abc", str2="xyz" + expect(utils.reverseString(str1) > utils.reverseString(str2)).toBeTruthy() + }) +}) + +describe('fetchApi', () => { + it('should handle fetch operation', () => { + utils.fetchApi('http://foo/bar', {method: "POST"}) + expect(fetch.mock.calls[0][0]).toEqual( + "http://foo/bar?_xsrf=undefined" + ) + fetch.mockClear() + + utils.fetchApi('http://foo?bar=1', {method: "POST"}) + expect(fetch.mock.calls[0][0]).toEqual( + "http://foo?bar=1&_xsrf=undefined" + ) + + }) + + it('should be possible to do put request', () => { + fetch.mockClear() + utils.fetchApi.put("http://foo", [1, 2, 3], {}) + expect(fetch.mock.calls[0]).toEqual( + [ + "http://foo?_xsrf=undefined", + { + body: "[1,2,3]", + credentials: "same-origin", + headers: { "Content-Type": "application/json" }, + method: "PUT" + }, + ] + ) + }) +}) + +describe('getDiff', () => { + it('should return json object including only the changed keys value pairs', () => { + let obj1 = {a: 1, b:{ foo: 1} , c: [3]}, + obj2 = {a: 1, b:{ foo: 2} , c: [4]} + expect(utils.getDiff(obj1, obj2)).toEqual({ b: {foo: 2}, c:[4]}) + }) +}) + +describe('pure', () => { + let tFunc = function({ className }) { + return (<p className={ className }>foo</p>) + }, + puredFunc = utils.pure(tFunc), + f = new puredFunc('bar') + + it('should display function name', () => { + expect(utils.pure(tFunc).displayName).toEqual('tFunc') + }) + + it('should suggest when should component update', () => { + expect(f.shouldComponentUpdate('foo')).toBeTruthy() + expect(f.shouldComponentUpdate('bar')).toBeFalsy() + }) + + it('should render properties', () => { + expect(f.render()).toEqual(tFunc('bar')) + }) + +}) |