diff options
25 files changed, 501 insertions, 307 deletions
diff --git a/docs/certinstall.rst b/docs/certinstall.rst index 1bd6df99..14d66d30 100644 --- a/docs/certinstall.rst +++ b/docs/certinstall.rst @@ -132,7 +132,7 @@ mitmproxy-ca-cert.cer Same file as .pem, but with an extension expected by some Using a custom certificate -------------------------- -You can use your own certificate by passing the ``--cert [domain=]path_to_certificate`` option to +You can use your own (leaf) certificate by passing the ``--cert [domain=]path_to_certificate`` option to mitmproxy. Mitmproxy then uses the provided certificate for interception of the specified domain instead of generating a certificate signed by its own CA. diff --git a/examples/complex/README.md b/examples/complex/README.md index 452f2395..77dbe2f5 100644 --- a/examples/complex/README.md +++ b/examples/complex/README.md @@ -5,7 +5,6 @@ | change_upstream_proxy.py | Dynamically change the upstream proxy. | | dns_spoofing.py | Use mitmproxy in a DNS spoofing scenario. | | dup_and_replay.py | Duplicates each request, changes it, and then replays the modified request. | -| flowbasic.py | Basic use of mitmproxy's FlowMaster directly. | | full_transparency_shim.c | Setuid wrapper that can be used to run mitmproxy in full transparency mode, as a normal user. | | har_dump.py | Dump flows as HAR files. | | mitmproxywrapper.py | Bracket mitmproxy run with proxy enable/disable on OS X | @@ -16,3 +15,4 @@ | stream_modify.py | Modify a streamed response body. | | tcp_message.py | Modify a raw TCP connection | | tls_passthrough.py | Use conditional TLS interception based on a user-defined strategy. | +| xss_scanner.py | Scan all visited webpages. | diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index 40d0373c..21bcc341 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -201,7 +201,7 @@ def format_request_cookies(fields): def format_response_cookies(fields): - return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields) + return format_cookies((c[0], c[1][0], c[1][1]) for c in fields) def name_value(obj): diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py index b4367d78..7a45106c 100644 --- a/mitmproxy/addons/__init__.py +++ b/mitmproxy/addons/__init__.py @@ -8,7 +8,6 @@ from mitmproxy.addons import disable_h2c from mitmproxy.addons import onboarding from mitmproxy.addons import proxyauth from mitmproxy.addons import replace -from mitmproxy.addons import readfile from mitmproxy.addons import script from mitmproxy.addons import serverplayback from mitmproxy.addons import setheaders @@ -38,6 +37,5 @@ def default_addons(): stickycookie.StickyCookie(), streambodies.StreamBodies(), streamfile.StreamFile(), - readfile.ReadFile(), upstream_auth.UpstreamAuth(), ] diff --git a/mitmproxy/addons/readfile.py b/mitmproxy/addons/readfile.py index 949da15d..05b6c309 100644 --- a/mitmproxy/addons/readfile.py +++ b/mitmproxy/addons/readfile.py @@ -1,38 +1,56 @@ import os.path +import sys +import typing from mitmproxy import ctx -from mitmproxy import io from mitmproxy import exceptions +from mitmproxy import io class ReadFile: """ An addon that handles reading from file on startup. """ - def load_flows_file(self, path: str) -> int: - path = os.path.expanduser(path) + + def load_flows(self, fo: typing.IO[bytes]) -> int: cnt = 0 + freader = io.FlowReader(fo) try: - with open(path, "rb") as f: - freader = io.FlowReader(f) - for i in freader.stream(): - cnt += 1 - ctx.master.load_flow(i) - return cnt - except (IOError, exceptions.FlowReadException) as v: + for flow in freader.stream(): + ctx.master.load_flow(flow) + cnt += 1 + except (IOError, exceptions.FlowReadException) as e: if cnt: - ctx.log.warn( - "Flow file corrupted - loaded %i flows." % cnt, - ) + ctx.log.warn("Flow file corrupted - loaded %i flows." % cnt) else: ctx.log.error("Flow file corrupted.") - raise exceptions.FlowReadException(v) + raise exceptions.FlowReadException(str(e)) from e + else: + return cnt + + def load_flows_from_path(self, path: str) -> int: + path = os.path.expanduser(path) + try: + with open(path, "rb") as f: + return self.load_flows(f) + except IOError as e: + ctx.log.error("Cannot load flows: {}".format(e)) + raise exceptions.FlowReadException(str(e)) from e def running(self): if ctx.options.rfile: try: - self.load_flows_file(ctx.options.rfile) - except exceptions.FlowReadException as v: - raise exceptions.OptionsError(v) + self.load_flows_from_path(ctx.options.rfile) + except exceptions.FlowReadException as e: + raise exceptions.OptionsError(e) from e finally: ctx.master.addons.trigger("processing_complete") + + +class ReadFileStdin(ReadFile): + """Support the special case of "-" for reading from stdin""" + def load_flows_from_path(self, path: str) -> int: + if path == "-": + return self.load_flows(sys.stdin.buffer) + else: + return super().load_flows_from_path(path) diff --git a/mitmproxy/addons/readstdin.py b/mitmproxy/addons/readstdin.py deleted file mode 100644 index 93a99f01..00000000 --- a/mitmproxy/addons/readstdin.py +++ /dev/null @@ -1,26 +0,0 @@ -from mitmproxy import ctx -from mitmproxy import io -from mitmproxy import exceptions -import sys - - -class ReadStdin: - """ - An addon that reads from stdin if we're not attached to (someting like) - a tty. - """ - def running(self, stdin = sys.stdin): - if not stdin.isatty(): - ctx.log.info("Reading from stdin") - try: - stdin.buffer.read(0) - except Exception as e: - ctx.log.warn("Cannot read from stdin: {}".format(e)) - return - freader = io.FlowReader(stdin.buffer) - try: - for i in freader.stream(): - ctx.master.load_flow(i) - except exceptions.FlowReadException as e: - ctx.log.error("Error reading from stdin: %s" % e) - ctx.master.addons.trigger("processing_complete") diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index 04d99975..e58e0a58 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -1,14 +1,14 @@ import collections from http import cookiejar +from typing import List, Tuple, Dict, Optional # noqa +from mitmproxy import http, flowfilter, ctx, exceptions from mitmproxy.net.http import cookies -from mitmproxy import exceptions -from mitmproxy import flowfilter -from mitmproxy import ctx +TOrigin = Tuple[str, int, str] -def ckey(attrs, f): +def ckey(attrs: Dict[str, str], f: http.HTTPFlow) -> TOrigin: """ Returns a (domain, port, path) tuple. """ @@ -21,18 +21,18 @@ def ckey(attrs, f): return (domain, f.request.port, path) -def domain_match(a, b): - if cookiejar.domain_match(a, b): +def domain_match(a: str, b: str) -> bool: + if cookiejar.domain_match(a, b): # type: ignore return True - elif cookiejar.domain_match(a, b.strip(".")): + elif cookiejar.domain_match(a, b.strip(".")): # type: ignore return True return False class StickyCookie: def __init__(self): - self.jar = collections.defaultdict(dict) - self.flt = None + self.jar = collections.defaultdict(dict) # type: Dict[TOrigin, Dict[str, str]] + self.flt = None # type: Optional[flowfilter.TFilter] def configure(self, updated): if "stickycookie" in updated: @@ -46,7 +46,7 @@ class StickyCookie: else: self.flt = None - def response(self, flow): + def response(self, flow: http.HTTPFlow): if self.flt: for name, (value, attrs) in flow.response.cookies.items(multi=True): # FIXME: We now know that Cookie.py screws up some cookies with @@ -63,24 +63,21 @@ class StickyCookie: if not self.jar[dom_port_path]: self.jar.pop(dom_port_path, None) else: - b = attrs.copy() - b.insert(0, name, value) - self.jar[dom_port_path][name] = b + self.jar[dom_port_path][name] = value - def request(self, flow): + def request(self, flow: http.HTTPFlow): if self.flt: - l = [] + cookie_list = [] # type: List[Tuple[str,str]] if flowfilter.match(self.flt, flow): - for domain, port, path in self.jar.keys(): + for (domain, port, path), c in self.jar.items(): match = [ domain_match(flow.request.host, domain), flow.request.port == port, flow.request.path.startswith(path) ] if all(match): - c = self.jar[(domain, port, path)] - l.extend([cookies.format_cookie_header(c[name].items(multi=True)) for name in c.keys()]) - if l: + cookie_list.extend(c.items()) + if cookie_list: # FIXME: we need to formalise this... - flow.request.stickycookie = True - flow.request.headers["cookie"] = "; ".join(l) + flow.metadata["stickycookie"] = True + flow.request.headers["cookie"] = cookies.format_cookie_header(cookie_list) diff --git a/mitmproxy/export.py b/mitmproxy/export.py index 235e754a..efa08874 100644 --- a/mitmproxy/export.py +++ b/mitmproxy/export.py @@ -6,19 +6,7 @@ import textwrap from typing import Any from mitmproxy import http - - -def _native(s): - if isinstance(s, bytes): - return s.decode() - return s - - -def dictstr(items, indent: str) -> str: - lines = [] - for k, v in items: - lines.append(indent + "%s: %s,\n" % (repr(_native(k)), repr(_native(v)))) - return "{\n%s}\n" % "".join(lines) +from mitmproxy.utils import strutils def curl_command(flow: http.HTTPFlow) -> str: @@ -36,7 +24,10 @@ def curl_command(flow: http.HTTPFlow) -> str: data += "'%s'" % request.url if request.content: - data += " --data-binary '%s'" % _native(request.content) + data += " --data-binary '%s'" % strutils.bytes_to_escaped_str( + request.content, + escape_single_quotes=True + ) return data @@ -127,10 +118,14 @@ def locust_code(flow): args = "" headers = "" + + def conv(x): + return strutils.bytes_to_escaped_str(x, escape_single_quotes=True) + if flow.request.headers: lines = [ - (_native(k), _native(v)) for k, v in flow.request.headers.fields - if _native(k).lower() not in [":authority", "host", "cookie"] + (conv(k), conv(v)) for k, v in flow.request.headers.fields + if conv(k).lower() not in [":authority", "host", "cookie"] ] lines = [" '%s': '%s',\n" % (k, v) for k, v in lines] headers += "\n headers = {\n%s }\n" % "".join(lines) @@ -148,7 +143,7 @@ def locust_code(flow): data = "" if flow.request.content: - data = "\n data = '''%s'''\n" % _native(flow.request.content) + data = "\n data = '''%s'''\n" % conv(flow.request.content) args += "\n data=data," code = code.format( diff --git a/mitmproxy/net/http/cookies.py b/mitmproxy/net/http/cookies.py index 01d42d46..5b410acc 100644 --- a/mitmproxy/net/http/cookies.py +++ b/mitmproxy/net/http/cookies.py @@ -1,7 +1,7 @@ -import collections import email.utils import re import time +from typing import Tuple, List, Iterable from mitmproxy.types import multidict @@ -23,10 +23,7 @@ cookies to be set in a single header. Serialization follows RFC6265. http://tools.ietf.org/html/rfc2965 """ -_cookie_params = set(( - 'expires', 'path', 'comment', 'max-age', - 'secure', 'httponly', 'version', -)) +_cookie_params = {'expires', 'path', 'comment', 'max-age', 'secure', 'httponly', 'version'} ESCAPE = re.compile(r"([\"\\])") @@ -43,7 +40,8 @@ class CookieAttrs(multidict.MultiDict): return values[-1] -SetCookie = collections.namedtuple("SetCookie", ["value", "attrs"]) +TSetCookie = Tuple[str, str, CookieAttrs] +TPairs = List[List[str]] # TODO: Should be List[Tuple[str,str]]? def _read_until(s, start, term): @@ -131,15 +129,15 @@ def _read_cookie_pairs(s, off=0): return pairs, off -def _read_set_cookie_pairs(s, off=0): +def _read_set_cookie_pairs(s: str, off=0) -> Tuple[List[TPairs], int]: """ Read pairs of lhs=rhs values from SetCookie headers while handling multiple cookies. off: start offset specials: attributes that are treated specially """ - cookies = [] - pairs = [] + cookies = [] # type: List[TPairs] + pairs = [] # type: TPairs while True: lhs, off = _read_key(s, off, ";=,") @@ -182,7 +180,7 @@ def _read_set_cookie_pairs(s, off=0): return cookies, off -def _has_special(s): +def _has_special(s: str) -> bool: for i in s: if i in '",;\\': return True @@ -238,41 +236,44 @@ def format_cookie_header(lst): return _format_pairs(lst) -def parse_set_cookie_header(line): +def parse_set_cookie_header(line: str) -> List[TSetCookie]: """ - Parse a Set-Cookie header value + Parse a Set-Cookie header value - Returns a list of (name, value, attrs) tuples, where attrs is a + Returns: + A list of (name, value, attrs) tuples, where attrs is a CookieAttrs dict of attributes. No attempt is made to parse attribute values - they are treated purely as strings. """ cookie_pairs, off = _read_set_cookie_pairs(line) - cookies = [ - (pairs[0][0], pairs[0][1], CookieAttrs(tuple(x) for x in pairs[1:])) - for pairs in cookie_pairs if pairs - ] + cookies = [] + for pairs in cookie_pairs: + if pairs: + cookie, *attrs = pairs + cookies.append(( + cookie[0], + cookie[1], + CookieAttrs(attrs) + )) return cookies -def parse_set_cookie_headers(headers): +def parse_set_cookie_headers(headers: Iterable[str]) -> List[TSetCookie]: rv = [] for header in headers: cookies = parse_set_cookie_header(header) - if cookies: - for name, value, attrs in cookies: - rv.append((name, SetCookie(value, attrs))) + rv.extend(cookies) return rv -def format_set_cookie_header(set_cookies): +def format_set_cookie_header(set_cookies: List[TSetCookie]) -> str: """ Formats a Set-Cookie header value. """ rv = [] - for set_cookie in set_cookies: - name, value, attrs = set_cookie + for name, value, attrs in set_cookies: pairs = [(name, value)] pairs.extend( @@ -284,37 +285,36 @@ def format_set_cookie_header(set_cookies): return ", ".join(rv) -def refresh_set_cookie_header(c, delta): +def refresh_set_cookie_header(c: str, delta: int) -> str: """ Args: c: A Set-Cookie string delta: Time delta in seconds Returns: A refreshed Set-Cookie string + Raises: + ValueError, if the cookie is invalid. """ - - name, value, attrs = parse_set_cookie_header(c)[0] - if not name or not value: - raise ValueError("Invalid Cookie") - - if "expires" in attrs: - e = email.utils.parsedate_tz(attrs["expires"]) - if e: - f = email.utils.mktime_tz(e) + delta - attrs.set_all("expires", [email.utils.formatdate(f)]) - else: - # This can happen when the expires tag is invalid. - # reddit.com sends a an expires tag like this: "Thu, 31 Dec - # 2037 23:59:59 GMT", which is valid RFC 1123, but not - # strictly correct according to the cookie spec. Browsers - # appear to parse this tolerantly - maybe we should too. - # For now, we just ignore this. - del attrs["expires"] - - rv = format_set_cookie_header([(name, value, attrs)]) - if not rv: - raise ValueError("Invalid Cookie") - return rv + cookies = parse_set_cookie_header(c) + for cookie in cookies: + name, value, attrs = cookie + if not name or not value: + raise ValueError("Invalid Cookie") + + if "expires" in attrs: + e = email.utils.parsedate_tz(attrs["expires"]) + if e: + f = email.utils.mktime_tz(e) + delta + attrs.set_all("expires", [email.utils.formatdate(f)]) + else: + # This can happen when the expires tag is invalid. + # reddit.com sends a an expires tag like this: "Thu, 31 Dec + # 2037 23:59:59 GMT", which is valid RFC 1123, but not + # strictly correct according to the cookie spec. Browsers + # appear to parse this tolerantly - maybe we should too. + # For now, we just ignore this. + del attrs["expires"] + return format_set_cookie_header(cookies) def get_expiration_ts(cookie_attrs): diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py index 8edd43b8..18950fc7 100644 --- a/mitmproxy/net/http/response.py +++ b/mitmproxy/net/http/response.py @@ -131,7 +131,11 @@ class Response(message.Message): def _get_cookies(self): h = self.headers.get_all("set-cookie") - return tuple(cookies.parse_set_cookie_headers(h)) + all_cookies = cookies.parse_set_cookie_headers(h) + return tuple( + (name, (value, attrs)) + for name, value, attrs in all_cookies + ) def _set_cookies(self, value): cookie_headers = [] diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index c1d584ac..e7a2c6ae 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -17,8 +17,9 @@ from mitmproxy import exceptions from mitmproxy import master from mitmproxy import io from mitmproxy import log -from mitmproxy.addons import view from mitmproxy.addons import intercept +from mitmproxy.addons import readfile +from mitmproxy.addons import view from mitmproxy.tools.console import flowlist from mitmproxy.tools.console import flowview from mitmproxy.tools.console import grideditor @@ -91,7 +92,12 @@ class ConsoleMaster(master.Master): signals.sig_add_log.connect(self.sig_add_log) self.addons.add(Logger()) self.addons.add(*addons.default_addons()) - self.addons.add(intercept.Intercept(), self.view, UnsupportedLog()) + self.addons.add( + intercept.Intercept(), + self.view, + UnsupportedLog(), + readfile.ReadFile(), + ) def sigint_handler(*args, **kwargs): self.prompt_for_exit() diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index 6329f6b7..4d0ccf4b 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -1,7 +1,7 @@ from mitmproxy import addons from mitmproxy import options from mitmproxy import master -from mitmproxy.addons import dumper, termlog, termstatus, readstdin, keepserving +from mitmproxy.addons import dumper, termlog, termstatus, keepserving, readfile class ErrorCheck: @@ -30,7 +30,7 @@ class DumpMaster(master.Master): if with_dumper: self.addons.add(dumper.Dumper()) self.addons.add( - readstdin.ReadStdin(), keepserving.KeepServing(), + readfile.ReadFileStdin(), self.errorcheck ) diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 0db5a09f..c09fe0a2 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -7,6 +7,7 @@ from mitmproxy import log from mitmproxy import master from mitmproxy.addons import eventstore from mitmproxy.addons import intercept +from mitmproxy.addons import readfile from mitmproxy.addons import termlog from mitmproxy.addons import view from mitmproxy.addons import termstatus @@ -32,6 +33,7 @@ class WebMaster(master.Master): self.addons.add(*addons.default_addons()) self.addons.add( intercept.Intercept(), + readfile.ReadFile(), self.view, self.events, ) diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py index 5df4ea4b..628ea642 100644 --- a/mitmproxy/utils/typecheck.py +++ b/mitmproxy/utils/typecheck.py @@ -68,5 +68,7 @@ def check_type(name: str, value: typing.Any, typeinfo: typing.Any) -> None: return else: raise e + elif typename.startswith("typing.Any"): + return elif not isinstance(value, typeinfo): raise e diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index d2cefe79..d8aa593b 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -68,7 +68,6 @@ def test_simple(): ctx.configure(d, flow_detail=4) flow = tflow.tflow() flow.request = tutils.treq() - flow.request.stickycookie = True flow.client_conn = mock.MagicMock() flow.client_conn.address[0] = "foo" flow.response = tutils.tresp(content=None) diff --git a/test/mitmproxy/addons/test_readfile.py b/test/mitmproxy/addons/test_readfile.py index b30c147b..813aa10e 100644 --- a/test/mitmproxy/addons/test_readfile.py +++ b/test/mitmproxy/addons/test_readfile.py @@ -1,62 +1,104 @@ +import io +from unittest import mock + +import pytest + +import mitmproxy.io +from mitmproxy import exceptions from mitmproxy.addons import readfile from mitmproxy.test import taddons from mitmproxy.test import tflow -from mitmproxy import io -from mitmproxy import exceptions -from unittest import mock -import pytest +@pytest.fixture +def data(): + f = io.BytesIO() + + w = mitmproxy.io.FlowWriter(f) + flows = [ + tflow.tflow(resp=True), + tflow.tflow(err=True), + tflow.ttcpflow(), + tflow.ttcpflow(err=True) + ] + for flow in flows: + w.add(flow) -def write_data(path, corrupt=False): - with open(path, "wb") as tf: - w = io.FlowWriter(tf) - for i in range(3): - f = tflow.tflow(resp=True) - w.add(f) - for i in range(3): - f = tflow.tflow(err=True) - w.add(f) - f = tflow.ttcpflow() - w.add(f) - f = tflow.ttcpflow(err=True) - w.add(f) - if corrupt: - tf.write(b"flibble") - - -@mock.patch('mitmproxy.master.Master.load_flow') -def test_configure(mck, tmpdir): - - rf = readfile.ReadFile() - with taddons.context() as tctx: - tf = str(tmpdir.join("tfile")) - write_data(tf) - tctx.configure(rf, rfile=str(tf)) - assert not mck.called - rf.running() - assert mck.called - - write_data(tf, corrupt=True) - tctx.configure(rf, rfile=str(tf)) - with pytest.raises(exceptions.OptionsError): + f.seek(0) + return f + + +@pytest.fixture +def corrupt_data(): + f = data() + f.seek(0, io.SEEK_END) + f.write(b"qibble") + f.seek(0) + return f + + +class TestReadFile: + @mock.patch('mitmproxy.master.Master.load_flow') + def test_configure(self, mck, tmpdir, data, corrupt_data): + rf = readfile.ReadFile() + with taddons.context() as tctx: + tf = tmpdir.join("tfile") + + tf.write(data.getvalue()) + tctx.configure(rf, rfile=str(tf)) + assert not mck.called rf.running() + assert mck.called + tf.write(corrupt_data.getvalue()) + tctx.configure(rf, rfile=str(tf)) + with pytest.raises(exceptions.OptionsError): + rf.running() -@mock.patch('mitmproxy.master.Master.load_flow') -def test_corruption(mck, tmpdir): + @mock.patch('mitmproxy.master.Master.load_flow') + def test_corrupt(self, mck, corrupt_data): + rf = readfile.ReadFile() + with taddons.context() as tctx: + with pytest.raises(exceptions.FlowReadException): + rf.load_flows(io.BytesIO(b"qibble")) + assert not mck.called + assert len(tctx.master.logs) == 1 - rf = readfile.ReadFile() - with taddons.context() as tctx: - with pytest.raises(exceptions.FlowReadException): - rf.load_flows_file("nonexistent") - assert not mck.called - assert len(tctx.master.logs) == 1 + with pytest.raises(exceptions.FlowReadException): + rf.load_flows(corrupt_data) + assert mck.called + assert len(tctx.master.logs) == 2 + + def test_nonexisting_file(self): + rf = readfile.ReadFile() + with taddons.context() as tctx: + with pytest.raises(exceptions.FlowReadException): + rf.load_flows_from_path("nonexistent") + assert len(tctx.master.logs) == 1 + + +class TestReadFileStdin: + @mock.patch('mitmproxy.master.Master.load_flow') + @mock.patch('sys.stdin') + def test_stdin(self, stdin, load_flow, data, corrupt_data): + rf = readfile.ReadFileStdin() + with taddons.context() as tctx: + stdin.buffer = data + tctx.configure(rf, rfile="-") + assert not load_flow.called + rf.running() + assert load_flow.called - tfc = str(tmpdir.join("tfile")) - write_data(tfc, corrupt=True) + stdin.buffer = corrupt_data + tctx.configure(rf, rfile="-") + with pytest.raises(exceptions.OptionsError): + rf.running() - with pytest.raises(exceptions.FlowReadException): - rf.load_flows_file(tfc) - assert mck.called - assert len(tctx.master.logs) == 2 + @mock.patch('mitmproxy.master.Master.load_flow') + def test_normal(self, load_flow, tmpdir, data): + rf = readfile.ReadFileStdin() + with taddons.context(): + tfile = tmpdir.join("tfile") + tfile.write(data.getvalue()) + rf.load_flows_from_path(str(tfile)) + assert load_flow.called diff --git a/test/mitmproxy/addons/test_readstdin.py b/test/mitmproxy/addons/test_readstdin.py deleted file mode 100644 index 76b01f4f..00000000 --- a/test/mitmproxy/addons/test_readstdin.py +++ /dev/null @@ -1,53 +0,0 @@ - -import io -from mitmproxy.addons import readstdin -from mitmproxy.test import taddons -from mitmproxy.test import tflow -import mitmproxy.io -from unittest import mock - - -def gen_data(corrupt=False): - tf = io.BytesIO() - w = mitmproxy.io.FlowWriter(tf) - for i in range(3): - f = tflow.tflow(resp=True) - w.add(f) - for i in range(3): - f = tflow.tflow(err=True) - w.add(f) - f = tflow.ttcpflow() - w.add(f) - f = tflow.ttcpflow(err=True) - w.add(f) - if corrupt: - tf.write(b"flibble") - tf.seek(0) - return tf - - -class mStdin: - def __init__(self, d): - self.buffer = d - - def isatty(self): - return False - - -@mock.patch('mitmproxy.master.Master.load_flow') -def test_read(m, tmpdir): - rf = readstdin.ReadStdin() - with taddons.context() as tctx: - assert not m.called - rf.running(stdin=mStdin(gen_data())) - assert m.called - - rf.running(stdin=mStdin(None)) - assert tctx.master.logs - tctx.master.clear() - - m.reset_mock() - assert not m.called - rf.running(stdin=mStdin(gen_data(corrupt=True))) - assert m.called - assert tctx.master.logs diff --git a/test/mitmproxy/addons/test_stickycookie.py b/test/mitmproxy/addons/test_stickycookie.py index 9092e09b..f77d019d 100644 --- a/test/mitmproxy/addons/test_stickycookie.py +++ b/test/mitmproxy/addons/test_stickycookie.py @@ -110,8 +110,8 @@ class TestStickyCookie: f.response.headers["Set-Cookie"] = c2 sc.response(f) googlekey = list(sc.jar.keys())[0] - assert len(sc.jar[googlekey].keys()) == 1 - assert list(sc.jar[googlekey]["somecookie"].items())[0][1] == "newvalue" + assert len(sc.jar[googlekey]) == 1 + assert sc.jar[googlekey]["somecookie"] == "newvalue" def test_response_delete(self): sc = stickycookie.StickyCookie() diff --git a/test/mitmproxy/data/test_flow_export/locust_task_post.py b/test/mitmproxy/data/test_flow_export/locust_task_post.py index 989df455..a5f307ee 100644 --- a/test/mitmproxy/data/test_flow_export/locust_task_post.py +++ b/test/mitmproxy/data/test_flow_export/locust_task_post.py @@ -2,7 +2,7 @@ def path(self): url = self.locust.host + '/path' - data = '''content''' + data = '''\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./0123456789:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff''' self.response = self.client.request( method='POST', diff --git a/test/mitmproxy/data/test_flow_export/python_post.py b/test/mitmproxy/data/test_flow_export/python_post.py index 6254adfb..42f1af9a 100644 --- a/test/mitmproxy/data/test_flow_export/python_post.py +++ b/test/mitmproxy/data/test_flow_export/python_post.py @@ -2,7 +2,16 @@ import requests response = requests.post( 'http://address:22/path', - data=b'content' + data=(b'\x00\x01\x02\x03\x04\x05\x06\x07\x08\t\n\x0b\x0c\r\x0e\x0f\x10\x11\x12\x13' + b'\x14\x15\x16\x17\x18\x19\x1a\x1b\x1c\x1d\x1e\x1f !"#$%&\'()*+,-./01234567' + b'89:;<=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~\x7f' + b'\x80\x81\x82\x83\x84\x85\x86\x87\x88\x89\x8a\x8b\x8c\x8d\x8e\x8f' + b'\x90\x91\x92\x93\x94\x95\x96\x97\x98\x99\x9a\x9b\x9c\x9d\x9e\x9f' + b'\xa0\xa1\xa2\xa3\xa4\xa5\xa6\xa7\xa8\xa9\xaa\xab\xac\xad\xae\xaf' + b'\xb0\xb1\xb2\xb3\xb4\xb5\xb6\xb7\xb8\xb9\xba\xbb\xbc\xbd\xbe\xbf' + b'\xc0\xc1\xc2\xc3\xc4\xc5\xc6\xc7\xc8\xc9\xca\xcb\xcc\xcd\xce\xcf' + b'\xd0\xd1\xd2\xd3\xd4\xd5\xd6\xd7\xd8\xd9\xda\xdb\xdc\xdd\xde\xdf' + b'\xe0\xe1\xe2\xe3\xe4\xe5\xe6\xe7\xe8\xe9\xea\xeb\xec\xed\xee\xef' + b'\xf0\xf1\xf2\xf3\xf4\xf5\xf6\xf7\xf8\xf9\xfa\xfb\xfc\xfd\xfe\xff') ) - print(response.text) diff --git a/test/mitmproxy/net/http/test_cookies.py b/test/mitmproxy/net/http/test_cookies.py index 5c30dbdb..680a5033 100644 --- a/test/mitmproxy/net/http/test_cookies.py +++ b/test/mitmproxy/net/http/test_cookies.py @@ -283,6 +283,10 @@ def test_refresh_cookie(): c = "foo/bar=bla" assert cookies.refresh_set_cookie_header(c, 0) + # https://github.com/mitmproxy/mitmproxy/issues/2250 + c = "" + assert cookies.refresh_set_cookie_header(c, 60) == "" + @mock.patch('time.time') def test_get_expiration_ts(*args): diff --git a/test/mitmproxy/test_export.py b/test/mitmproxy/test_export.py index 457d8836..b789e6b5 100644 --- a/test/mitmproxy/test_export.py +++ b/test/mitmproxy/test_export.py @@ -1,13 +1,15 @@ -from mitmproxy.test import tflow import re -from mitmproxy.net.http import Headers +import pytest + from mitmproxy import export # heh +from mitmproxy.net.http import Headers +from mitmproxy.test import tflow from mitmproxy.test import tutils def clean_blanks(s): - return re.sub(r"^(\s+)$", "", s, flags=re.MULTILINE) + return re.sub(r"^\s+", "", s, flags=re.MULTILINE) def python_equals(testdata, text): @@ -19,85 +21,110 @@ def python_equals(testdata, text): assert clean_blanks(text).rstrip() == clean_blanks(d).rstrip() -def req_get(): - return tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz") +@pytest.fixture +def get_request(): + return tflow.tflow( + req=tutils.treq( + method=b'GET', + content=b'', + path=b"/path?a=foo&a=bar&b=baz" + ) + ) + + +@pytest.fixture +def post_request(): + return tflow.tflow( + req=tutils.treq( + method=b'POST', + headers=(), + content=bytes(range(256)) + ) + ) + + +@pytest.fixture +def patch_request(): + return tflow.tflow( + req=tutils.treq(method=b'PATCH', path=b"/path?query=param") + ) -def req_post(): - return tutils.treq(method=b'POST', headers=()) +class TExport: + def test_get(self, get_request): + raise NotImplementedError() + def test_post(self, post_request): + raise NotImplementedError() -def req_patch(): - return tutils.treq(method=b'PATCH', path=b"/path?query=param") + def test_patch(self, patch_request): + raise NotImplementedError() -class TestExportCurlCommand: - def test_get(self): - flow = tflow.tflow(req=req_get()) +class TestExportCurlCommand(TExport): + def test_get(self, get_request): result = """curl -H 'header:qvalue' -H 'content-length:7' 'http://address:22/path?a=foo&a=bar&b=baz'""" - assert export.curl_command(flow) == result + assert export.curl_command(get_request) == result - def test_post(self): - flow = tflow.tflow(req=req_post()) - result = """curl -X POST 'http://address:22/path' --data-binary 'content'""" - assert export.curl_command(flow) == result + def test_post(self, post_request): + result = "curl -X POST 'http://address:22/path' --data-binary '{}'".format( + str(bytes(range(256)))[2:-1] + ) + assert export.curl_command(post_request) == result - def test_patch(self): - flow = tflow.tflow(req=req_patch()) + def test_patch(self, patch_request): result = """curl -H 'header:qvalue' -H 'content-length:7' -X PATCH 'http://address:22/path?query=param' --data-binary 'content'""" - assert export.curl_command(flow) == result + assert export.curl_command(patch_request) == result -class TestExportPythonCode: - def test_get(self): - flow = tflow.tflow(req=req_get()) - python_equals("mitmproxy/data/test_flow_export/python_get.py", export.python_code(flow)) +class TestExportPythonCode(TExport): + def test_get(self, get_request): + python_equals("mitmproxy/data/test_flow_export/python_get.py", + export.python_code(get_request)) - def test_post(self): - flow = tflow.tflow(req=req_post()) - python_equals("mitmproxy/data/test_flow_export/python_post.py", export.python_code(flow)) + def test_post(self, post_request): + python_equals("mitmproxy/data/test_flow_export/python_post.py", + export.python_code(post_request)) - def test_post_json(self): - p = req_post() - p.content = b'{"name": "example", "email": "example@example.com"}' - p.headers = Headers(content_type="application/json") - flow = tflow.tflow(req=p) - python_equals("mitmproxy/data/test_flow_export/python_post_json.py", export.python_code(flow)) + def test_post_json(self, post_request): + post_request.request.content = b'{"name": "example", "email": "example@example.com"}' + post_request.request.headers = Headers(content_type="application/json") + python_equals("mitmproxy/data/test_flow_export/python_post_json.py", + export.python_code(post_request)) - def test_patch(self): - flow = tflow.tflow(req=req_patch()) - python_equals("mitmproxy/data/test_flow_export/python_patch.py", export.python_code(flow)) + def test_patch(self, patch_request): + python_equals("mitmproxy/data/test_flow_export/python_patch.py", + export.python_code(patch_request)) -class TestExportLocustCode: - def test_get(self): - flow = tflow.tflow(req=req_get()) - python_equals("mitmproxy/data/test_flow_export/locust_get.py", export.locust_code(flow)) +class TestExportLocustCode(TExport): + def test_get(self, get_request): + python_equals("mitmproxy/data/test_flow_export/locust_get.py", + export.locust_code(get_request)) - def test_post(self): - p = req_post() - p.content = b'content' - p.headers = '' - flow = tflow.tflow(req=p) - python_equals("mitmproxy/data/test_flow_export/locust_post.py", export.locust_code(flow)) + def test_post(self, post_request): + post_request.request.content = b'content' + post_request.request.headers.clear() + python_equals("mitmproxy/data/test_flow_export/locust_post.py", + export.locust_code(post_request)) - def test_patch(self): - flow = tflow.tflow(req=req_patch()) - python_equals("mitmproxy/data/test_flow_export/locust_patch.py", export.locust_code(flow)) + def test_patch(self, patch_request): + python_equals("mitmproxy/data/test_flow_export/locust_patch.py", + export.locust_code(patch_request)) -class TestExportLocustTask: - def test_get(self): - flow = tflow.tflow(req=req_get()) - python_equals("mitmproxy/data/test_flow_export/locust_task_get.py", export.locust_task(flow)) +class TestExportLocustTask(TExport): + def test_get(self, get_request): + python_equals("mitmproxy/data/test_flow_export/locust_task_get.py", + export.locust_task(get_request)) - def test_post(self): - flow = tflow.tflow(req=req_post()) - python_equals("mitmproxy/data/test_flow_export/locust_task_post.py", export.locust_task(flow)) + def test_post(self, post_request): + python_equals("mitmproxy/data/test_flow_export/locust_task_post.py", + export.locust_task(post_request)) - def test_patch(self): - flow = tflow.tflow(req=req_patch()) - python_equals("mitmproxy/data/test_flow_export/locust_task_patch.py", export.locust_task(flow)) + def test_patch(self, patch_request): + python_equals("mitmproxy/data/test_flow_export/locust_task_patch.py", + export.locust_task(patch_request)) class TestURL: diff --git a/test/mitmproxy/utils/test_typecheck.py b/test/mitmproxy/utils/test_typecheck.py index d99a914f..fd0c6e0c 100644 --- a/test/mitmproxy/utils/test_typecheck.py +++ b/test/mitmproxy/utils/test_typecheck.py @@ -79,3 +79,9 @@ def test_check_io(): typecheck.check_type("foo", io.StringIO(), typing.IO[str]) with pytest.raises(TypeError): typecheck.check_type("foo", "foo", typing.IO[str]) + + +def test_check_any(): + typecheck.check_type("foo", 42, typing.Any) + typecheck.check_type("foo", object(), typing.Any) + typecheck.check_type("foo", None, typing.Any) diff --git a/web/src/js/__tests__/flow/utilsSpec.js b/web/src/js/__tests__/flow/utilsSpec.js new file mode 100644 index 00000000..2d8f0456 --- /dev/null +++ b/web/src/js/__tests__/flow/utilsSpec.js @@ -0,0 +1,69 @@ +import * as utils from '../../flow/utils' + +describe('MessageUtils', () => { + it('should be possible to get first header', () => { + let msg = { headers: [["foo", "bar"]]} + expect(utils.MessageUtils.get_first_header(msg, "foo")).toEqual("bar") + expect(utils.MessageUtils.get_first_header(msg, "123")).toEqual(undefined) + }) + + it('should be possible to get Content-Type', () => { + let type = "text/html", + msg = { headers: [["Content-Type", type]]} + expect(utils.MessageUtils.getContentType(msg)).toEqual(type) + }) + + it('should be possible to match header', () => { + let h1 = ["foo", "bar"], + msg = {headers : [h1]} + expect(utils.MessageUtils.match_header(msg, /foo/i)).toEqual(h1) + expect(utils.MessageUtils.match_header(msg, /123/i)).toBeFalsy() + }) + + it('should be possible to get content URL', () => { + // request + let msg = "foo", view = "bar", + flow = { request: msg, id: 1} + expect(utils.MessageUtils.getContentURL(flow, msg, view)).toEqual( + "/flows/1/request/content/bar" + ) + expect(utils.MessageUtils.getContentURL(flow, msg, '')).toEqual( + "/flows/1/request/content" + ) + // response + flow = {response: msg, id: 2} + expect(utils.MessageUtils.getContentURL(flow, msg, view)).toEqual( + "/flows/2/response/content/bar" + ) + }) +}) + +describe('RequestUtils', () => { + it('should be possible prettify url', () => { + let request = {port: 4444, scheme: "http", pretty_host: "foo", path: "/bar"} + expect(utils.RequestUtils.pretty_url(request)).toEqual( + "http://foo:4444/bar" + ) + }) +}) + +describe('parseUrl', () => { + it('should be possible to parse url', () => { + let url = "http://foo:4444/bar" + expect(utils.parseUrl(url)).toEqual({ + port: 4444, + scheme: 'http', + host: 'foo', + path: '/bar' + }) + + expect(utils.parseUrl("foo:foo")).toBeFalsy() + }) +}) + +describe('isValidHttpVersion', () => { + it('should be possible to validate http version', () => { + expect(utils.isValidHttpVersion("HTTP/1.1")).toBeTruthy() + expect(utils.isValidHttpVersion("HTTP//1")).toBeFalsy() + }) +}) diff --git a/web/src/js/__tests__/utilsSpec.js b/web/src/js/__tests__/utilsSpec.js new file mode 100644 index 00000000..9a1a0750 --- /dev/null +++ b/web/src/js/__tests__/utilsSpec.js @@ -0,0 +1,95 @@ +import * as utils from '../utils' + +global.fetch = jest.fn() + +describe('formatSize', () => { + it('should return 0 when 0 byte', () => { + expect(utils.formatSize(0)).toEqual('0') + }) + + it('should return formatted size', () => { + expect(utils.formatSize(27104011)).toEqual("25.8mb") + expect(utils.formatSize(1023)).toEqual("1023b") + }) +}) + +describe('formatTimeDelta', () => { + it('should return formatted time', () => { + expect(utils.formatTimeDelta(3600100)).toEqual("1h") + }) +}) + +describe('formatTimeSTamp', () => { + it('should return formatted time', () => { + expect(utils.formatTimeStamp(1483228800)).toEqual("2017-01-01 00:00:00.000") + }) +}) + +describe('reverseString', () => { + it('should return reversed string', () => { + let str1 = "abc", str2="xyz" + expect(utils.reverseString(str1) > utils.reverseString(str2)).toBeTruthy() + }) +}) + +describe('fetchApi', () => { + it('should handle fetch operation', () => { + utils.fetchApi('http://foo/bar', {method: "POST"}) + expect(fetch.mock.calls[0][0]).toEqual( + "http://foo/bar?_xsrf=undefined" + ) + fetch.mockClear() + + utils.fetchApi('http://foo?bar=1', {method: "POST"}) + expect(fetch.mock.calls[0][0]).toEqual( + "http://foo?bar=1&_xsrf=undefined" + ) + + }) + + it('should be possible to do put request', () => { + fetch.mockClear() + utils.fetchApi.put("http://foo", [1, 2, 3], {}) + expect(fetch.mock.calls[0]).toEqual( + [ + "http://foo?_xsrf=undefined", + { + body: "[1,2,3]", + credentials: "same-origin", + headers: { "Content-Type": "application/json" }, + method: "PUT" + }, + ] + ) + }) +}) + +describe('getDiff', () => { + it('should return json object including only the changed keys value pairs', () => { + let obj1 = {a: 1, b:{ foo: 1} , c: [3]}, + obj2 = {a: 1, b:{ foo: 2} , c: [4]} + expect(utils.getDiff(obj1, obj2)).toEqual({ b: {foo: 2}, c:[4]}) + }) +}) + +describe('pure', () => { + let tFunc = function({ className }) { + return (<p className={ className }>foo</p>) + }, + puredFunc = utils.pure(tFunc), + f = new puredFunc('bar') + + it('should display function name', () => { + expect(utils.pure(tFunc).displayName).toEqual('tFunc') + }) + + it('should suggest when should component update', () => { + expect(f.shouldComponentUpdate('foo')).toBeTruthy() + expect(f.shouldComponentUpdate('bar')).toBeFalsy() + }) + + it('should render properties', () => { + expect(f.render()).toEqual(tFunc('bar')) + }) + +}) |