diff options
Diffstat (limited to 'test')
59 files changed, 2115 insertions, 1214 deletions
diff --git a/test/mitmproxy/builtins/__init__.py b/test/mitmproxy/builtins/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/mitmproxy/builtins/__init__.py diff --git a/test/mitmproxy/builtins/test_anticache.py b/test/mitmproxy/builtins/test_anticache.py new file mode 100644 index 00000000..127e1c1a --- /dev/null +++ b/test/mitmproxy/builtins/test_anticache.py @@ -0,0 +1,23 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import anticache +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy.flow import options + + +class TestAntiCache(mastertest.MasterTest): + def test_simple(self): + s = state.State() + m = master.FlowMaster(options.Options(anticache = True), None, s) + sa = anticache.AntiCache() + m.addons.add(sa) + + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + + f = tutils.tflow(resp=True) + f.request.headers["if-modified-since"] = "test" + f.request.headers["if-none-match"] = "test" + self.invoke(m, "request", f) + assert "if-modified-since" not in f.request.headers + assert "if-none-match" not in f.request.headers diff --git a/test/mitmproxy/builtins/test_anticomp.py b/test/mitmproxy/builtins/test_anticomp.py new file mode 100644 index 00000000..601e56c8 --- /dev/null +++ b/test/mitmproxy/builtins/test_anticomp.py @@ -0,0 +1,22 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import anticomp +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy.flow import options + + +class TestAntiComp(mastertest.MasterTest): + def test_simple(self): + s = state.State() + m = master.FlowMaster(options.Options(anticomp = True), None, s) + sa = anticomp.AntiComp() + m.addons.add(sa) + + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + + f = tutils.tflow(resp=True) + + f.request.headers["Accept-Encoding"] = "foobar" + self.invoke(m, "request", f) + assert f.request.headers["Accept-Encoding"] == "identity" diff --git a/test/mitmproxy/builtins/test_dumper.py b/test/mitmproxy/builtins/test_dumper.py new file mode 100644 index 00000000..57e3d036 --- /dev/null +++ b/test/mitmproxy/builtins/test_dumper.py @@ -0,0 +1,86 @@ +from .. import tutils, mastertest +from six.moves import cStringIO as StringIO + +from mitmproxy.builtins import dumper +from mitmproxy.flow import state +from mitmproxy import exceptions +from mitmproxy import dump +from mitmproxy import models +import netlib.tutils +import mock + + +class TestDumper(mastertest.MasterTest): + def test_simple(self): + d = dumper.Dumper() + sio = StringIO() + + d.configure(dump.Options(tfile = sio, flow_detail = 0)) + d.response(tutils.tflow()) + assert not sio.getvalue() + + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + d.response(tutils.tflow()) + assert sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + d.response(tutils.tflow(resp=True)) + assert "<<" in sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + d.response(tutils.tflow(err=True)) + assert "<<" in sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + flow = tutils.tflow() + flow.request = netlib.tutils.treq() + flow.request.stickycookie = True + flow.client_conn = mock.MagicMock() + flow.client_conn.address.host = "foo" + flow.response = netlib.tutils.tresp(content=None) + flow.response.is_replay = True + flow.response.status_code = 300 + d.response(flow) + assert sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + flow = tutils.tflow(resp=netlib.tutils.tresp(content=b"{")) + flow.response.headers["content-type"] = "application/json" + flow.response.status_code = 400 + d.response(flow) + assert sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio)) + flow = tutils.tflow() + flow.request.content = None + flow.response = models.HTTPResponse.wrap(netlib.tutils.tresp()) + flow.response.content = None + d.response(flow) + assert "content missing" in sio.getvalue() + + +class TestContentView(mastertest.MasterTest): + @mock.patch("mitmproxy.contentviews.get_content_view") + def test_contentview(self, get_content_view): + se = exceptions.ContentViewException(""), ("x", iter([])) + get_content_view.side_effect = se + + s = state.State() + sio = StringIO() + m = mastertest.RecordingMaster( + dump.Options( + flow_detail=4, + verbosity=3, + tfile=sio, + ), + None, s + ) + d = dumper.Dumper() + m.addons.add(d) + self.invoke(m, "response", tutils.tflow()) + assert "Content viewer failed" in m.event_log[0][1] diff --git a/test/mitmproxy/builtins/test_filestreamer.py b/test/mitmproxy/builtins/test_filestreamer.py new file mode 100644 index 00000000..002006b7 --- /dev/null +++ b/test/mitmproxy/builtins/test_filestreamer.py @@ -0,0 +1,46 @@ +from __future__ import absolute_import, print_function, division + +from .. import tutils, mastertest + +import os.path + +from mitmproxy.builtins import filestreamer +from mitmproxy.flow import master, FlowReader +from mitmproxy.flow import state +from mitmproxy.flow import options + + +class TestStream(mastertest.MasterTest): + def test_stream(self): + with tutils.tmpdir() as tdir: + p = os.path.join(tdir, "foo") + + def r(): + r = FlowReader(open(p, "rb")) + return list(r.stream()) + + s = state.State() + m = master.FlowMaster( + options.Options( + outfile = (p, "wb") + ), + None, + s + ) + sa = filestreamer.FileStreamer() + + m.addons.add(sa) + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + self.invoke(m, "response", f) + m.addons.remove(sa) + + assert r()[0].response + + m.options.outfile = (p, "ab") + + m.addons.add(sa) + f = tutils.tflow() + self.invoke(m, "request", f) + m.addons.remove(sa) + assert not r()[1].response diff --git a/test/mitmproxy/builtins/test_replace.py b/test/mitmproxy/builtins/test_replace.py new file mode 100644 index 00000000..f8010bec --- /dev/null +++ b/test/mitmproxy/builtins/test_replace.py @@ -0,0 +1,52 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import replace +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy.flow import options + + +class TestReplace(mastertest.MasterTest): + def test_configure(self): + r = replace.Replace() + r.configure(options.Options( + replacements=[("one", "two", "three")] + )) + tutils.raises( + "invalid filter pattern", + r.configure, + options.Options( + replacements=[("~b", "two", "three")] + ) + ) + tutils.raises( + "invalid regular expression", + r.configure, + options.Options( + replacements=[("foo", "+", "three")] + ) + ) + + def test_simple(self): + s = state.State() + m = master.FlowMaster( + options.Options( + replacements = [ + ("~q", "foo", "bar"), + ("~s", "foo", "bar"), + ] + ), + None, + s + ) + sa = replace.Replace() + m.addons.add(sa) + + f = tutils.tflow() + f.request.content = b"foo" + self.invoke(m, "request", f) + assert f.request.content == b"bar" + + f = tutils.tflow(resp=True) + f.response.content = b"foo" + self.invoke(m, "response", f) + assert f.response.content == b"bar" diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py new file mode 100644 index 00000000..c9616249 --- /dev/null +++ b/test/mitmproxy/builtins/test_script.py @@ -0,0 +1,191 @@ +import time + +from mitmproxy.builtins import script +from mitmproxy import exceptions +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy.flow import options + +from .. import tutils, mastertest + + +class TestParseCommand: + def test_empty_command(self): + with tutils.raises(exceptions.AddonError): + script.parse_command("") + + with tutils.raises(exceptions.AddonError): + script.parse_command(" ") + + def test_no_script_file(self): + with tutils.raises("not found"): + script.parse_command("notfound") + + with tutils.tmpdir() as dir: + with tutils.raises("not a file"): + script.parse_command(dir) + + def test_parse_args(self): + with tutils.chdir(tutils.test_data.dirname): + assert script.parse_command("data/addonscripts/recorder.py") == ("data/addonscripts/recorder.py", []) + assert script.parse_command("data/addonscripts/recorder.py foo bar") == ("data/addonscripts/recorder.py", ["foo", "bar"]) + assert script.parse_command("data/addonscripts/recorder.py 'foo bar'") == ("data/addonscripts/recorder.py", ["foo bar"]) + + @tutils.skip_not_windows + def test_parse_windows(self): + with tutils.chdir(tutils.test_data.dirname): + assert script.parse_command( + "data\\addonscripts\\recorder.py" + ) == ("data\\addonscripts\\recorder.py", []) + assert script.parse_command( + "data\\addonscripts\\recorder.py 'foo \\ bar'" + ) == ("data\\addonscripts\\recorder.py", ['foo \\ bar']) + + +def test_load_script(): + ns = script.load_script( + tutils.test_data.path( + "data/addonscripts/recorder.py" + ), [] + ) + assert ns["configure"] + + +class TestScript(mastertest.MasterTest): + def test_simple(self): + s = state.State() + m = master.FlowMaster(options.Options(), None, s) + sc = script.Script( + tutils.test_data.path( + "data/addonscripts/recorder.py" + ) + ) + m.addons.add(sc) + assert sc.ns["call_log"] == [ + ("solo", "start", (), {}), + ("solo", "configure", (options.Options(),), {}) + ] + + sc.ns["call_log"] = [] + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + + recf = sc.ns["call_log"][0] + assert recf[1] == "request" + + def test_reload(self): + s = state.State() + m = mastertest.RecordingMaster(options.Options(), None, s) + with tutils.tmpdir(): + with open("foo.py", "w"): + pass + sc = script.Script("foo.py") + m.addons.add(sc) + + for _ in range(100): + with open("foo.py", "a") as f: + f.write(".") + m.addons.invoke_with_context(sc, "tick") + time.sleep(0.1) + if m.event_log: + return + raise AssertionError("Change event not detected.") + + def test_exception(self): + s = state.State() + m = mastertest.RecordingMaster(options.Options(), None, s) + sc = script.Script( + tutils.test_data.path("data/addonscripts/error.py") + ) + m.addons.add(sc) + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + assert m.event_log[0][0] == "error" + + def test_duplicate_flow(self): + s = state.State() + fm = master.FlowMaster(None, None, s) + fm.addons.add( + script.Script( + tutils.test_data.path("data/addonscripts/duplicate_flow.py") + ) + ) + f = tutils.tflow() + fm.request(f) + assert fm.state.flow_count() == 2 + assert not fm.state.view[0].request.is_replay + assert fm.state.view[1].request.is_replay + + +class TestScriptLoader(mastertest.MasterTest): + def test_simple(self): + s = state.State() + o = options.Options(scripts=[]) + m = master.FlowMaster(o, None, s) + sc = script.ScriptLoader() + m.addons.add(sc) + assert len(m.addons) == 1 + o.update( + scripts = [ + tutils.test_data.path("data/addonscripts/recorder.py") + ] + ) + assert len(m.addons) == 2 + o.update(scripts = []) + assert len(m.addons) == 1 + + def test_dupes(self): + s = state.State() + o = options.Options(scripts=["one", "one"]) + m = master.FlowMaster(o, None, s) + sc = script.ScriptLoader() + tutils.raises(exceptions.OptionsError, m.addons.add, sc) + + def test_order(self): + rec = tutils.test_data.path("data/addonscripts/recorder.py") + + s = state.State() + o = options.Options( + scripts = [ + "%s %s" % (rec, "a"), + "%s %s" % (rec, "b"), + "%s %s" % (rec, "c"), + ] + ) + m = mastertest.RecordingMaster(o, None, s) + sc = script.ScriptLoader() + m.addons.add(sc) + + debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"] + assert debug == [ + ('debug', 'a start'), ('debug', 'a configure'), + ('debug', 'b start'), ('debug', 'b configure'), + ('debug', 'c start'), ('debug', 'c configure') + ] + m.event_log[:] = [] + + o.scripts = [ + "%s %s" % (rec, "c"), + "%s %s" % (rec, "a"), + "%s %s" % (rec, "b"), + ] + debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"] + assert debug == [ + ('debug', 'c configure'), + ('debug', 'a configure'), + ('debug', 'b configure'), + ] + m.event_log[:] = [] + + o.scripts = [ + "%s %s" % (rec, "x"), + "%s %s" % (rec, "a"), + ] + debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"] + assert debug == [ + ('debug', 'c done'), + ('debug', 'b done'), + ('debug', 'x start'), + ('debug', 'x configure'), + ('debug', 'a configure'), + ] diff --git a/test/mitmproxy/builtins/test_setheaders.py b/test/mitmproxy/builtins/test_setheaders.py new file mode 100644 index 00000000..1a8d048c --- /dev/null +++ b/test/mitmproxy/builtins/test_setheaders.py @@ -0,0 +1,64 @@ +from .. import tutils, mastertest + +from mitmproxy.builtins import setheaders +from mitmproxy.flow import state +from mitmproxy.flow import options + + +class TestSetHeaders(mastertest.MasterTest): + def mkmaster(self, **opts): + s = state.State() + m = mastertest.RecordingMaster(options.Options(**opts), None, s) + sh = setheaders.SetHeaders() + m.addons.add(sh) + return m, sh + + def test_configure(self): + sh = setheaders.SetHeaders() + tutils.raises( + "invalid setheader filter pattern", + sh.configure, + options.Options( + setheaders = [("~b", "one", "two")] + ) + ) + + def test_setheaders(self): + m, sh = self.mkmaster( + setheaders = [ + ("~q", "one", "two"), + ("~s", "one", "three") + ] + ) + f = tutils.tflow() + f.request.headers["one"] = "xxx" + self.invoke(m, "request", f) + assert f.request.headers["one"] == "two" + + f = tutils.tflow(resp=True) + f.response.headers["one"] = "xxx" + self.invoke(m, "response", f) + assert f.response.headers["one"] == "three" + + m, sh = self.mkmaster( + setheaders = [ + ("~s", "one", "two"), + ("~s", "one", "three") + ] + ) + f = tutils.tflow(resp=True) + f.request.headers["one"] = "xxx" + f.response.headers["one"] = "xxx" + self.invoke(m, "response", f) + assert f.response.headers.get_all("one") == ["two", "three"] + + m, sh = self.mkmaster( + setheaders = [ + ("~q", "one", "two"), + ("~q", "one", "three") + ] + ) + f = tutils.tflow() + f.request.headers["one"] = "xxx" + self.invoke(m, "request", f) + assert f.request.headers.get_all("one") == ["two", "three"] diff --git a/test/mitmproxy/builtins/test_stickyauth.py b/test/mitmproxy/builtins/test_stickyauth.py new file mode 100644 index 00000000..1e617402 --- /dev/null +++ b/test/mitmproxy/builtins/test_stickyauth.py @@ -0,0 +1,23 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import stickyauth +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy.flow import options + + +class TestStickyAuth(mastertest.MasterTest): + def test_simple(self): + s = state.State() + m = master.FlowMaster(options.Options(stickyauth = ".*"), None, s) + sa = stickyauth.StickyAuth() + m.addons.add(sa) + + f = tutils.tflow(resp=True) + f.request.headers["authorization"] = "foo" + self.invoke(m, "request", f) + + assert "address" in sa.hosts + + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + assert f.request.headers["authorization"] == "foo" diff --git a/test/mitmproxy/builtins/test_stickycookie.py b/test/mitmproxy/builtins/test_stickycookie.py new file mode 100644 index 00000000..b8d703bd --- /dev/null +++ b/test/mitmproxy/builtins/test_stickycookie.py @@ -0,0 +1,131 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import stickycookie +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy.flow import options +from netlib import tutils as ntutils + + +def test_domain_match(): + assert stickycookie.domain_match("www.google.com", ".google.com") + assert stickycookie.domain_match("google.com", ".google.com") + + +class TestStickyCookie(mastertest.MasterTest): + def mk(self): + s = state.State() + m = master.FlowMaster(options.Options(stickycookie = ".*"), None, s) + sc = stickycookie.StickyCookie() + m.addons.add(sc) + return s, m, sc + + def test_config(self): + sc = stickycookie.StickyCookie() + tutils.raises( + "invalid filter", + sc.configure, + options.Options(stickycookie = "~b") + ) + + def test_simple(self): + s, m, sc = self.mk() + m.addons.add(sc) + + f = tutils.tflow(resp=True) + f.response.headers["set-cookie"] = "foo=bar" + self.invoke(m, "request", f) + + f.reply.acked = False + self.invoke(m, "response", f) + + assert sc.jar + assert "cookie" not in f.request.headers + + f = f.copy() + f.reply.acked = False + self.invoke(m, "request", f) + assert f.request.headers["cookie"] == "foo=bar" + + def _response(self, s, m, sc, cookie, host): + f = tutils.tflow(req=ntutils.treq(host=host, port=80), resp=True) + f.response.headers["Set-Cookie"] = cookie + self.invoke(m, "response", f) + return f + + def test_response(self): + s, m, sc = self.mk() + + c = "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " \ + "Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; " + + self._response(s, m, sc, c, "host") + assert not sc.jar.keys() + + self._response(s, m, sc, c, "www.google.com") + assert sc.jar.keys() + + sc.jar.clear() + self._response( + s, m, sc, "SSID=mooo", "www.google.com" + ) + assert list(sc.jar.keys())[0] == ('www.google.com', 80, '/') + + def test_response_multiple(self): + s, m, sc = self.mk() + + # Test setting of multiple cookies + c1 = "somecookie=test; Path=/" + c2 = "othercookie=helloworld; Path=/" + f = self._response(s, m, sc, c1, "www.google.com") + f.response.headers["Set-Cookie"] = c2 + self.invoke(m, "response", f) + googlekey = list(sc.jar.keys())[0] + assert len(sc.jar[googlekey].keys()) == 2 + + def test_response_weird(self): + s, m, sc = self.mk() + + # Test setting of weird cookie keys + f = tutils.tflow(req=ntutils.treq(host="www.google.com", port=80), resp=True) + cs = [ + "foo/bar=hello", + "foo:bar=world", + "foo@bar=fizz", + "foo,bar=buzz", + ] + for c in cs: + f.response.headers["Set-Cookie"] = c + self.invoke(m, "response", f) + googlekey = list(sc.jar.keys())[0] + assert len(sc.jar[googlekey].keys()) == len(cs) + + def test_response_overwrite(self): + s, m, sc = self.mk() + + # Test overwriting of a cookie value + c1 = "somecookie=helloworld; Path=/" + c2 = "somecookie=newvalue; Path=/" + f = self._response(s, m, sc, c1, "www.google.com") + f.response.headers["Set-Cookie"] = c2 + self.invoke(m, "response", f) + googlekey = list(sc.jar.keys())[0] + assert len(sc.jar[googlekey].keys()) == 1 + assert list(sc.jar[googlekey]["somecookie"].items())[0][1] == "newvalue" + + def test_response_delete(self): + s, m, sc = self.mk() + + # Test that a cookie is be deleted + # by setting the expire time in the past + f = self._response(s, m, sc, "duffer=zafar; Path=/", "www.google.com") + f.response.headers["Set-Cookie"] = "duffer=; Expires=Thu, 01-Jan-1970 00:00:00 GMT" + self.invoke(m, "response", f) + assert not sc.jar.keys() + + def test_request(self): + s, m, sc = self.mk() + + f = self._response(s, m, sc, "SSID=mooo", "www.google.com") + assert "cookie" not in f.request.headers + self.invoke(m, "request", f) + assert "cookie" in f.request.headers diff --git a/test/mitmproxy/console/test_master.py b/test/mitmproxy/console/test_master.py index 33261c28..b84e4c1c 100644 --- a/test/mitmproxy/console/test_master.py +++ b/test/mitmproxy/console/test_master.py @@ -111,12 +111,14 @@ def test_options(): class TestMaster(mastertest.MasterTest): - def mkmaster(self, filt, **options): - o = console.master.Options(filtstr=filt, **options) + def mkmaster(self, **options): + if "verbosity" not in options: + options["verbosity"] = 0 + o = console.master.Options(**options) return console.master.ConsoleMaster(None, o) def test_basic(self): - m = self.mkmaster(None) + m = self.mkmaster() for i in (1, 2, 3): - self.dummy_cycle(m, 1, "") + self.dummy_cycle(m, 1, b"") assert len(m.state.flows) == i diff --git a/test/mitmproxy/data/scripts/concurrent_decorator.py b/test/mitmproxy/data/addonscripts/concurrent_decorator.py index e017f605..a56c2af1 100644 --- a/test/mitmproxy/data/scripts/concurrent_decorator.py +++ b/test/mitmproxy/data/addonscripts/concurrent_decorator.py @@ -1,7 +1,6 @@ import time from mitmproxy.script import concurrent - @concurrent -def request(context, flow): +def request(flow): time.sleep(0.1) diff --git a/test/mitmproxy/data/scripts/concurrent_decorator_err.py b/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py index 349e5dd6..756869c8 100644 --- a/test/mitmproxy/data/scripts/concurrent_decorator_err.py +++ b/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py @@ -2,5 +2,5 @@ from mitmproxy.script import concurrent @concurrent -def start(context): +def start(): pass diff --git a/test/mitmproxy/data/addonscripts/duplicate_flow.py b/test/mitmproxy/data/addonscripts/duplicate_flow.py new file mode 100644 index 00000000..b466423c --- /dev/null +++ b/test/mitmproxy/data/addonscripts/duplicate_flow.py @@ -0,0 +1,6 @@ +from mitmproxy import ctx + + +def request(flow): + f = ctx.master.duplicate_flow(flow) + ctx.master.replay_request(f, block=True) diff --git a/test/mitmproxy/data/addonscripts/error.py b/test/mitmproxy/data/addonscripts/error.py new file mode 100644 index 00000000..8ece9fce --- /dev/null +++ b/test/mitmproxy/data/addonscripts/error.py @@ -0,0 +1,7 @@ + +def mkerr(): + raise ValueError("Error!") + + +def request(flow): + mkerr() diff --git a/test/mitmproxy/data/addonscripts/recorder.py b/test/mitmproxy/data/addonscripts/recorder.py new file mode 100644 index 00000000..b6ac8d89 --- /dev/null +++ b/test/mitmproxy/data/addonscripts/recorder.py @@ -0,0 +1,25 @@ +from mitmproxy import controller +from mitmproxy import ctx +import sys + +call_log = [] + +if len(sys.argv) > 1: + name = sys.argv[1] +else: + name = "solo" + +# Keep a log of all possible event calls +evts = list(controller.Events) + ["configure"] +for i in evts: + def mkprox(): + evt = i + + def prox(*args, **kwargs): + lg = (name, evt, args, kwargs) + if evt != "log": + ctx.log.info(str(lg)) + call_log.append(lg) + ctx.log.debug("%s %s" % (name, evt)) + return prox + globals()[i] = mkprox() diff --git a/test/mitmproxy/data/scripts/stream_modify.py b/test/mitmproxy/data/addonscripts/stream_modify.py index 8221b0dd..bc616342 100644 --- a/test/mitmproxy/data/scripts/stream_modify.py +++ b/test/mitmproxy/data/addonscripts/stream_modify.py @@ -1,7 +1,8 @@ + def modify(chunks): for chunk in chunks: yield chunk.replace(b"foo", b"bar") -def responseheaders(context, flow): +def responseheaders(flow): flow.response.stream = modify diff --git a/test/mitmproxy/data/scripts/tcp_stream_modify.py b/test/mitmproxy/data/addonscripts/tcp_stream_modify.py index 0965beba..af4ccf7e 100644 --- a/test/mitmproxy/data/scripts/tcp_stream_modify.py +++ b/test/mitmproxy/data/addonscripts/tcp_stream_modify.py @@ -1,4 +1,5 @@ -def tcp_message(ctx, flow): + +def tcp_message(flow): message = flow.messages[-1] if not message.from_client: message.content = message.content.replace(b"foo", b"bar") diff --git a/test/mitmproxy/data/dumpfile-010 b/test/mitmproxy/data/dumpfile-010 Binary files differnew file mode 100644 index 00000000..435795bf --- /dev/null +++ b/test/mitmproxy/data/dumpfile-010 diff --git a/test/mitmproxy/data/dumpfile-011 b/test/mitmproxy/data/dumpfile-011 Binary files differnew file mode 100644 index 00000000..2534ad89 --- /dev/null +++ b/test/mitmproxy/data/dumpfile-011 diff --git a/test/mitmproxy/data/dumpfile-012 b/test/mitmproxy/data/dumpfile-012 deleted file mode 100644 index 49c2350d..00000000 --- a/test/mitmproxy/data/dumpfile-012 +++ /dev/null @@ -1,35 +0,0 @@ -4092:8:response,491:11:httpversion,8:1:1#1:1#]13:timestamp_end,14:1449080668.874^3:msg,12:Not Modified,15:timestamp_start,14:1449080668.863^7:headers,330:35:13:Cache-Control,14:max-age=604800,]40:4:Date,29:Wed, 02 Dec 2015 18:24:32 GMT,]32:4:Etag,21:"359670651+gzip+gzip",]43:7:Expires,29:Wed, 09 Dec 2015 18:24:32 GMT,]50:13:Last-Modified,29:Fri, 09 Aug 2013 23:54:35 GMT,]27:6:Server,14:ECS (lga/1312),]26:4:Vary,15:Accept-Encoding,]16:7:X-Cache,3:HIT,]25:17:x-ec-custom-error,1:1,]]7:content,0:,4:code,3:304#}4:type,4:http,2:id,36:d209a4fc-8e12-43cb-9250-b0b052d2caf8,5:error,0:~7:version,9:1:0#2:12#]11:client_conn,208:15:ssl_established,4:true!10:clientcert,0:~13:timestamp_end,0:~19:timestamp_ssl_setup,14:1449080668.754^7:address,53:7:address,20:9:127.0.0.1,5:58199#]8:use_ipv6,5:false!}15:timestamp_start,14:1449080666.523^}11:server_conn,2479:15:ssl_established,4:true!14:source_address,57:7:address,24:12:10.67.56.236,5:58201#]8:use_ipv6,5:false!}13:timestamp_end,0:~7:address,54:7:address,21:11:example.com,3:443#]8:use_ipv6,5:false!}15:timestamp_start,14:1449080668.046^3:sni,11:example.com,4:cert,2122:-----BEGIN CERTIFICATE----- -MIIF8jCCBNqgAwIBAgIQDmTF+8I2reFLFyrrQceMsDANBgkqhkiG9w0BAQsFADBw -MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMS8wLQYDVQQDEyZEaWdpQ2VydCBTSEEyIEhpZ2ggQXNz -dXJhbmNlIFNlcnZlciBDQTAeFw0xNTExMDMwMDAwMDBaFw0xODExMjgxMjAwMDBa -MIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxML -TG9zIEFuZ2VsZXMxPDA6BgNVBAoTM0ludGVybmV0IENvcnBvcmF0aW9uIGZvciBB -c3NpZ25lZCBOYW1lcyBhbmQgTnVtYmVyczETMBEGA1UECxMKVGVjaG5vbG9neTEY -MBYGA1UEAxMPd3d3LmV4YW1wbGUub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAs0CWL2FjPiXBl61lRfvvE0KzLJmG9LWAC3bcBjgsH6NiVVo2dt6u -Xfzi5bTm7F3K7srfUBYkLO78mraM9qizrHoIeyofrV/n+pZZJauQsPjCPxMEJnRo -D8Z4KpWKX0LyDu1SputoI4nlQ/htEhtiQnuoBfNZxF7WxcxGwEsZuS1KcXIkHl5V -RJOreKFHTaXcB1qcZ/QRaBIv0yhxvK1yBTwWddT4cli6GfHcCe3xGMaSL328Fgs3 -jYrvG29PueB6VJi/tbbPu6qTfwp/H1brqdjh29U52Bhb0fJkM9DWxCP/Cattcc7a -z8EXnCO+LK8vkhw/kAiJWPKx4RBvgy73nwIDAQABo4ICUDCCAkwwHwYDVR0jBBgw -FoAUUWj/kK8CB3U8zNllZGKiErhZcjswHQYDVR0OBBYEFKZPYB4fLdHn8SOgKpUW -5Oia6m5IMIGBBgNVHREEejB4gg93d3cuZXhhbXBsZS5vcmeCC2V4YW1wbGUuY29t -ggtleGFtcGxlLmVkdYILZXhhbXBsZS5uZXSCC2V4YW1wbGUub3Jngg93d3cuZXhh -bXBsZS5jb22CD3d3dy5leGFtcGxlLmVkdYIPd3d3LmV4YW1wbGUubmV0MA4GA1Ud -DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0f -BG4wbDA0oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItaGEtc2Vy -dmVyLWc0LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTIt -aGEtc2VydmVyLWc0LmNybDBMBgNVHSAERTBDMDcGCWCGSAGG/WwBATAqMCgGCCsG -AQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAgGBmeBDAECAjCB -gwYIKwYBBQUHAQEEdzB1MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2Vy -dC5jb20wTQYIKwYBBQUHMAKGQWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9E -aWdpQ2VydFNIQTJIaWdoQXNzdXJhbmNlU2VydmVyQ0EuY3J0MAwGA1UdEwEB/wQC -MAAwDQYJKoZIhvcNAQELBQADggEBAISomhGn2L0LJn5SJHuyVZ3qMIlRCIdvqe0Q -6ls+C8ctRwRO3UU3x8q8OH+2ahxlQmpzdC5al4XQzJLiLjiJ2Q1p+hub8MFiMmVP -PZjb2tZm2ipWVuMRM+zgpRVM6nVJ9F3vFfUSHOb4/JsEIUvPY+d8/Krc+kPQwLvy -ieqRbcuFjmqfyPmUv1U9QoI4TQikpw7TZU0zYZANP4C/gj4Ry48/znmUaRvy2kvI -l7gRQ21qJTK5suoiYoYNo3J9T+pXPGU7Lydz/HwW+w0DpArtAaukI8aNX4ohFUKS -wDSiIIWIWJiJGbEeIO0TIFwEVWTOnbNl/faPXpk5IRXicapqiII= ------END CERTIFICATE----- -,19:timestamp_ssl_setup,14:1449080668.358^5:state,0:]19:timestamp_tcp_setup,14:1449080668.177^}11:intercepted,5:false!7:request,727:9:is_replay,5:false!4:port,3:443#6:scheme,5:https,6:method,3:GET,4:path,1:/,8:form_out,8:relative,11:httpversion,8:1:1#1:1#]4:host,11:example.com,7:headers,460:22:4:Host,11:example.com,]91:10:User-Agent,73:Mozilla/5.0 (Windows NT 10.0; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0,]76:6:Accept,63:text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8,]46:15:Accept-Language,23:de,en-US;q=0.7,en;q=0.3,]36:15:Accept-Encoding,13:gzip, deflate,]28:10:Connection,10:keep-alive,]54:17:If-Modified-Since,29:Fri, 09 Aug 2013 23:54:35 GMT,]42:13:If-None-Match,21:"359670651+gzip+gzip",]29:13:Cache-Control,9:max-age=0,]]7:content,0:,7:form_in,8:relative,15:timestamp_start,14:1449080668.754^13:timestamp_end,14:1449080668.757^}}
\ No newline at end of file diff --git a/test/mitmproxy/data/dumpfile-013 b/test/mitmproxy/data/dumpfile-013 deleted file mode 100644 index ede06f23..00000000 --- a/test/mitmproxy/data/dumpfile-013 +++ /dev/null @@ -1,35 +0,0 @@ -4092:8:response,491:11:httpversion,8:1:1#1:1#]13:timestamp_end,14:1449080668.874^3:msg,12:Not Modified,15:timestamp_start,14:1449080668.863^7:headers,330:35:13:Cache-Control,14:max-age=604800,]40:4:Date,29:Wed, 02 Dec 2015 18:24:32 GMT,]32:4:Etag,21:"359670651+gzip+gzip",]43:7:Expires,29:Wed, 09 Dec 2015 18:24:32 GMT,]50:13:Last-Modified,29:Fri, 09 Aug 2013 23:54:35 GMT,]27:6:Server,14:ECS (lga/1312),]26:4:Vary,15:Accept-Encoding,]16:7:X-Cache,3:HIT,]25:17:x-ec-custom-error,1:1,]]7:content,0:,4:code,3:304#}4:type,4:http,2:id,36:d209a4fc-8e12-43cb-9250-b0b052d2caf8,5:error,0:~7:version,9:1:0#2:13#]11:client_conn,208:15:ssl_established,4:true!10:clientcert,0:~13:timestamp_end,0:~19:timestamp_ssl_setup,14:1449080668.754^7:address,53:7:address,20:9:127.0.0.1,5:58199#]8:use_ipv6,5:false!}15:timestamp_start,14:1449080666.523^}11:server_conn,2479:15:ssl_established,4:true!14:source_address,57:7:address,24:12:10.67.56.236,5:58201#]8:use_ipv6,5:false!}13:timestamp_end,0:~7:address,54:7:address,21:11:example.com,3:443#]8:use_ipv6,5:false!}15:timestamp_start,14:1449080668.046^3:sni,11:example.com,4:cert,2122:-----BEGIN CERTIFICATE----- -MIIF8jCCBNqgAwIBAgIQDmTF+8I2reFLFyrrQceMsDANBgkqhkiG9w0BAQsFADBw -MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMS8wLQYDVQQDEyZEaWdpQ2VydCBTSEEyIEhpZ2ggQXNz -dXJhbmNlIFNlcnZlciBDQTAeFw0xNTExMDMwMDAwMDBaFw0xODExMjgxMjAwMDBa -MIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxML -TG9zIEFuZ2VsZXMxPDA6BgNVBAoTM0ludGVybmV0IENvcnBvcmF0aW9uIGZvciBB -c3NpZ25lZCBOYW1lcyBhbmQgTnVtYmVyczETMBEGA1UECxMKVGVjaG5vbG9neTEY -MBYGA1UEAxMPd3d3LmV4YW1wbGUub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAs0CWL2FjPiXBl61lRfvvE0KzLJmG9LWAC3bcBjgsH6NiVVo2dt6u -Xfzi5bTm7F3K7srfUBYkLO78mraM9qizrHoIeyofrV/n+pZZJauQsPjCPxMEJnRo -D8Z4KpWKX0LyDu1SputoI4nlQ/htEhtiQnuoBfNZxF7WxcxGwEsZuS1KcXIkHl5V -RJOreKFHTaXcB1qcZ/QRaBIv0yhxvK1yBTwWddT4cli6GfHcCe3xGMaSL328Fgs3 -jYrvG29PueB6VJi/tbbPu6qTfwp/H1brqdjh29U52Bhb0fJkM9DWxCP/Cattcc7a -z8EXnCO+LK8vkhw/kAiJWPKx4RBvgy73nwIDAQABo4ICUDCCAkwwHwYDVR0jBBgw -FoAUUWj/kK8CB3U8zNllZGKiErhZcjswHQYDVR0OBBYEFKZPYB4fLdHn8SOgKpUW -5Oia6m5IMIGBBgNVHREEejB4gg93d3cuZXhhbXBsZS5vcmeCC2V4YW1wbGUuY29t -ggtleGFtcGxlLmVkdYILZXhhbXBsZS5uZXSCC2V4YW1wbGUub3Jngg93d3cuZXhh -bXBsZS5jb22CD3d3dy5leGFtcGxlLmVkdYIPd3d3LmV4YW1wbGUubmV0MA4GA1Ud -DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0f -BG4wbDA0oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItaGEtc2Vy -dmVyLWc0LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTIt -aGEtc2VydmVyLWc0LmNybDBMBgNVHSAERTBDMDcGCWCGSAGG/WwBATAqMCgGCCsG -AQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAgGBmeBDAECAjCB -gwYIKwYBBQUHAQEEdzB1MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2Vy -dC5jb20wTQYIKwYBBQUHMAKGQWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9E -aWdpQ2VydFNIQTJIaWdoQXNzdXJhbmNlU2VydmVyQ0EuY3J0MAwGA1UdEwEB/wQC -MAAwDQYJKoZIhvcNAQELBQADggEBAISomhGn2L0LJn5SJHuyVZ3qMIlRCIdvqe0Q -6ls+C8ctRwRO3UU3x8q8OH+2ahxlQmpzdC5al4XQzJLiLjiJ2Q1p+hub8MFiMmVP -PZjb2tZm2ipWVuMRM+zgpRVM6nVJ9F3vFfUSHOb4/JsEIUvPY+d8/Krc+kPQwLvy -ieqRbcuFjmqfyPmUv1U9QoI4TQikpw7TZU0zYZANP4C/gj4Ry48/znmUaRvy2kvI -l7gRQ21qJTK5suoiYoYNo3J9T+pXPGU7Lydz/HwW+w0DpArtAaukI8aNX4ohFUKS -wDSiIIWIWJiJGbEeIO0TIFwEVWTOnbNl/faPXpk5IRXicapqiII= ------END CERTIFICATE----- -,19:timestamp_ssl_setup,14:1449080668.358^5:state,0:]19:timestamp_tcp_setup,14:1449080668.177^}11:intercepted,5:false!7:request,727:9:is_replay,5:false!4:port,3:443#6:scheme,5:https,6:method,3:GET,4:path,1:/,8:form_out,8:relative,11:httpversion,8:1:1#1:1#]4:host,11:example.com,7:headers,460:22:4:Host,11:example.com,]91:10:User-Agent,73:Mozilla/5.0 (Windows NT 10.0; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0,]76:6:Accept,63:text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8,]46:15:Accept-Language,23:de,en-US;q=0.7,en;q=0.3,]36:15:Accept-Encoding,13:gzip, deflate,]28:10:Connection,10:keep-alive,]54:17:If-Modified-Since,29:Fri, 09 Aug 2013 23:54:35 GMT,]42:13:If-None-Match,21:"359670651+gzip+gzip",]29:13:Cache-Control,9:max-age=0,]]7:content,0:,7:form_in,8:relative,15:timestamp_start,14:1449080668.754^13:timestamp_end,14:1449080668.757^}}
\ No newline at end of file diff --git a/test/mitmproxy/data/scripts/a.py b/test/mitmproxy/data/scripts/a.py deleted file mode 100644 index 33dbaa64..00000000 --- a/test/mitmproxy/data/scripts/a.py +++ /dev/null @@ -1,20 +0,0 @@ -import sys - -from a_helper import parser - -var = 0 - - -def start(ctx): - global var - var = parser.parse_args(sys.argv[1:]).var - - -def here(ctx): - global var - var += 1 - return var - - -def errargs(): - pass diff --git a/test/mitmproxy/data/scripts/a_helper.py b/test/mitmproxy/data/scripts/a_helper.py deleted file mode 100644 index e1f1c649..00000000 --- a/test/mitmproxy/data/scripts/a_helper.py +++ /dev/null @@ -1,4 +0,0 @@ -import argparse - -parser = argparse.ArgumentParser() -parser.add_argument('--var', type=int) diff --git a/test/mitmproxy/data/scripts/all.py b/test/mitmproxy/data/scripts/all.py index dad2aade..bf8e93ec 100644 --- a/test/mitmproxy/data/scripts/all.py +++ b/test/mitmproxy/data/scripts/all.py @@ -1,36 +1,37 @@ +import mitmproxy log = [] -def clientconnect(ctx, cc): - ctx.log("XCLIENTCONNECT") +def clientconnect(cc): + mitmproxy.ctx.log("XCLIENTCONNECT") log.append("clientconnect") -def serverconnect(ctx, cc): - ctx.log("XSERVERCONNECT") +def serverconnect(cc): + mitmproxy.ctx.log("XSERVERCONNECT") log.append("serverconnect") -def request(ctx, f): - ctx.log("XREQUEST") +def request(f): + mitmproxy.ctx.log("XREQUEST") log.append("request") -def response(ctx, f): - ctx.log("XRESPONSE") +def response(f): + mitmproxy.ctx.log("XRESPONSE") log.append("response") -def responseheaders(ctx, f): - ctx.log("XRESPONSEHEADERS") +def responseheaders(f): + mitmproxy.ctx.log("XRESPONSEHEADERS") log.append("responseheaders") -def clientdisconnect(ctx, cc): - ctx.log("XCLIENTDISCONNECT") +def clientdisconnect(cc): + mitmproxy.ctx.log("XCLIENTDISCONNECT") log.append("clientdisconnect") -def error(ctx, cc): - ctx.log("XERROR") +def error(cc): + mitmproxy.ctx.log("XERROR") log.append("error") diff --git a/test/mitmproxy/data/scripts/duplicate_flow.py b/test/mitmproxy/data/scripts/duplicate_flow.py deleted file mode 100644 index e13af786..00000000 --- a/test/mitmproxy/data/scripts/duplicate_flow.py +++ /dev/null @@ -1,4 +0,0 @@ - -def request(ctx, f): - f = ctx.duplicate_flow(f) - ctx.replay_request(f) diff --git a/test/mitmproxy/data/scripts/loaderr.py b/test/mitmproxy/data/scripts/loaderr.py deleted file mode 100644 index 8dc4d56d..00000000 --- a/test/mitmproxy/data/scripts/loaderr.py +++ /dev/null @@ -1,3 +0,0 @@ - - -a = x diff --git a/test/mitmproxy/data/scripts/reqerr.py b/test/mitmproxy/data/scripts/reqerr.py deleted file mode 100644 index e7c503a8..00000000 --- a/test/mitmproxy/data/scripts/reqerr.py +++ /dev/null @@ -1,2 +0,0 @@ -def request(ctx, r): - raise ValueError diff --git a/test/mitmproxy/data/scripts/starterr.py b/test/mitmproxy/data/scripts/starterr.py deleted file mode 100644 index 82d773bd..00000000 --- a/test/mitmproxy/data/scripts/starterr.py +++ /dev/null @@ -1,3 +0,0 @@ - -def start(ctx): - raise ValueError() diff --git a/test/mitmproxy/data/scripts/syntaxerr.py b/test/mitmproxy/data/scripts/syntaxerr.py deleted file mode 100644 index 219d6b84..00000000 --- a/test/mitmproxy/data/scripts/syntaxerr.py +++ /dev/null @@ -1,3 +0,0 @@ - - -a + diff --git a/test/mitmproxy/data/scripts/unloaderr.py b/test/mitmproxy/data/scripts/unloaderr.py deleted file mode 100644 index fba02734..00000000 --- a/test/mitmproxy/data/scripts/unloaderr.py +++ /dev/null @@ -1,2 +0,0 @@ -def done(ctx): - raise RuntimeError() diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index 9e726a32..dcc0dc48 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -3,24 +3,31 @@ import mock from . import tutils import netlib.tutils -from mitmproxy import flow, proxy, models +from mitmproxy.flow import master +from mitmproxy import flow, proxy, models, controller class MasterTest: + def invoke(self, master, handler, *message): + with master.handlecontext(): + func = getattr(master, handler) + func(*message) + if message: + message[0].reply = controller.DummyReply() + def cycle(self, master, content): f = tutils.tflow(req=netlib.tutils.treq(content=content)) l = proxy.Log("connect") l.reply = mock.MagicMock() master.log(l) - master.clientconnect(f.client_conn) - master.serverconnect(f.server_conn) - master.request(f) + self.invoke(master, "clientconnect", f.client_conn) + self.invoke(master, "clientconnect", f.client_conn) + self.invoke(master, "serverconnect", f.server_conn) + self.invoke(master, "request", f) if not f.error: f.response = models.HTTPResponse.wrap(netlib.tutils.tresp(content=content)) - f.reply.acked = False - f = master.response(f) - f.client_conn.reply.acked = False - master.clientdisconnect(f.client_conn) + self.invoke(master, "response", f) + self.invoke(master, "clientdisconnect", f) return f def dummy_cycle(self, master, n, content): @@ -34,3 +41,12 @@ class MasterTest: t = tutils.tflow(resp=True) fw.add(t) f.close() + + +class RecordingMaster(master.FlowMaster): + def __init__(self, *args, **kwargs): + master.FlowMaster.__init__(self, *args, **kwargs) + self.event_log = [] + + def add_log(self, e, level): + self.event_log.append((level, e)) diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index 62541f3f..080746e8 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -1,28 +1,46 @@ -from mitmproxy.script import Script -from test.mitmproxy import tutils +from test.mitmproxy import tutils, mastertest from mitmproxy import controller +from mitmproxy.builtins import script +from mitmproxy import options +from mitmproxy.flow import master +from mitmproxy.flow import state import time class Thing: def __init__(self): self.reply = controller.DummyReply() + self.live = True -@tutils.skip_appveyor -def test_concurrent(): - with Script(tutils.test_data.path("data/scripts/concurrent_decorator.py"), None) as s: - f1, f2 = Thing(), Thing() - s.run("request", f1) - s.run("request", f2) +class TestConcurrent(mastertest.MasterTest): + @tutils.skip_appveyor + def test_concurrent(self): + s = state.State() + m = master.FlowMaster(options.Options(), None, s) + sc = script.Script( + tutils.test_data.path( + "data/addonscripts/concurrent_decorator.py" + ) + ) + m.addons.add(sc) + f1, f2 = tutils.tflow(), tutils.tflow() + self.invoke(m, "request", f1) + self.invoke(m, "request", f2) start = time.time() while time.time() - start < 5: if f1.reply.acked and f2.reply.acked: return raise ValueError("Script never acked") - -def test_concurrent_err(): - s = Script(tutils.test_data.path("data/scripts/concurrent_decorator_err.py"), None) - with tutils.raises("Concurrent decorator not supported for 'start' method"): - s.load() + def test_concurrent_err(self): + s = state.State() + m = mastertest.RecordingMaster(options.Options(), None, s) + sc = script.Script( + tutils.test_data.path( + "data/addonscripts/concurrent_decorator_err.py" + ) + ) + with m.handlecontext(): + sc.start() + assert "decorator not supported" in m.event_log[0][1] diff --git a/test/mitmproxy/script/test_reloader.py b/test/mitmproxy/script/test_reloader.py deleted file mode 100644 index 0345f6ed..00000000 --- a/test/mitmproxy/script/test_reloader.py +++ /dev/null @@ -1,34 +0,0 @@ -import mock -from mitmproxy.script.reloader import watch, unwatch -from test.mitmproxy import tutils -from threading import Event - - -def test_simple(): - with tutils.tmpdir(): - with open("foo.py", "w"): - pass - - script = mock.Mock() - script.filename = "foo.py" - - e = Event() - - def _onchange(): - e.set() - - watch(script, _onchange) - with tutils.raises("already observed"): - watch(script, _onchange) - - # Some reloaders don't register a change directly after watching, because they first need to initialize. - # To test if watching works at all, we do repeated writes every 100ms. - for _ in range(100): - with open("foo.py", "a") as f: - f.write(".") - if e.wait(0.1): - break - else: - raise AssertionError("No change detected.") - - unwatch(script) diff --git a/test/mitmproxy/script/test_script.py b/test/mitmproxy/script/test_script.py deleted file mode 100644 index fe98fab5..00000000 --- a/test/mitmproxy/script/test_script.py +++ /dev/null @@ -1,83 +0,0 @@ -from mitmproxy.script import Script -from mitmproxy.exceptions import ScriptException -from test.mitmproxy import tutils - - -class TestParseCommand: - def test_empty_command(self): - with tutils.raises(ScriptException): - Script.parse_command("") - - with tutils.raises(ScriptException): - Script.parse_command(" ") - - def test_no_script_file(self): - with tutils.raises("not found"): - Script.parse_command("notfound") - - with tutils.tmpdir() as dir: - with tutils.raises("not a file"): - Script.parse_command(dir) - - def test_parse_args(self): - with tutils.chdir(tutils.test_data.dirname): - assert Script.parse_command("data/scripts/a.py") == ["data/scripts/a.py"] - assert Script.parse_command("data/scripts/a.py foo bar") == ["data/scripts/a.py", "foo", "bar"] - assert Script.parse_command("data/scripts/a.py 'foo bar'") == ["data/scripts/a.py", "foo bar"] - - @tutils.skip_not_windows - def test_parse_windows(self): - with tutils.chdir(tutils.test_data.dirname): - assert Script.parse_command("data\\scripts\\a.py") == ["data\\scripts\\a.py"] - assert Script.parse_command("data\\scripts\\a.py 'foo \\ bar'") == ["data\\scripts\\a.py", 'foo \\ bar'] - - -def test_simple(): - with tutils.chdir(tutils.test_data.path("data/scripts")): - s = Script("a.py --var 42", None) - assert s.filename == "a.py" - assert s.ns is None - - s.load() - assert s.ns["var"] == 42 - - s.run("here") - assert s.ns["var"] == 43 - - s.unload() - assert s.ns is None - - with tutils.raises(ScriptException): - s.run("here") - - with Script("a.py --var 42", None) as s: - s.run("here") - - -def test_script_exception(): - with tutils.chdir(tutils.test_data.path("data/scripts")): - s = Script("syntaxerr.py", None) - with tutils.raises(ScriptException): - s.load() - - s = Script("starterr.py", None) - with tutils.raises(ScriptException): - s.load() - - s = Script("a.py", None) - s.load() - with tutils.raises(ScriptException): - s.load() - - s = Script("a.py", None) - with tutils.raises(ScriptException): - s.run("here") - - with tutils.raises(ScriptException): - with Script("reqerr.py", None) as s: - s.run("request", None) - - s = Script("unloaderr.py", None) - s.load() - with tutils.raises(ScriptException): - s.unload() diff --git a/test/mitmproxy/test_addons.py b/test/mitmproxy/test_addons.py new file mode 100644 index 00000000..1861d4ac --- /dev/null +++ b/test/mitmproxy/test_addons.py @@ -0,0 +1,20 @@ +from __future__ import absolute_import, print_function, division +from mitmproxy import addons +from mitmproxy import controller +from mitmproxy import options + + +class TAddon: + def __init__(self, name): + self.name = name + + def __repr__(self): + return "Addon(%s)" % self.name + + +def test_simple(): + m = controller.Master(options.Options()) + a = addons.Addons(m) + a.add(TAddon("one")) + assert a.has_addon("one") + assert not a.has_addon("two") diff --git a/test/mitmproxy/test_cmdline.py b/test/mitmproxy/test_cmdline.py index 4fe2cf94..55627408 100644 --- a/test/mitmproxy/test_cmdline.py +++ b/test/mitmproxy/test_cmdline.py @@ -1,5 +1,4 @@ import argparse -import base64 from mitmproxy import cmdline from . import tutils @@ -36,34 +35,6 @@ def test_parse_replace_hook(): ) -def test_parse_server_spec(): - tutils.raises("Invalid server specification", cmdline.parse_server_spec, "") - assert cmdline.parse_server_spec( - "http://foo.com:88") == (b"http", (b"foo.com", 88)) - assert cmdline.parse_server_spec( - "http://foo.com") == (b"http", (b"foo.com", 80)) - assert cmdline.parse_server_spec( - "https://foo.com") == (b"https", (b"foo.com", 443)) - tutils.raises( - "Invalid server specification", - cmdline.parse_server_spec, - "foo.com") - tutils.raises( - "Invalid server specification", - cmdline.parse_server_spec, - "http://") - - -def test_parse_upstream_auth(): - tutils.raises("Invalid upstream auth specification", cmdline.parse_upstream_auth, "") - tutils.raises("Invalid upstream auth specification", cmdline.parse_upstream_auth, ":") - tutils.raises("Invalid upstream auth specification", cmdline.parse_upstream_auth, ":test") - assert cmdline.parse_upstream_auth( - "test:test") == b"Basic" + b" " + base64.b64encode(b"test:test") - assert cmdline.parse_upstream_auth( - "test:") == b"Basic" + b" " + base64.b64encode(b"test:") - - def test_parse_setheaders(): x = cmdline.parse_setheader("/foo/bar/voing") assert x == ("foo", "bar", "voing") diff --git a/test/mitmproxy/test_contentview.py b/test/mitmproxy/test_contentview.py index c11a5fe5..2db9ab40 100644 --- a/test/mitmproxy/test_contentview.py +++ b/test/mitmproxy/test_contentview.py @@ -1,6 +1,5 @@ from mitmproxy.exceptions import ContentViewException from netlib.http import Headers -from netlib import encoding from netlib.http import url from netlib import multidict @@ -216,28 +215,6 @@ Larry headers=Headers() ) - r = cv.get_content_view( - cv.get("Auto"), - encoding.encode('gzip', b"[1, 2, 3]"), - headers=Headers( - content_type="application/json", - content_encoding="gzip" - ) - ) - assert "decoded gzip" in r[0] - assert "JSON" in r[0] - - r = cv.get_content_view( - cv.get("XML"), - encoding.encode('gzip', b"[1, 2, 3]"), - headers=Headers( - content_type="application/json", - content_encoding="gzip" - ) - ) - assert "decoded gzip" in r[0] - assert "Raw" in r[0] - def test_add_cv(self): class TestContentView(cv.View): name = "test" diff --git a/test/mitmproxy/test_controller.py b/test/mitmproxy/test_controller.py index 5a68e15b..6d4b8fe6 100644 --- a/test/mitmproxy/test_controller.py +++ b/test/mitmproxy/test_controller.py @@ -25,7 +25,7 @@ class TestMaster(object): # Speed up test super(DummyMaster, self).tick(0) - m = DummyMaster() + m = DummyMaster(None) assert not m.should_exit.is_set() msg = TMsg() msg.reply = controller.DummyReply() @@ -34,7 +34,7 @@ class TestMaster(object): assert m.should_exit.is_set() def test_server_simple(self): - m = controller.Master() + m = controller.Master(None) s = DummyServer(None) m.add_server(s) m.start() diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index 234490f8..90f33264 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -1,127 +1,79 @@ import os from six.moves import cStringIO as StringIO -from mitmproxy.exceptions import ContentViewException -import netlib.tutils - -from mitmproxy import dump, flow, models +from mitmproxy import dump, flow, exceptions from . import tutils, mastertest import mock -def test_strfuncs(): - o = dump.Options() - m = dump.DumpMaster(None, o) - - m.outfile = StringIO() - m.o.flow_detail = 0 - m.echo_flow(tutils.tflow()) - assert not m.outfile.getvalue() - - m.o.flow_detail = 4 - m.echo_flow(tutils.tflow()) - assert m.outfile.getvalue() - - m.outfile = StringIO() - m.echo_flow(tutils.tflow(resp=True)) - assert "<<" in m.outfile.getvalue() - - m.outfile = StringIO() - m.echo_flow(tutils.tflow(err=True)) - assert "<<" in m.outfile.getvalue() - - flow = tutils.tflow() - flow.request = netlib.tutils.treq() - flow.request.stickycookie = True - flow.client_conn = mock.MagicMock() - flow.client_conn.address.host = "foo" - flow.response = netlib.tutils.tresp(content=None) - flow.response.is_replay = True - flow.response.status_code = 300 - m.echo_flow(flow) - - flow = tutils.tflow(resp=netlib.tutils.tresp(content="{")) - flow.response.headers["content-type"] = "application/json" - flow.response.status_code = 400 - m.echo_flow(flow) - - -@mock.patch("mitmproxy.contentviews.get_content_view") -def test_contentview(get_content_view): - get_content_view.side_effect = ContentViewException(""), ("x", iter([])) - - o = dump.Options(flow_detail=4, verbosity=3) - m = dump.DumpMaster(None, o, StringIO()) - m.echo_flow(tutils.tflow()) - assert "Content viewer failed" in m.outfile.getvalue() - - class TestDumpMaster(mastertest.MasterTest): def dummy_cycle(self, master, n, content): mastertest.MasterTest.dummy_cycle(self, master, n, content) - return master.outfile.getvalue() + return master.options.tfile.getvalue() def mkmaster(self, filt, **options): - cs = StringIO() - o = dump.Options(filtstr=filt, **options) - return dump.DumpMaster(None, o, outfile=cs) + if "verbosity" not in options: + options["verbosity"] = 0 + if "flow_detail" not in options: + options["flow_detail"] = 0 + o = dump.Options(filtstr=filt, tfile=StringIO(), **options) + return dump.DumpMaster(None, o) def test_basic(self): for i in (1, 2, 3): - assert "GET" in self.dummy_cycle(self.mkmaster("~s", flow_detail=i), 1, "") assert "GET" in self.dummy_cycle( self.mkmaster("~s", flow_detail=i), 1, - "\x00\x00\x00" + b"" + ) + assert "GET" in self.dummy_cycle( + self.mkmaster("~s", flow_detail=i), + 1, + b"\x00\x00\x00" ) assert "GET" in self.dummy_cycle( self.mkmaster("~s", flow_detail=i), - 1, "ascii" + 1, + b"ascii" ) def test_error(self): - cs = StringIO() - o = dump.Options(flow_detail=1) - m = dump.DumpMaster(None, o, outfile=cs) + o = dump.Options( + tfile=StringIO(), + flow_detail=1 + ) + m = dump.DumpMaster(None, o) f = tutils.tflow(err=True) - m.request(f) + m.error(f) assert m.error(f) - assert "error" in cs.getvalue() - - def test_missing_content(self): - cs = StringIO() - o = dump.Options(flow_detail=3) - m = dump.DumpMaster(None, o, outfile=cs) - f = tutils.tflow() - f.request.content = None - m.request(f) - f.response = models.HTTPResponse.wrap(netlib.tutils.tresp()) - f.response.content = None - m.response(f) - assert "content missing" in cs.getvalue() + assert "error" in o.tfile.getvalue() def test_replay(self): - cs = StringIO() - o = dump.Options(server_replay=["nonexistent"], kill=True) - tutils.raises(dump.DumpError, dump.DumpMaster, None, o, outfile=cs) + tutils.raises(dump.DumpError, dump.DumpMaster, None, o) with tutils.tmpdir() as t: p = os.path.join(t, "rep") self.flowfile(p) o = dump.Options(server_replay=[p], kill=True) - m = dump.DumpMaster(None, o, outfile=cs) + o.verbosity = 0 + o.flow_detail = 0 + m = dump.DumpMaster(None, o) - self.cycle(m, "content") - self.cycle(m, "content") + self.cycle(m, b"content") + self.cycle(m, b"content") o = dump.Options(server_replay=[p], kill=False) - m = dump.DumpMaster(None, o, outfile=cs) - self.cycle(m, "nonexistent") + o.verbosity = 0 + o.flow_detail = 0 + m = dump.DumpMaster(None, o) + self.cycle(m, b"nonexistent") o = dump.Options(client_replay=[p], kill=False) - m = dump.DumpMaster(None, o, outfile=cs) + o.verbosity = 0 + o.flow_detail = 0 + m = dump.DumpMaster(None, o) def test_read(self): with tutils.tmpdir() as t: @@ -129,9 +81,8 @@ class TestDumpMaster(mastertest.MasterTest): self.flowfile(p) assert "GET" in self.dummy_cycle( self.mkmaster(None, flow_detail=1, rfile=p), - 0, "", + 1, b"", ) - tutils.raises( dump.DumpError, self.mkmaster, None, verbosity=1, rfile="/nonexistent" @@ -147,7 +98,7 @@ class TestDumpMaster(mastertest.MasterTest): def test_filter(self): assert "GET" not in self.dummy_cycle( - self.mkmaster("~u foo", verbosity=1), 1, "" + self.mkmaster("~u foo", verbosity=1), 1, b"" ) def test_app(self): @@ -157,24 +108,32 @@ class TestDumpMaster(mastertest.MasterTest): assert len(m.apps.apps) == 1 def test_replacements(self): - cs = StringIO() - o = dump.Options(replacements=[(".*", "content", "foo")]) - m = dump.DumpMaster(None, o, outfile=cs) - f = self.cycle(m, "content") - assert f.request.content == "foo" + o = dump.Options( + replacements=[(".*", "content", "foo")], + tfile = StringIO(), + ) + o.verbosity = 0 + o.flow_detail = 0 + m = dump.DumpMaster(None, o) + f = self.cycle(m, b"content") + assert f.request.content == b"foo" def test_setheader(self): - cs = StringIO() - o = dump.Options(setheaders=[(".*", "one", "two")]) - m = dump.DumpMaster(None, o, outfile=cs) - f = self.cycle(m, "content") + o = dump.Options( + setheaders=[(".*", "one", "two")], + tfile=StringIO() + ) + o.verbosity = 0 + o.flow_detail = 0 + m = dump.DumpMaster(None, o) + f = self.cycle(m, b"content") assert f.request.headers["one"] == "two" def test_write(self): with tutils.tmpdir() as d: p = os.path.join(d, "a") self.dummy_cycle( - self.mkmaster(None, outfile=(p, "wb"), verbosity=0), 1, "" + self.mkmaster(None, outfile=(p, "wb"), verbosity=0), 1, b"" ) assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 1 @@ -183,17 +142,17 @@ class TestDumpMaster(mastertest.MasterTest): p = os.path.join(d, "a.append") self.dummy_cycle( self.mkmaster(None, outfile=(p, "wb"), verbosity=0), - 1, "" + 1, b"" ) self.dummy_cycle( self.mkmaster(None, outfile=(p, "ab"), verbosity=0), - 1, "" + 1, b"" ) assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 2 def test_write_err(self): tutils.raises( - dump.DumpError, + exceptions.OptionsError, self.mkmaster, None, outfile = ("nonexistentdir/foo", "wb") ) @@ -201,9 +160,10 @@ class TestDumpMaster(mastertest.MasterTest): ret = self.dummy_cycle( self.mkmaster( None, - scripts=[tutils.test_data.path("data/scripts/all.py")], verbosity=1 + scripts=[tutils.test_data.path("data/scripts/all.py")], + verbosity=2 ), - 1, "", + 1, b"", ) assert "XCLIENTCONNECT" in ret assert "XSERVERCONNECT" in ret @@ -211,12 +171,12 @@ class TestDumpMaster(mastertest.MasterTest): assert "XRESPONSE" in ret assert "XCLIENTDISCONNECT" in ret tutils.raises( - dump.DumpError, + exceptions.AddonError, self.mkmaster, None, scripts=["nonexistent"] ) tutils.raises( - dump.DumpError, + exceptions.AddonError, self.mkmaster, None, scripts=["starterr.py"] ) @@ -224,11 +184,11 @@ class TestDumpMaster(mastertest.MasterTest): def test_stickycookie(self): self.dummy_cycle( self.mkmaster(None, stickycookie = ".*"), - 1, "" + 1, b"" ) def test_stickyauth(self): self.dummy_cycle( self.mkmaster(None, stickyauth = ".*"), - 1, "" + 1, b"" ) diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index f30973e7..0ec85f52 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -1,162 +1,126 @@ -import glob import json -import os -import sys -from contextlib import contextmanager -from mitmproxy import script -from mitmproxy.proxy import config +import six +import sys +import os.path +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy import options +from mitmproxy import contentviews +from mitmproxy.builtins import script import netlib.utils from netlib import tutils as netutils from netlib.http import Headers -from . import tservers, tutils - -example_dir = netlib.utils.Data(__name__).path("../../examples") - - -class DummyContext(object): - """Emulate script.ScriptContext() functionality.""" - - contentview = None - - def log(self, *args, **kwargs): - pass +from . import tutils, mastertest - def add_contentview(self, view_obj): - self.contentview = view_obj +example_dir = netlib.utils.Data(__name__).push("../../examples") - def remove_contentview(self, view_obj): - self.contentview = None +class ScriptError(Exception): + pass -@contextmanager -def example(command): - command = os.path.join(example_dir, command) - ctx = DummyContext() - with script.Script(command, ctx) as s: - yield s +class RaiseMaster(master.FlowMaster): + def add_log(self, e, level): + if level in ("warn", "error"): + raise ScriptError(e) -def test_load_scripts(): - scripts = glob.glob("%s/*.py" % example_dir) - tmaster = tservers.TestMaster(config.ProxyConfig()) +def tscript(cmd, args=""): + cmd = example_dir.path(cmd) + " " + args + m = RaiseMaster(options.Options(), None, state.State()) + sc = script.Script(cmd) + m.addons.add(sc) + return m, sc - for f in scripts: - if "har_extractor" in f: - continue - if "flowwriter" in f: - f += " -" - if "iframe_injector" in f: - f += " foo" # one argument required - if "filt" in f: - f += " ~a" - if "modify_response_body" in f: - f += " foo bar" # two arguments required - s = script.Script(f, script.ScriptContext(tmaster)) - try: - s.load() - except Exception as v: - if "ImportError" not in str(v): - raise - else: - s.unload() +class TestScripts(mastertest.MasterTest): + def test_add_header(self): + m, _ = tscript("add_header.py") + f = tutils.tflow(resp=netutils.tresp()) + self.invoke(m, "response", f) + assert f.response.headers["newheader"] == "foo" - -def test_add_header(): - flow = tutils.tflow(resp=netutils.tresp()) - with example("add_header.py") as ex: - ex.run("response", flow) - assert flow.response.headers["newheader"] == "foo" - - -def test_custom_contentviews(): - with example("custom_contentviews.py") as ex: - pig = ex.ctx.contentview + def test_custom_contentviews(self): + m, sc = tscript("custom_contentviews.py") + pig = contentviews.get("pig_latin_HTML") _, fmt = pig(b"<html>test!</html>") assert any(b'esttay!' in val[0][1] for val in fmt) assert not pig(b"gobbledygook") + def test_iframe_injector(self): + with tutils.raises(ScriptError): + tscript("iframe_injector.py") -def test_iframe_injector(): - with tutils.raises(script.ScriptException): - with example("iframe_injector.py") as ex: - pass - - flow = tutils.tflow(resp=netutils.tresp(content=b"<html>mitmproxy</html>")) - with example("iframe_injector.py http://example.org/evil_iframe") as ex: - ex.run("response", flow) + m, sc = tscript("iframe_injector.py", "http://example.org/evil_iframe") + flow = tutils.tflow(resp=netutils.tresp(content=b"<html>mitmproxy</html>")) + self.invoke(m, "response", flow) content = flow.response.content assert b'iframe' in content and b'evil_iframe' in content - -def test_modify_form(): - form_header = Headers(content_type="application/x-www-form-urlencoded") - flow = tutils.tflow(req=netutils.treq(headers=form_header)) - with example("modify_form.py") as ex: - ex.run("request", flow) - assert flow.request.urlencoded_form[b"mitmproxy"] == b"rocks" - - flow.request.headers["content-type"] = "" - ex.run("request", flow) - assert list(flow.request.urlencoded_form.items()) == [(b"foo", b"bar")] - - -def test_modify_querystring(): - flow = tutils.tflow(req=netutils.treq(path=b"/search?q=term")) - with example("modify_querystring.py") as ex: - ex.run("request", flow) - assert flow.request.query["mitmproxy"] == "rocks" - - flow.request.path = "/" - ex.run("request", flow) - assert flow.request.query["mitmproxy"] == "rocks" - - -def test_modify_response_body(): - with tutils.raises(script.ScriptException): - with example("modify_response_body.py"): - assert True - - flow = tutils.tflow(resp=netutils.tresp(content=b"I <3 mitmproxy")) - with example("modify_response_body.py mitmproxy rocks") as ex: - assert ex.ctx.old == b"mitmproxy" and ex.ctx.new == b"rocks" - ex.run("response", flow) - assert flow.response.content == b"I <3 rocks" - - -def test_redirect_requests(): - flow = tutils.tflow(req=netutils.treq(host=b"example.org")) - with example("redirect_requests.py") as ex: - ex.run("request", flow) - assert flow.request.host == "mitmproxy.org" - - -def test_har_extractor(): - if sys.version_info >= (3, 0): - with tutils.raises("does not work on Python 3"): - with example("har_extractor.py -"): - pass - return - - with tutils.raises(script.ScriptException): - with example("har_extractor.py"): - pass - - times = dict( - timestamp_start=746203272, - timestamp_end=746203272, - ) - - flow = tutils.tflow( - req=netutils.treq(**times), - resp=netutils.tresp(**times) - ) - - with example("har_extractor.py -") as ex: - ex.run("response", flow) - - with open(tutils.test_data.path("data/har_extractor.har")) as fp: - test_data = json.load(fp) - assert json.loads(ex.ctx.HARLog.json()) == test_data["test_response"] + def test_modify_form(self): + m, sc = tscript("modify_form.py") + + form_header = Headers(content_type="application/x-www-form-urlencoded") + f = tutils.tflow(req=netutils.treq(headers=form_header)) + self.invoke(m, "request", f) + + assert f.request.urlencoded_form[b"mitmproxy"] == b"rocks" + + f.request.headers["content-type"] = "" + self.invoke(m, "request", f) + assert list(f.request.urlencoded_form.items()) == [(b"foo", b"bar")] + + def test_modify_querystring(self): + m, sc = tscript("modify_querystring.py") + f = tutils.tflow(req=netutils.treq(path="/search?q=term")) + + self.invoke(m, "request", f) + assert f.request.query["mitmproxy"] == "rocks" + + f.request.path = "/" + self.invoke(m, "request", f) + assert f.request.query["mitmproxy"] == "rocks" + + def test_modify_response_body(self): + with tutils.raises(ScriptError): + tscript("modify_response_body.py") + + m, sc = tscript("modify_response_body.py", "mitmproxy rocks") + f = tutils.tflow(resp=netutils.tresp(content=b"I <3 mitmproxy")) + self.invoke(m, "response", f) + assert f.response.content == b"I <3 rocks" + + def test_redirect_requests(self): + m, sc = tscript("redirect_requests.py") + f = tutils.tflow(req=netutils.treq(host="example.org")) + self.invoke(m, "request", f) + assert f.request.host == "mitmproxy.org" + + def test_har_extractor(self): + if sys.version_info >= (3, 0): + with tutils.raises("does not work on Python 3"): + tscript("har_extractor.py") + return + + with tutils.raises(ScriptError): + tscript("har_extractor.py") + + with tutils.tmpdir() as tdir: + times = dict( + timestamp_start=746203272, + timestamp_end=746203272, + ) + + path = os.path.join(tdir, "file") + m, sc = tscript("har_extractor.py", six.moves.shlex_quote(path)) + f = tutils.tflow( + req=netutils.treq(**times), + resp=netutils.tresp(**times) + ) + self.invoke(m, "response", f) + m.addons.remove(sc) + + with open(path, "rb") as f: + test_data = json.load(f) + assert len(test_data["log"]["pages"]) == 1 diff --git a/test/mitmproxy/test_filt.py b/test/mitmproxy/test_filt.py index 9fe36b2a..69f042bb 100644 --- a/test/mitmproxy/test_filt.py +++ b/test/mitmproxy/test_filt.py @@ -1,6 +1,8 @@ from six.moves import cStringIO as StringIO -from mitmproxy import filt from mock import patch + +from mitmproxy import filt + from . import tutils @@ -73,7 +75,7 @@ class TestParsing: self._dump(a) -class TestMatching: +class TestMatchingHTTPFlow: def req(self): return tutils.tflow() @@ -87,6 +89,11 @@ class TestMatching: def q(self, q, o): return filt.parse(q)(o) + def test_http(self): + s = self.req() + assert self.q("~http", s) + assert not self.q("~tcp", s) + def test_asset(self): s = self.resp() assert not self.q("~a", s) @@ -247,6 +254,186 @@ class TestMatching: assert not self.q("!~c 201 !~c 200", s) +class TestMatchingTCPFlow: + + def flow(self): + return tutils.ttcpflow() + + def err(self): + return tutils.ttcpflow(err=True) + + def q(self, q, o): + return filt.parse(q)(o) + + def test_tcp(self): + f = self.flow() + assert self.q("~tcp", f) + assert not self.q("~http", f) + + def test_ferr(self): + e = self.err() + assert self.q("~e", e) + + def test_body(self): + f = self.flow() + + # Messages sent by client or server + assert self.q("~b hello", f) + assert self.q("~b me", f) + assert not self.q("~b nonexistent", f) + + # Messages sent by client + assert self.q("~bq hello", f) + assert not self.q("~bq me", f) + assert not self.q("~bq nonexistent", f) + + # Messages sent by server + assert self.q("~bs me", f) + assert not self.q("~bs hello", f) + assert not self.q("~bs nonexistent", f) + + def test_src(self): + f = self.flow() + assert self.q("~src address", f) + assert not self.q("~src foobar", f) + assert self.q("~src :22", f) + assert not self.q("~src :99", f) + assert self.q("~src address:22", f) + + def test_dst(self): + f = self.flow() + f.server_conn = tutils.tserver_conn() + assert self.q("~dst address", f) + assert not self.q("~dst foobar", f) + assert self.q("~dst :22", f) + assert not self.q("~dst :99", f) + assert self.q("~dst address:22", f) + + def test_and(self): + f = self.flow() + f.server_conn = tutils.tserver_conn() + assert self.q("~b hello & ~b me", f) + assert not self.q("~src wrongaddress & ~b hello", f) + assert self.q("(~src :22 & ~dst :22) & ~b hello", f) + assert not self.q("(~src address:22 & ~dst :22) & ~b nonexistent", f) + assert not self.q("(~src address:22 & ~dst :99) & ~b hello", f) + + def test_or(self): + f = self.flow() + f.server_conn = tutils.tserver_conn() + assert self.q("~b hello | ~b me", f) + assert self.q("~src :22 | ~b me", f) + assert not self.q("~src :99 | ~dst :99", f) + assert self.q("(~src :22 | ~dst :22) | ~b me", f) + + def test_not(self): + f = self.flow() + assert not self.q("! ~src :22", f) + assert self.q("! ~src :99", f) + assert self.q("!~src :99 !~src :99", f) + assert not self.q("!~src :99 !~src :22", f) + + def test_request(self): + f = self.flow() + assert not self.q("~q", f) + + def test_response(self): + f = self.flow() + assert not self.q("~s", f) + + def test_headers(self): + f = self.flow() + assert not self.q("~h whatever", f) + + # Request headers + assert not self.q("~hq whatever", f) + + # Response headers + assert not self.q("~hs whatever", f) + + def test_content_type(self): + f = self.flow() + assert not self.q("~t whatever", f) + + # Request content-type + assert not self.q("~tq whatever", f) + + # Response content-type + assert not self.q("~ts whatever", f) + + def test_code(self): + f = self.flow() + assert not self.q("~c 200", f) + + def test_domain(self): + f = self.flow() + assert not self.q("~d whatever", f) + + def test_method(self): + f = self.flow() + assert not self.q("~m whatever", f) + + def test_url(self): + f = self.flow() + assert not self.q("~u whatever", f) + + +class TestMatchingDummyFlow: + + def flow(self): + return tutils.tdummyflow() + + def err(self): + return tutils.tdummyflow(err=True) + + def q(self, q, o): + return filt.parse(q)(o) + + def test_filters(self): + e = self.err() + f = self.flow() + f.server_conn = tutils.tserver_conn() + + assert not self.q("~a", f) + + assert not self.q("~b whatever", f) + assert not self.q("~bq whatever", f) + assert not self.q("~bs whatever", f) + + assert not self.q("~c 0", f) + + assert not self.q("~d whatever", f) + + assert self.q("~dst address", f) + assert not self.q("~dst nonexistent", f) + + assert self.q("~e", e) + assert not self.q("~e", f) + + assert not self.q("~http", f) + + assert not self.q("~h whatever", f) + assert not self.q("~hq whatever", f) + assert not self.q("~hs whatever", f) + + assert not self.q("~m whatever", f) + + assert not self.q("~s", f) + + assert self.q("~src address", f) + assert not self.q("~src nonexistent", f) + + assert not self.q("~tcp", f) + + assert not self.q("~t whatever", f) + assert not self.q("~tq whatever", f) + assert not self.q("~ts whatever", f) + + assert not self.q("~u whatever", f) + + assert not self.q("~q", f) + + @patch('traceback.extract_tb') def test_pyparsing_bug(extract_tb): """https://github.com/mitmproxy/mitmproxy/issues/1087""" diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index bf7622f6..e17a125c 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -1,19 +1,17 @@ -import os.path - import mock import io import netlib.utils from netlib.http import Headers from mitmproxy import filt, controller, flow +from mitmproxy.flow import options from mitmproxy.contrib import tnetstring -from mitmproxy.exceptions import FlowReadException, ScriptException +from mitmproxy.exceptions import FlowReadException from mitmproxy.models import Error from mitmproxy.models import Flow from mitmproxy.models import HTTPFlow from mitmproxy.models import HTTPRequest from mitmproxy.models import HTTPResponse -from mitmproxy.proxy.config import HostMatcher from mitmproxy.proxy import ProxyConfig from mitmproxy.proxy.server import DummyServer from mitmproxy.models.connections import ClientConnection @@ -40,94 +38,12 @@ def test_app_registry(): assert ar.get(r) -class TestStickyCookieState: - - def _response(self, cookie, host): - s = flow.StickyCookieState(filt.parse(".*")) - f = tutils.tflow(req=netlib.tutils.treq(host=host, port=80), resp=True) - f.response.headers["Set-Cookie"] = cookie - s.handle_response(f) - return s, f - - def test_domain_match(self): - s = flow.StickyCookieState(filt.parse(".*")) - assert s.domain_match("www.google.com", ".google.com") - assert s.domain_match("google.com", ".google.com") - - def test_response(self): - c = "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " \ - "Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; " - - s, f = self._response(c, "host") - assert not s.jar.keys() - - s, f = self._response(c, "www.google.com") - assert s.jar.keys() - - s, f = self._response("SSID=mooo", "www.google.com") - assert list(s.jar.keys())[0] == ('www.google.com', 80, '/') - - # Test setting of multiple cookies - c1 = "somecookie=test; Path=/" - c2 = "othercookie=helloworld; Path=/" - s, f = self._response(c1, "www.google.com") - f.response.headers["Set-Cookie"] = c2 - s.handle_response(f) - googlekey = list(s.jar.keys())[0] - assert len(s.jar[googlekey].keys()) == 2 - - # Test setting of weird cookie keys - s = flow.StickyCookieState(filt.parse(".*")) - f = tutils.tflow(req=netlib.tutils.treq(host="www.google.com", port=80), resp=True) - cs = [ - "foo/bar=hello", - "foo:bar=world", - "foo@bar=fizz", - "foo,bar=buzz", - ] - for c in cs: - f.response.headers["Set-Cookie"] = c - s.handle_response(f) - googlekey = list(s.jar.keys())[0] - assert len(s.jar[googlekey]) == len(cs) - - # Test overwriting of a cookie value - c1 = "somecookie=helloworld; Path=/" - c2 = "somecookie=newvalue; Path=/" - s, f = self._response(c1, "www.google.com") - f.response.headers["Set-Cookie"] = c2 - s.handle_response(f) - googlekey = list(s.jar.keys())[0] - assert len(s.jar[googlekey]) == 1 - assert list(s.jar[googlekey]["somecookie"].values())[0] == "newvalue" - - def test_request(self): - s, f = self._response("SSID=mooo", b"www.google.com") - assert "cookie" not in f.request.headers - s.handle_request(f) - assert "cookie" in f.request.headers - - -class TestStickyAuthState: - - def test_response(self): - s = flow.StickyAuthState(filt.parse(".*")) - f = tutils.tflow(resp=True) - f.request.headers["authorization"] = "foo" - s.handle_request(f) - assert "address" in s.hosts - - f = tutils.tflow(resp=True) - s.handle_request(f) - assert f.request.headers["authorization"] == "foo" - - class TestClientPlaybackState: def test_tick(self): first = tutils.tflow() s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.start_client_playback([first, tutils.tflow()], True) c = fm.client_playback c.testing = True @@ -377,7 +293,7 @@ class TestServerPlaybackState: assert s._hash(r) == s._hash(r2) -class TestFlow(object): +class TestHTTPFlow(object): def test_copy(self): f = tutils.tflow(resp=True) @@ -458,7 +374,7 @@ class TestFlow(object): def test_kill(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow() f.intercept(mock.Mock()) f.kill(fm) @@ -467,7 +383,7 @@ class TestFlow(object): def test_killall(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow() f.intercept(fm) @@ -518,13 +434,27 @@ class TestFlow(object): f.replace("foo", "bar") - assert f.request.content != b"abarb" + assert f.request.raw_content != b"abarb" f.request.decode() - assert f.request.content == b"abarb" + assert f.request.raw_content == b"abarb" - assert f.response.content != b"abarb" + assert f.response.raw_content != b"abarb" f.response.decode() - assert f.response.content == b"abarb" + assert f.response.raw_content == b"abarb" + + +class TestTCPFlow: + + def test_match(self): + f = tutils.ttcpflow() + assert not f.match("~b nonexistent") + assert f.match(None) + assert not f.match("~b nonexistent") + + f = tutils.ttcpflow(err=True) + assert f.match("~e") + + tutils.raises(ValueError, f.match, "~") class TestState: @@ -702,18 +632,19 @@ class TestSerialize: def test_load_flows(self): r = self._treader() s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.load_flows(r) assert len(s.flows) == 6 def test_load_flows_reverse(self): r = self._treader() s = flow.State() - conf = ProxyConfig( + opts = options.Options( mode="reverse", - upstream_server=("https", ("use-this-domain", 80)) + upstream_server="https://use-this-domain" ) - fm = flow.FlowMaster(DummyServer(conf), s) + conf = ProxyConfig(opts) + fm = flow.FlowMaster(opts, DummyServer(conf), s) fm.load_flows(r) assert s.flows[0].request.host == "use-this-domain" @@ -758,32 +689,9 @@ class TestSerialize: class TestFlowMaster: - def test_load_script(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - - fm.load_script(tutils.test_data.path("data/scripts/a.py")) - fm.load_script(tutils.test_data.path("data/scripts/a.py")) - fm.unload_scripts() - with tutils.raises(ScriptException): - fm.load_script("nonexistent") - try: - fm.load_script(tutils.test_data.path("data/scripts/starterr.py")) - except ScriptException as e: - assert "ValueError" in str(e) - assert len(fm.scripts) == 0 - - def test_getset_ignore(self): - p = mock.Mock() - p.config.check_ignore = HostMatcher() - fm = flow.FlowMaster(p, flow.State()) - assert not fm.get_ignore_filter() - fm.set_ignore_filter(["^apple\.com:", ":443$"]) - assert fm.get_ignore_filter() - def test_replay(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow(resp=True) f.request.content = None assert "missing" in fm.replay_request(f) @@ -792,55 +700,11 @@ class TestFlowMaster: assert "intercepting" in fm.replay_request(f) f.live = True - assert "live" in fm.replay_request(f, run_scripthooks=True) - - def test_script_reqerr(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - fm.load_script(tutils.test_data.path("data/scripts/reqerr.py")) - f = tutils.tflow() - fm.clientconnect(f.client_conn) - assert fm.request(f) - - def test_script(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - fm.load_script(tutils.test_data.path("data/scripts/all.py")) - f = tutils.tflow(resp=True) - - f.client_conn.acked = False - fm.clientconnect(f.client_conn) - assert fm.scripts[0].ns["log"][-1] == "clientconnect" - f.server_conn.acked = False - fm.serverconnect(f.server_conn) - assert fm.scripts[0].ns["log"][-1] == "serverconnect" - f.reply.acked = False - fm.request(f) - assert fm.scripts[0].ns["log"][-1] == "request" - f.reply.acked = False - fm.response(f) - assert fm.scripts[0].ns["log"][-1] == "response" - # load second script - fm.load_script(tutils.test_data.path("data/scripts/all.py")) - assert len(fm.scripts) == 2 - f.server_conn.reply.acked = False - fm.clientdisconnect(f.server_conn) - assert fm.scripts[0].ns["log"][-1] == "clientdisconnect" - assert fm.scripts[1].ns["log"][-1] == "clientdisconnect" - - # unload first script - fm.unload_scripts() - assert len(fm.scripts) == 0 - fm.load_script(tutils.test_data.path("data/scripts/all.py")) - - f.error = tutils.terr() - f.reply.acked = False - fm.error(f) - assert fm.scripts[0].ns["log"][-1] == "error" + assert "live" in fm.replay_request(f) def test_duplicate_flow(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow(resp=True) fm.load_flow(f) assert s.flow_count() == 1 @@ -851,14 +715,12 @@ class TestFlowMaster: def test_create_flow(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) assert fm.create_request("GET", "http", "example.com", 80, "/") def test_all(self): s = flow.State() - fm = flow.FlowMaster(None, s) - fm.anticache = True - fm.anticomp = True + fm = flow.FlowMaster(None, None, s) f = tutils.tflow(req=None) fm.clientconnect(f.client_conn) f.request = HTTPRequest.wrap(netlib.tutils.treq()) @@ -875,7 +737,6 @@ class TestFlowMaster: f.error.reply = controller.DummyReply() fm.error(f) - fm.load_script(tutils.test_data.path("data/scripts/a.py")) fm.shutdown() def test_client_playback(self): @@ -883,7 +744,11 @@ class TestFlowMaster: f = tutils.tflow(resp=True) pb = [tutils.tflow(resp=True), f] - fm = flow.FlowMaster(DummyServer(ProxyConfig()), s) + fm = flow.FlowMaster( + flow.options.Options(), + DummyServer(ProxyConfig(options.Options())), + s + ) assert not fm.start_server_playback( pb, False, @@ -911,7 +776,7 @@ class TestFlowMaster: f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) pb = [f] - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(flow.options.Options(), None, s) fm.refresh_server_playback = True assert not fm.do_server_playback(tutils.tflow()) @@ -953,7 +818,7 @@ class TestFlowMaster: f = tutils.tflow() f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) pb = [f] - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.refresh_server_playback = True fm.start_server_playback( pb, @@ -971,77 +836,6 @@ class TestFlowMaster: fm.process_new_request(f) assert "killed" in f.error.msg - def test_stickycookie(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - assert "Invalid" in fm.set_stickycookie("~h") - fm.set_stickycookie(".*") - assert fm.stickycookie_state - fm.set_stickycookie(None) - assert not fm.stickycookie_state - - fm.set_stickycookie(".*") - f = tutils.tflow(resp=True) - f.response.headers["set-cookie"] = "foo=bar" - fm.request(f) - f.reply.acked = False - fm.response(f) - assert fm.stickycookie_state.jar - assert "cookie" not in f.request.headers - f = f.copy() - f.reply.acked = False - fm.request(f) - assert f.request.headers["cookie"] == "foo=bar" - - def test_stickyauth(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - assert "Invalid" in fm.set_stickyauth("~h") - fm.set_stickyauth(".*") - assert fm.stickyauth_state - fm.set_stickyauth(None) - assert not fm.stickyauth_state - - fm.set_stickyauth(".*") - f = tutils.tflow(resp=True) - f.request.headers["authorization"] = "foo" - fm.request(f) - - f = tutils.tflow(resp=True) - assert fm.stickyauth_state.hosts - assert "authorization" not in f.request.headers - fm.request(f) - assert f.request.headers["authorization"] == "foo" - - def test_stream(self): - with tutils.tmpdir() as tdir: - p = os.path.join(tdir, "foo") - - def read(): - with open(p, "rb") as f: - r = flow.FlowReader(f) - return list(r.stream()) - - s = flow.State() - fm = flow.FlowMaster(None, s) - f = tutils.tflow(resp=True) - - with open(p, "ab") as tmpfile: - fm.start_stream(tmpfile, None) - fm.request(f) - fm.response(f) - fm.stop_stream() - - assert read()[0].response - - with open(p, "ab") as tmpfile: - f = tutils.tflow() - fm.start_stream(tmpfile, None) - fm.request(f) - fm.shutdown() - - assert not read()[1].response - class TestRequest: @@ -1076,15 +870,6 @@ class TestRequest: assert r.url == "https://address:22/path" assert r.pretty_url == "https://foo.com:22/path" - def test_anticache(self): - r = HTTPRequest.wrap(netlib.tutils.treq()) - r.headers = Headers() - r.headers["if-modified-since"] = "test" - r.headers["if-none-match"] = "test" - r.anticache() - assert "if-modified-since" not in r.headers - assert "if-none-match" not in r.headers - def test_replace(self): r = HTTPRequest.wrap(netlib.tutils.treq()) r.path = "path/foo" @@ -1105,16 +890,6 @@ class TestRequest: r.constrain_encoding() assert "oink" not in r.headers["accept-encoding"] - def test_get_decoded_content(self): - r = HTTPRequest.wrap(netlib.tutils.treq()) - r.content = None - r.headers["content-encoding"] = "identity" - assert r.get_decoded_content() is None - - r.content = b"falafel" - r.encode("gzip") - assert r.get_decoded_content() == b"falafel" - def test_get_content_type(self): resp = HTTPResponse.wrap(netlib.tutils.tresp()) resp.headers = Headers(content_type="text/plain") @@ -1183,104 +958,3 @@ class TestClientConnection: assert c3.get_state() == c.get_state() assert str(c) - - -def test_replacehooks(): - h = flow.ReplaceHooks() - h.add("~q", "foo", "bar") - assert h.lst - - h.set( - [ - (".*", "one", "two"), - (".*", "three", "four"), - ] - ) - assert h.count() == 2 - - h.clear() - assert not h.lst - - h.add("~q", "foo", "bar") - h.add("~s", "foo", "bar") - - v = h.get_specs() - assert v == [('~q', 'foo', 'bar'), ('~s', 'foo', 'bar')] - assert h.count() == 2 - h.clear() - assert h.count() == 0 - - f = tutils.tflow() - f.request.content = b"foo" - h.add("~s", "foo", "bar") - h.run(f) - assert f.request.content == b"foo" - - f = tutils.tflow(resp=True) - f.request.content = b"foo" - f.response.content = b"foo" - h.run(f) - assert f.response.content == b"bar" - assert f.request.content == b"foo" - - f = tutils.tflow() - h.clear() - h.add("~q", "foo", "bar") - f.request.content = b"foo" - h.run(f) - assert f.request.content == b"bar" - - assert not h.add("~", "foo", "bar") - assert not h.add("foo", "*", "bar") - - -def test_setheaders(): - h = flow.SetHeaders() - h.add("~q", "foo", "bar") - assert h.lst - - h.set( - [ - (".*", "one", "two"), - (".*", "three", "four"), - ] - ) - assert h.count() == 2 - - h.clear() - assert not h.lst - - h.add("~q", "foo", "bar") - h.add("~s", "foo", "bar") - - v = h.get_specs() - assert v == [('~q', 'foo', 'bar'), ('~s', 'foo', 'bar')] - assert h.count() == 2 - h.clear() - assert h.count() == 0 - - f = tutils.tflow() - f.request.content = b"foo" - h.add("~s", "foo", "bar") - h.run(f) - assert f.request.content == b"foo" - - h.clear() - h.add("~s", "one", "two") - h.add("~s", "one", "three") - f = tutils.tflow(resp=True) - f.request.headers["one"] = "xxx" - f.response.headers["one"] = "xxx" - h.run(f) - assert f.request.headers["one"] == "xxx" - assert f.response.headers.get_all("one") == ["two", "three"] - - h.clear() - h.add("~q", "one", "two") - h.add("~q", "one", "three") - f = tutils.tflow() - f.request.headers["one"] = "xxx" - h.run(f) - assert f.request.headers.get_all("one") == ["two", "three"] - - assert not h.add("~", "foo", "bar") diff --git a/test/mitmproxy/test_flow_format_compat.py b/test/mitmproxy/test_flow_format_compat.py index b2cef88d..cc80db81 100644 --- a/test/mitmproxy/test_flow_format_compat.py +++ b/test/mitmproxy/test_flow_format_compat.py @@ -4,7 +4,7 @@ from . import tutils def test_load(): - with open(tutils.test_data.path("data/dumpfile-013"), "rb") as f: + with open(tutils.test_data.path("data/dumpfile-011"), "rb") as f: flow_reader = FlowReader(f) flows = list(flow_reader.stream()) assert len(flows) == 1 @@ -12,7 +12,7 @@ def test_load(): def test_cannot_convert(): - with open(tutils.test_data.path("data/dumpfile-012"), "rb") as f: + with open(tutils.test_data.path("data/dumpfile-010"), "rb") as f: flow_reader = FlowReader(f) with tutils.raises(FlowReadException): list(flow_reader.stream()) diff --git a/test/mitmproxy/test_options.py b/test/mitmproxy/test_options.py new file mode 100644 index 00000000..af619b27 --- /dev/null +++ b/test/mitmproxy/test_options.py @@ -0,0 +1,100 @@ +from __future__ import absolute_import, print_function, division +import copy + +from mitmproxy import options +from mitmproxy import exceptions +from netlib import tutils + + +class TO(options.Options): + def __init__(self, one=None, two=None): + self.one = one + self.two = two + super(TO, self).__init__() + + +def test_options(): + o = TO(two="three") + assert o.one is None + assert o.two == "three" + o.one = "one" + assert o.one == "one" + + with tutils.raises(TypeError): + TO(nonexistent = "value") + with tutils.raises("no such option"): + o.nonexistent = "value" + with tutils.raises("no such option"): + o.update(nonexistent = "value") + + rec = [] + + def sub(opts): + rec.append(copy.copy(opts)) + + o.changed.connect(sub) + + o.one = "ninety" + assert len(rec) == 1 + assert rec[-1].one == "ninety" + + o.update(one="oink") + assert len(rec) == 2 + assert rec[-1].one == "oink" + + +def test_setter(): + o = TO(two="three") + f = o.setter("two") + f("xxx") + assert o.two == "xxx" + with tutils.raises("no such option"): + o.setter("nonexistent") + + +def test_toggler(): + o = TO(two=True) + f = o.toggler("two") + f() + assert o.two is False + f() + assert o.two is True + with tutils.raises("no such option"): + o.toggler("nonexistent") + + +def test_rollback(): + o = TO(one="two") + + rec = [] + + def sub(opts): + rec.append(copy.copy(opts)) + + recerr = [] + + def errsub(opts, **kwargs): + recerr.append(kwargs) + + def err(opts): + if opts.one == "ten": + raise exceptions.OptionsError() + + o.changed.connect(sub) + o.changed.connect(err) + o.errored.connect(errsub) + + o.one = "ten" + assert isinstance(recerr[0]["exc"], exceptions.OptionsError) + assert o.one == "two" + assert len(rec) == 2 + assert rec[0].one == "ten" + assert rec[1].one == "two" + + +def test_repr(): + assert repr(TO()) == "test.mitmproxy.test_options.TO({'one': None, 'two': None})" + assert repr(TO(one='x' * 60)) == """test.mitmproxy.test_options.TO({ + 'one': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', + 'two': None +})""" diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index 58ffb787..a7a3ba3f 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -9,6 +9,7 @@ import traceback import h2 +from mitmproxy.flow import options from mitmproxy.proxy.config import ProxyConfig from mitmproxy.cmdline import APP_HOST, APP_PORT @@ -60,7 +61,10 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): except HttpException: print(traceback.format_exc()) assert False + except netlib.exceptions.TcpDisconnect: + break except: + print(traceback.format_exc()) break self.wfile.write(h2_conn.data_to_send()) self.wfile.flush() @@ -70,8 +74,11 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): if not self.server.handle_server_event(event, h2_conn, self.rfile, self.wfile): done = True break + except netlib.exceptions.TcpDisconnect: + done = True except: done = True + print(traceback.format_exc()) break def handle_server_event(self, h2_conn, rfile, wfile): @@ -81,39 +88,37 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): class _Http2TestBase(object): @classmethod - def setup_class(self): - self.config = ProxyConfig(**self.get_proxy_config()) + def setup_class(cls): + cls.masteroptions = options.Options() + cnf, opts = cls.get_proxy_config() + cls.config = ProxyConfig(opts, **cnf) - tmaster = tservers.TestMaster(self.config) + tmaster = tservers.TestMaster(opts, cls.config) tmaster.start_app(APP_HOST, APP_PORT) - self.proxy = tservers.ProxyThread(tmaster) - self.proxy.start() + cls.proxy = tservers.ProxyThread(tmaster) + cls.proxy.start() @classmethod def teardown_class(cls): cls.proxy.shutdown() + @classmethod + def get_proxy_config(cls): + opts = options.Options(listen_port=0, no_upstream_cert=False) + opts.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy") + d = dict() + return d, opts + @property def master(self): return self.proxy.tmaster - @classmethod - def get_proxy_config(cls): - cls.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy") - return dict( - no_upstream_cert = False, - cadir = cls.cadir, - authenticator = None, - ) - def setup(self): self.master.clear_log() self.master.state.clear() self.server.server.handle_server_event = self.handle_server_event def _setup_connection(self): - self.config.http2 = True - client = netlib.tcp.TCPClient(("127.0.0.1", self.proxy.port)) client.connect() @@ -138,11 +143,26 @@ class _Http2TestBase(object): return client, h2_conn - def _send_request(self, wfile, h2_conn, stream_id=1, headers=[], body=b''): + def _send_request(self, + wfile, + h2_conn, + stream_id=1, + headers=[], + body=b'', + end_stream=None, + priority_exclusive=None, + priority_depends_on=None, + priority_weight=None): + if end_stream is None: + end_stream = (len(body) == 0) + h2_conn.send_headers( stream_id=stream_id, headers=headers, - end_stream=(len(body) == 0), + end_stream=end_stream, + priority_exclusive=priority_exclusive, + priority_depends_on=priority_depends_on, + priority_weight=priority_weight, ) if body: h2_conn.send_data(stream_id, body) @@ -151,8 +171,7 @@ class _Http2TestBase(object): wfile.flush() -@requires_alpn -class TestSimple(_Http2TestBase, _Http2ServerBase): +class _Http2Test(_Http2TestBase, _Http2ServerBase): @classmethod def setup_class(self): @@ -164,6 +183,11 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): _Http2TestBase.teardown_class() _Http2ServerBase.teardown_class() + +@requires_alpn +class TestSimple(_Http2Test): + request_body_buffer = b'' + @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): @@ -171,7 +195,7 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): elif isinstance(event, h2.events.RequestReceived): assert (b'client-foo', b'client-bar-1') in event.headers assert (b'client-foo', b'client-bar-2') in event.headers - + elif isinstance(event, h2.events.StreamEnded): import warnings with warnings.catch_warnings(): # Ignore UnicodeWarning: @@ -187,23 +211,30 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): ('föo', 'bär'), ('X-Stream-ID', str(event.stream_id)), ]) - h2_conn.send_data(event.stream_id, b'foobar') + h2_conn.send_data(event.stream_id, b'response body') h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() + elif isinstance(event, h2.events.DataReceived): + self.request_body_buffer += event.data return True def test_simple(self): + response_body_buffer = b'' client, h2_conn = self._setup_connection() - self._send_request(client.wfile, h2_conn, headers=[ - (':authority', "127.0.0.1:%s" % self.server.server.address.port), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ('ClIeNt-FoO', 'client-bar-1'), - ('ClIeNt-FoO', 'client-bar-2'), - ], body=b'my request body echoed back to me') + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ('ClIeNt-FoO', 'client-bar-1'), + ('ClIeNt-FoO', 'client-bar-2'), + ], + body=b'request body') done = False while not done: @@ -218,7 +249,9 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): client.wfile.flush() for event in events: - if isinstance(event, h2.events.StreamEnded): + if isinstance(event, h2.events.DataReceived): + response_body_buffer += event.data + elif isinstance(event, h2.events.StreamEnded): done = True h2_conn.close_connection() @@ -229,41 +262,226 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): assert self.master.state.flows[0].response.status_code == 200 assert self.master.state.flows[0].response.headers['server-foo'] == 'server-bar' assert self.master.state.flows[0].response.headers['föo'] == 'bär' - assert self.master.state.flows[0].response.body == b'foobar' + assert self.master.state.flows[0].response.body == b'response body' + assert self.request_body_buffer == b'request body' + assert response_body_buffer == b'response body' @requires_alpn -class TestWithBodies(_Http2TestBase, _Http2ServerBase): - tmp_data_buffer_foobar = b'' +class TestRequestWithPriority(_Http2Test): @classmethod - def setup_class(self): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class() + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.RequestReceived): + import warnings + with warnings.catch_warnings(): + # Ignore UnicodeWarning: + # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison + # failed to convert both arguments to Unicode - interpreting + # them as being unequal. + # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: + + warnings.simplefilter("ignore") + + headers = [(':status', '200')] + if event.priority_updated: + headers.append(('priority_exclusive', event.priority_updated.exclusive)) + headers.append(('priority_depends_on', event.priority_updated.depends_on)) + headers.append(('priority_weight', event.priority_updated.weight)) + h2_conn.send_headers(event.stream_id, headers) + h2_conn.end_stream(event.stream_id) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return True + + def test_request_with_priority(self): + client, h2_conn = self._setup_connection() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + priority_exclusive = True, + priority_depends_on = 42424242, + priority_weight = 42, + ) + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 1 + assert self.master.state.flows[0].response.headers['priority_exclusive'] == 'True' + assert self.master.state.flows[0].response.headers['priority_depends_on'] == '42424242' + assert self.master.state.flows[0].response.headers['priority_weight'] == '42' + + def test_request_without_priority(self): + client, h2_conn = self._setup_connection() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + ) + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 1 + assert 'priority_exclusive' not in self.master.state.flows[0].response.headers + assert 'priority_depends_on' not in self.master.state.flows[0].response.headers + assert 'priority_weight' not in self.master.state.flows[0].response.headers + + +@requires_alpn +class TestPriority(_Http2Test): + priority_data = None @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.PriorityUpdated): + self.priority_data = (event.exclusive, event.depends_on, event.weight) + elif isinstance(event, h2.events.RequestReceived): + import warnings + with warnings.catch_warnings(): + # Ignore UnicodeWarning: + # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison + # failed to convert both arguments to Unicode - interpreting + # them as being unequal. + # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: + + warnings.simplefilter("ignore") + + headers = [(':status', '200')] + h2_conn.send_headers(event.stream_id, headers) + h2_conn.end_stream(event.stream_id) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return True + + def test_priority(self): + client, h2_conn = self._setup_connection() + + h2_conn.prioritize(1, exclusive=True, depends_on=0, weight=42) + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + ) + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 1 + assert self.priority_data == (True, 0, 42) + + +@requires_alpn +class TestPriorityWithExistingStream(_Http2Test): + priority_data = [] @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): return False - if isinstance(event, h2.events.DataReceived): - self.tmp_data_buffer_foobar += event.data + elif isinstance(event, h2.events.PriorityUpdated): + self.priority_data.append((event.exclusive, event.depends_on, event.weight)) + elif isinstance(event, h2.events.RequestReceived): + assert not event.priority_updated + + import warnings + with warnings.catch_warnings(): + # Ignore UnicodeWarning: + # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison + # failed to convert both arguments to Unicode - interpreting + # them as being unequal. + # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: + + warnings.simplefilter("ignore") + + headers = [(':status', '200')] + h2_conn.send_headers(event.stream_id, headers) + wfile.write(h2_conn.data_to_send()) + wfile.flush() elif isinstance(event, h2.events.StreamEnded): - h2_conn.send_headers(1, [ - (':status', '200'), - ]) - h2_conn.send_data(1, self.tmp_data_buffer_foobar) - h2_conn.end_stream(1) + h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() - return True - def test_with_bodies(self): + def test_priority_with_existing_stream(self): client, h2_conn = self._setup_connection() self._send_request( @@ -275,9 +493,14 @@ class TestWithBodies(_Http2TestBase, _Http2ServerBase): (':scheme', 'https'), (':path', '/'), ], - body=b'foobar with request body', + end_stream=False, ) + h2_conn.prioritize(1, exclusive=True, depends_on=0, weight=42) + h2_conn.end_stream(1) + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + done = False while not done: try: @@ -298,21 +521,112 @@ class TestWithBodies(_Http2TestBase, _Http2ServerBase): client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() - assert self.master.state.flows[0].response.body == b'foobar with request body' + assert len(self.master.state.flows) == 1 + assert self.priority_data == [(True, 0, 42)] @requires_alpn -class TestPushPromise(_Http2TestBase, _Http2ServerBase): +class TestStreamResetFromServer(_Http2Test): @classmethod - def setup_class(self): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class() + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.RequestReceived): + h2_conn.reset_stream(event.stream_id, 0x8) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return True + + def test_request_with_priority(self): + client, h2_conn = self._setup_connection() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + ) + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamReset): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 1 + assert self.master.state.flows[0].response is None + + +@requires_alpn +class TestBodySizeLimit(_Http2Test): @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + return True + + def test_body_size_limit(self): + self.config.options.body_size_limit = 20 + + client, h2_conn = self._setup_connection() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + body=b'very long body over 20 characters long', + ) + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamReset): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 0 + + +@requires_alpn +class TestPushPromise(_Http2Test): @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): @@ -465,17 +779,7 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): @requires_alpn -class TestConnectionLost(_Http2TestBase, _Http2ServerBase): - - @classmethod - def setup_class(self): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class() - - @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() +class TestConnectionLost(_Http2Test): @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): @@ -517,7 +821,7 @@ class TestConnectionLost(_Http2TestBase, _Http2ServerBase): @requires_alpn -class TestMaxConcurrentStreams(_Http2TestBase, _Http2ServerBase): +class TestMaxConcurrentStreams(_Http2Test): @classmethod def setup_class(self): @@ -525,11 +829,6 @@ class TestMaxConcurrentStreams(_Http2TestBase, _Http2ServerBase): _Http2ServerBase.setup_class(h2_server_settings={h2.settings.MAX_CONCURRENT_STREAMS: 2}) @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() - - @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): return False @@ -583,17 +882,7 @@ class TestMaxConcurrentStreams(_Http2TestBase, _Http2ServerBase): @requires_alpn -class TestConnectionTerminated(_Http2TestBase, _Http2ServerBase): - - @classmethod - def setup_class(self): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class() - - @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() +class TestConnectionTerminated(_Http2Test): @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index cd24fc9f..7095d9d2 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -4,9 +4,10 @@ from OpenSSL import SSL from mitmproxy import cmdline from mitmproxy.proxy import ProxyConfig -from mitmproxy.proxy.config import process_proxy_options from mitmproxy.models.connections import ServerConnection from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler +from mitmproxy.flow import options +from mitmproxy.proxy import config from netlib.exceptions import TcpDisconnect from pathod import test from netlib.http import http1 @@ -58,8 +59,10 @@ class TestProcessProxyOptions: def p(self, *args): parser = tutils.MockParser() cmdline.common_options(parser) - opts = parser.parse_args(args=args) - return parser, process_proxy_options(parser, opts) + args = parser.parse_args(args=args) + opts = cmdline.get_common_options(args) + pconf = config.ProxyConfig(options.Options(**opts)) + return parser, pconf def assert_err(self, err, *args): tutils.raises(err, self.p, *args) @@ -82,24 +85,29 @@ class TestProcessProxyOptions: @mock.patch("mitmproxy.platform.resolver") def test_modes(self, _): - self.assert_noerr("-R", "http://localhost") - self.assert_err("expected one argument", "-R") - self.assert_err("Invalid server specification", "-R", "reverse") - - self.assert_noerr("-T") - - self.assert_noerr("-U", "http://localhost") - self.assert_err("expected one argument", "-U") - self.assert_err("Invalid server specification", "-U", "upstream") - - self.assert_noerr("--upstream-auth", "test:test") - self.assert_err("expected one argument", "--upstream-auth") - self.assert_err("Invalid upstream auth specification", "--upstream-auth", "test") - - self.assert_err("mutually exclusive", "-R", "http://localhost", "-T") + # self.assert_noerr("-R", "http://localhost") + # self.assert_err("expected one argument", "-R") + # self.assert_err("Invalid server specification", "-R", "reverse") + # + # self.assert_noerr("-T") + # + # self.assert_noerr("-U", "http://localhost") + # self.assert_err("expected one argument", "-U") + # self.assert_err("Invalid server specification", "-U", "upstream") + # + # self.assert_noerr("--upstream-auth", "test:test") + # self.assert_err("expected one argument", "--upstream-auth") + self.assert_err( + "Invalid upstream auth specification", "--upstream-auth", "test" + ) + # self.assert_err("mutually exclusive", "-R", "http://localhost", "-T") def test_socks_auth(self): - self.assert_err("Proxy Authentication not supported in SOCKS mode.", "--socks", "--nonanonymous") + self.assert_err( + "Proxy Authentication not supported in SOCKS mode.", + "--socks", + "--nonanonymous" + ) def test_client_certs(self): with tutils.tmpdir() as cadir: @@ -145,12 +153,12 @@ class TestProcessProxyOptions: def test_upstream_trusted_cadir(self): expected_dir = "/path/to/a/ca/dir" p = self.assert_noerr("--upstream-trusted-cadir", expected_dir) - assert p.openssl_trusted_cadir_server == expected_dir + assert p.options.ssl_verify_upstream_trusted_cadir == expected_dir def test_upstream_trusted_ca(self): expected_file = "/path/to/a/cert/file" p = self.assert_noerr("--upstream-trusted-ca", expected_file) - assert p.openssl_trusted_ca_server == expected_file + assert p.options.ssl_verify_upstream_trusted_ca == expected_file class TestProxyServer: @@ -159,13 +167,13 @@ class TestProxyServer: @tutils.skip_windows def test_err(self): conf = ProxyConfig( - port=1 + options.Options(listen_port=1), ) tutils.raises("error starting proxy server", ProxyServer, conf) def test_err_2(self): conf = ProxyConfig( - host="invalidhost" + options.Options(listen_host="invalidhost"), ) tutils.raises("error starting proxy server", ProxyServer, conf) @@ -184,7 +192,7 @@ class TestConnectionHandler: config = mock.Mock() root_layer = mock.Mock() root_layer.side_effect = RuntimeError - config.mode.return_value = root_layer + config.options.mode.return_value = root_layer channel = mock.Mock() def ask(_, x): diff --git a/test/mitmproxy/test_proxy_config.py b/test/mitmproxy/test_proxy_config.py new file mode 100644 index 00000000..d8085eb8 --- /dev/null +++ b/test/mitmproxy/test_proxy_config.py @@ -0,0 +1,48 @@ +from . import tutils +import base64 +from mitmproxy.proxy import config + + +def test_parse_server_spec(): + tutils.raises( + "Invalid server specification", config.parse_server_spec, "" + ) + assert config.parse_server_spec("http://foo.com:88") == ( + "http", ("foo.com", 88) + ) + assert config.parse_server_spec("http://foo.com") == ( + "http", ("foo.com", 80) + ) + assert config.parse_server_spec("https://foo.com") == ( + "https", ("foo.com", 443) + ) + tutils.raises( + "Invalid server specification", + config.parse_server_spec, + "foo.com" + ) + tutils.raises( + "Invalid server specification", + config.parse_server_spec, + "http://" + ) + + +def test_parse_upstream_auth(): + tutils.raises( + "Invalid upstream auth specification", + config.parse_upstream_auth, + "" + ) + tutils.raises( + "Invalid upstream auth specification", + config.parse_upstream_auth, + ":" + ) + tutils.raises( + "Invalid upstream auth specification", + config.parse_upstream_auth, + ":test" + ) + assert config.parse_upstream_auth("test:test") == b"Basic" + b" " + base64.b64encode(b"test:test") + assert config.parse_upstream_auth("test:") == b"Basic" + b" " + base64.b64encode(b"test:") diff --git a/test/mitmproxy/test_script.py b/test/mitmproxy/test_script.py deleted file mode 100644 index 81994780..00000000 --- a/test/mitmproxy/test_script.py +++ /dev/null @@ -1,13 +0,0 @@ -from mitmproxy import flow -from . import tutils - - -def test_duplicate_flow(): - s = flow.State() - fm = flow.FlowMaster(None, s) - fm.load_script(tutils.test_data.path("data/scripts/duplicate_flow.py")) - f = tutils.tflow() - fm.request(f) - assert fm.state.flow_count() == 2 - assert not fm.state.view[0].request.is_replay - assert fm.state.view[1].request.is_replay diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 0ab7624e..b8b057fd 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -1,6 +1,7 @@ import os import socket import time +import types from OpenSSL import SSL from netlib.exceptions import HttpReadDisconnect, HttpException from netlib.tcp import Address @@ -12,8 +13,9 @@ from netlib.http import authentication, http1 from netlib.tutils import raises from pathod import pathoc, pathod +from mitmproxy.builtins import script from mitmproxy import controller -from mitmproxy.proxy.config import HostMatcher +from mitmproxy.proxy.config import HostMatcher, parse_server_spec from mitmproxy.models import Error, HTTPResponse, HTTPFlow from . import tutils, tservers @@ -286,20 +288,18 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin): self.master.set_stream_large_bodies(None) def test_stream_modify(self): - self.master.load_script(tutils.test_data.path("data/scripts/stream_modify.py")) + s = script.Script( + tutils.test_data.path("data/addonscripts/stream_modify.py") + ) + self.master.addons.add(s) d = self.pathod('200:b"foo"') assert d.content == b"bar" - self.master.unload_scripts() + self.master.addons.remove(s) class TestHTTPAuth(tservers.HTTPProxyTest): - authenticator = http.authentication.BasicProxyAuth( - http.authentication.PassManSingleUser( - "test", - "test"), - "realm") - def test_auth(self): + self.master.options.auth_singleuser = "test:test" assert self.pathod("202").status_code == 407 p = self.pathoc() ret = p.request(""" @@ -363,15 +363,17 @@ class TestHTTPSUpstreamServerVerificationWTrustedCert(tservers.HTTPProxyTest): ]) def test_verification_w_cadir(self): - self.config.openssl_verification_mode_server = SSL.VERIFY_PEER - self.config.openssl_trusted_cadir_server = tutils.test_data.path( - "data/trusted-cadir/") - + self.config.options.update( + ssl_verify_upstream_cert = True, + ssl_verify_upstream_trusted_cadir = tutils.test_data.path( + "data/trusted-cadir/" + ) + ) self.pathoc() def test_verification_w_pemfile(self): self.config.openssl_verification_mode_server = SSL.VERIFY_PEER - self.config.openssl_trusted_ca_server = tutils.test_data.path( + self.config.options.ssl_verify_upstream_trusted_ca = tutils.test_data.path( "data/trusted-cadir/trusted-ca.pem") self.pathoc() @@ -396,23 +398,29 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest): def test_default_verification_w_bad_cert(self): """Should use no verification.""" - self.config.openssl_trusted_ca_server = tutils.test_data.path( - "data/trusted-cadir/trusted-ca.pem") - + self.config.options.update( + ssl_verify_upstream_trusted_ca = tutils.test_data.path( + "data/trusted-cadir/trusted-ca.pem" + ) + ) assert self._request().status_code == 242 def test_no_verification_w_bad_cert(self): - self.config.openssl_verification_mode_server = SSL.VERIFY_NONE - self.config.openssl_trusted_ca_server = tutils.test_data.path( - "data/trusted-cadir/trusted-ca.pem") - + self.config.options.update( + ssl_verify_upstream_cert = False, + ssl_verify_upstream_trusted_ca = tutils.test_data.path( + "data/trusted-cadir/trusted-ca.pem" + ) + ) assert self._request().status_code == 242 def test_verification_w_bad_cert(self): - self.config.openssl_verification_mode_server = SSL.VERIFY_PEER - self.config.openssl_trusted_ca_server = tutils.test_data.path( - "data/trusted-cadir/trusted-ca.pem") - + self.config.options.update( + ssl_verify_upstream_cert = True, + ssl_verify_upstream_trusted_ca = tutils.test_data.path( + "data/trusted-cadir/trusted-ca.pem" + ) + ) assert self._request().status_code == 502 @@ -479,9 +487,10 @@ class TestHttps2Http(tservers.ReverseProxyTest): @classmethod def get_proxy_config(cls): - d = super(TestHttps2Http, cls).get_proxy_config() - d["upstream_server"] = ("http", d["upstream_server"][1]) - return d + d, opts = super(TestHttps2Http, cls).get_proxy_config() + s = parse_server_spec(opts.upstream_server) + opts.upstream_server = "http://%s" % s.address + return d, opts def pathoc(self, ssl, sni=None): """ @@ -511,15 +520,15 @@ class TestTransparent(tservers.TransparentProxyTest, CommonMixin, TcpMixin): ssl = False def test_tcp_stream_modify(self): - self.master.load_script(tutils.test_data.path("data/scripts/tcp_stream_modify.py")) - + s = script.Script( + tutils.test_data.path("data/addonscripts/tcp_stream_modify.py") + ) + self.master.addons.add(s) self._tcpproxy_on() d = self.pathod('200:b"foo"') self._tcpproxy_off() - assert d.content == b"bar" - - self.master.unload_scripts() + self.master.addons.remove(s) class TestTransparentSSL(tservers.TransparentProxyTest, CommonMixin, TcpMixin): @@ -834,17 +843,12 @@ class TestUpstreamProxy(tservers.HTTPUpstreamProxyTest, CommonMixin, AppMixin): ssl = False def test_order(self): - self.proxy.tmaster.replacehooks.add( - "~q", - "foo", - "bar") # replace in request - self.chain[0].tmaster.replacehooks.add("~q", "bar", "baz") - self.chain[1].tmaster.replacehooks.add("~q", "foo", "oh noes!") - self.chain[0].tmaster.replacehooks.add( - "~s", - "baz", - "ORLY") # replace in response - + self.proxy.tmaster.options.replacements = [ + ("~q", "foo", "bar"), + ("~q", "bar", "baz"), + ("~q", "foo", "oh noes!"), + ("~s", "baz", "ORLY") + ] p = self.pathoc() req = p.request("get:'%s/p/418:b\"foo\"'" % self.server.urlbase) assert req.content == b"ORLY" @@ -945,7 +949,7 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest): f.reply.kill() return _func(f) - setattr(master, attr, handler) + setattr(master, attr, types.MethodType(handler, master)) kill_requests( self.chain[1].tmaster, diff --git a/test/mitmproxy/test_web_master.py b/test/mitmproxy/test_web_master.py index f0fafe24..2ab440ce 100644 --- a/test/mitmproxy/test_web_master.py +++ b/test/mitmproxy/test_web_master.py @@ -3,15 +3,12 @@ from . import mastertest class TestWebMaster(mastertest.MasterTest): - def mkmaster(self, filt, **options): - o = master.Options( - filtstr=filt, - **options - ) + def mkmaster(self, **options): + o = master.Options(**options) return master.WebMaster(None, o) def test_basic(self): - m = self.mkmaster(None) + m = self.mkmaster() for i in (1, 2, 3): self.dummy_cycle(m, 1, b"") assert len(m.state.flows) == i diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 0760cb53..495765da 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -9,7 +9,9 @@ from mitmproxy.proxy.server import ProxyServer import pathod.test import pathod.pathoc from mitmproxy import flow, controller +from mitmproxy.flow import options from mitmproxy.cmdline import APP_HOST, APP_PORT +from mitmproxy import builtins testapp = flask.Flask(__name__) @@ -30,11 +32,11 @@ def errapp(environ, start_response): class TestMaster(flow.FlowMaster): - def __init__(self, config): - config.port = 0 + def __init__(self, opts, config): s = ProxyServer(config) state = flow.State() - flow.FlowMaster.__init__(self, s, state) + flow.FlowMaster.__init__(self, opts, s, state) + self.addons.add(*builtins.default_addons()) self.apps.add(testapp, "testapp", 80) self.apps.add(errapp, "errapp", 80) self.clear_log() @@ -42,7 +44,7 @@ class TestMaster(flow.FlowMaster): def clear_log(self): self.tlog = [] - def add_event(self, message, level=None): + def add_log(self, message, level=None): self.tlog.append(message) @@ -52,7 +54,8 @@ class ProxyThread(threading.Thread): threading.Thread.__init__(self) self.tmaster = tmaster self.name = "ProxyThread (%s:%s)" % ( - tmaster.server.address.host, tmaster.server.address.port) + tmaster.server.address.host, tmaster.server.address.port + ) controller.should_exit = False @property @@ -75,7 +78,6 @@ class ProxyTestBase(object): ssl = None ssloptions = False no_upstream_cert = False - authenticator = None masterclass = TestMaster add_upstream_certs_to_client_chain = False @@ -88,9 +90,9 @@ class ProxyTestBase(object): ssl=cls.ssl, ssloptions=cls.ssloptions) - cls.config = ProxyConfig(**cls.get_proxy_config()) - - tmaster = cls.masterclass(cls.config) + cnf, opts = cls.get_proxy_config() + cls.config = ProxyConfig(opts, **cnf) + tmaster = cls.masterclass(opts, cls.config) tmaster.start_app(APP_HOST, APP_PORT) cls.proxy = ProxyThread(tmaster) cls.proxy.start() @@ -117,11 +119,12 @@ class ProxyTestBase(object): @classmethod def get_proxy_config(cls): cls.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy") - return dict( + cnf = dict() + return cnf, options.Options( + listen_port=0, + cadir=cls.cadir, no_upstream_cert = cls.no_upstream_cert, - cadir = cls.cadir, - authenticator = cls.authenticator, - add_upstream_certs_to_client_chain = cls.add_upstream_certs_to_client_chain, + add_upstream_certs_to_client_chain=cls.add_upstream_certs_to_client_chain ) @@ -196,9 +199,9 @@ class TransparentProxyTest(ProxyTestBase): @classmethod def get_proxy_config(cls): - d = ProxyTestBase.get_proxy_config() - d["mode"] = "transparent" - return d + d, opts = ProxyTestBase.get_proxy_config() + opts.mode = "transparent" + return d, opts def pathod(self, spec, sni=None): """ @@ -228,13 +231,17 @@ class ReverseProxyTest(ProxyTestBase): @classmethod def get_proxy_config(cls): - d = ProxyTestBase.get_proxy_config() - d["upstream_server"] = ( - "https" if cls.ssl else "http", - ("127.0.0.1", cls.server.port) + d, opts = ProxyTestBase.get_proxy_config() + opts.upstream_server = "".join( + [ + "https" if cls.ssl else "http", + "://", + "127.0.0.1:", + str(cls.server.port) + ] ) - d["mode"] = "reverse" - return d + opts.mode = "reverse" + return d, opts def pathoc(self, sni=None): """ @@ -263,9 +270,9 @@ class SocksModeTest(HTTPProxyTest): @classmethod def get_proxy_config(cls): - d = ProxyTestBase.get_proxy_config() - d["mode"] = "socks5" - return d + d, opts = ProxyTestBase.get_proxy_config() + opts.mode = "socks5" + return d, opts class ChainProxyTest(ProxyTestBase): @@ -284,15 +291,16 @@ class ChainProxyTest(ProxyTestBase): cls.chain = [] super(ChainProxyTest, cls).setup_class() for _ in range(cls.n): - config = ProxyConfig(**cls.get_proxy_config()) - tmaster = cls.masterclass(config) + cnf, opts = cls.get_proxy_config() + config = ProxyConfig(opts, **cnf) + tmaster = cls.masterclass(opts, config) proxy = ProxyThread(tmaster) proxy.start() cls.chain.insert(0, proxy) # Patch the orginal proxy to upstream mode - cls.config = cls.proxy.tmaster.config = cls.proxy.tmaster.server.config = ProxyConfig( - **cls.get_proxy_config()) + cnf, opts = cls.get_proxy_config() + cls.config = cls.proxy.tmaster.config = cls.proxy.tmaster.server.config = ProxyConfig(opts, **cnf) @classmethod def teardown_class(cls): @@ -308,13 +316,13 @@ class ChainProxyTest(ProxyTestBase): @classmethod def get_proxy_config(cls): - d = super(ChainProxyTest, cls).get_proxy_config() + d, opts = super(ChainProxyTest, cls).get_proxy_config() if cls.chain: # First proxy is in normal mode. - d.update( + opts.update( mode="upstream", - upstream_server=("http", ("127.0.0.1", cls.chain[0].port)) + upstream_server="http://127.0.0.1:%s" % cls.chain[0].port ) - return d + return d, opts class HTTPUpstreamProxyTest(ChainProxyTest, HTTPProxyTest): diff --git a/test/mitmproxy/tutils.py b/test/mitmproxy/tutils.py index d0a09035..d743a9e6 100644 --- a/test/mitmproxy/tutils.py +++ b/test/mitmproxy/tutils.py @@ -4,18 +4,19 @@ import tempfile import argparse import sys -from mitmproxy.models.tcp import TCPMessage -from six.moves import cStringIO as StringIO from contextlib import contextmanager - from unittest.case import SkipTest +from six.moves import cStringIO as StringIO + import netlib.utils import netlib.tutils from mitmproxy import controller from mitmproxy.models import ( ClientConnection, ServerConnection, Error, HTTPRequest, HTTPResponse, HTTPFlow, TCPFlow ) +from mitmproxy.models.tcp import TCPMessage +from mitmproxy.models.flow import Flow def _skip_windows(*args): @@ -47,6 +48,27 @@ def skip_appveyor(fn): return fn +class DummyFlow(Flow): + """A flow that is neither HTTP nor TCP.""" + + def __init__(self, client_conn, server_conn, live=None): + super(DummyFlow, self).__init__("dummy", client_conn, server_conn, live) + + +def tdummyflow(client_conn=True, server_conn=True, err=None): + if client_conn is True: + client_conn = tclient_conn() + if server_conn is True: + server_conn = tserver_conn() + if err is True: + err = terr() + + f = DummyFlow(client_conn, server_conn) + f.error = err + f.reply = controller.DummyReply() + return f + + def ttcpflow(client_conn=True, server_conn=True, messages=True, err=None): if client_conn is True: client_conn = tclient_conn() diff --git a/test/netlib/http/http1/test_read.py b/test/netlib/http/http1/test_read.py index 5285ac1d..c8a40ecb 100644 --- a/test/netlib/http/http1/test_read.py +++ b/test/netlib/http/http1/test_read.py @@ -1,6 +1,9 @@ from __future__ import absolute_import, print_function, division + from io import BytesIO from mock import Mock +import pytest + from netlib.exceptions import HttpException, HttpSyntaxException, HttpReadDisconnect, TcpDisconnect from netlib.http import Headers from netlib.http.http1.read import ( @@ -23,11 +26,18 @@ def test_get_header_tokens(): assert get_header_tokens(headers, "foo") == ["bar", "voing", "oink"] -def test_read_request(): - rfile = BytesIO(b"GET / HTTP/1.1\r\n\r\nskip") +@pytest.mark.parametrize("input", [ + b"GET / HTTP/1.1\r\n\r\nskip", + b"GET / HTTP/1.1\r\n\r\nskip", + b"GET / HTTP/1.1\r\n\r\nskip", + b"GET / HTTP/1.1 \r\n\r\nskip", +]) +def test_read_request(input): + rfile = BytesIO(input) r = read_request(rfile) assert r.method == "GET" assert r.content == b"" + assert r.http_version == "HTTP/1.1" assert r.timestamp_end assert rfile.read() == b"skip" @@ -50,11 +60,19 @@ def test_read_request_head(): assert rfile.read() == b"skip" -def test_read_response(): +@pytest.mark.parametrize("input", [ + b"HTTP/1.1 418 I'm a teapot\r\n\r\nbody", + b"HTTP/1.1 418 I'm a teapot\r\n\r\nbody", + b"HTTP/1.1 418 I'm a teapot\r\n\r\nbody", + b"HTTP/1.1 418 I'm a teapot \r\n\r\nbody", +]) +def test_read_response(input): req = treq() - rfile = BytesIO(b"HTTP/1.1 418 I'm a teapot\r\n\r\nbody") + rfile = BytesIO(input) r = read_response(rfile, req) + assert r.http_version == "HTTP/1.1" assert r.status_code == 418 + assert r.reason == "I'm a teapot" assert r.content == b"body" assert r.timestamp_end diff --git a/test/netlib/http/test_cookies.py b/test/netlib/http/test_cookies.py index 83b85656..17e21b94 100644 --- a/test/netlib/http/test_cookies.py +++ b/test/netlib/http/test_cookies.py @@ -245,3 +245,24 @@ def test_refresh_cookie(): assert cookies.refresh_set_cookie_header(c, 0) c = "foo/bar=bla" assert cookies.refresh_set_cookie_header(c, 0) + + +def test_is_expired(): + CA = cookies.CookieAttrs + + # A cookie can be expired + # by setting the expire time in the past + assert cookies.is_expired(CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT")])) + + # or by setting Max-Age to 0 + assert cookies.is_expired(CA([("Max-Age", "0")])) + + # or both + assert cookies.is_expired(CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT"), ("Max-Age", "0")])) + + assert not cookies.is_expired(CA([("Expires", "Thu, 24-Aug-2063 00:00:00 GMT")])) + assert not cookies.is_expired(CA([("Max-Age", "1")])) + assert not cookies.is_expired(CA([("Expires", "Thu, 15-Jul-2068 00:00:00 GMT"), ("Max-Age", "1")])) + + assert not cookies.is_expired(CA([("Max-Age", "nan")])) + assert not cookies.is_expired(CA([("Expires", "false")])) diff --git a/test/netlib/http/test_headers.py b/test/netlib/http/test_headers.py index 51819b86..51537310 100644 --- a/test/netlib/http/test_headers.py +++ b/test/netlib/http/test_headers.py @@ -1,4 +1,6 @@ -from netlib.http import Headers, parse_content_type +import collections + +from netlib.http.headers import Headers, parse_content_type, assemble_content_type from netlib.tutils import raises @@ -81,3 +83,10 @@ def test_parse_content_type(): v = p("text/html; charset=UTF-8") assert v == ('text', 'html', {'charset': 'UTF-8'}) + + +def test_assemble_content_type(): + p = assemble_content_type + assert p("text", "html", {}) == "text/html" + assert p("text", "html", {"charset": "utf8"}) == "text/html; charset=utf8" + assert p("text", "html", collections.OrderedDict([("charset", "utf8"), ("foo", "bar")])) == "text/html; charset=utf8; foo=bar" diff --git a/test/netlib/http/test_message.py b/test/netlib/http/test_message.py index ab2ac628..deebd6f2 100644 --- a/test/netlib/http/test_message.py +++ b/test/netlib/http/test_message.py @@ -1,8 +1,11 @@ # -*- coding: utf-8 -*- from __future__ import absolute_import, print_function, division -from netlib.http import decoded +import mock +import six + from netlib.tutils import tresp +from netlib import http, tutils def _test_passthrough_attr(message, attr): @@ -68,6 +71,15 @@ class TestMessage(object): assert resp != 0 + def test_hash(self): + resp = tresp() + assert hash(resp) + + def test_serializable(self): + resp = tresp() + resp2 = http.Response.from_state(resp.get_state()) + assert resp == resp2 + def test_content_length_update(self): resp = tresp() resp.content = b"foo" @@ -76,9 +88,9 @@ class TestMessage(object): resp.content = b"" assert resp.data.content == b"" assert resp.headers["content-length"] == "0" - - def test_content_basic(self): - _test_passthrough_attr(tresp(), "content") + resp.raw_content = b"bar" + assert resp.data.content == b"bar" + assert resp.headers["content-length"] == "0" def test_headers(self): _test_passthrough_attr(tresp(), "headers") @@ -89,65 +101,201 @@ class TestMessage(object): def test_timestamp_end(self): _test_passthrough_attr(tresp(), "timestamp_end") - def teste_http_version(self): + def test_http_version(self): _test_decoded_attr(tresp(), "http_version") -class TestDecodedDecorator(object): - +class TestMessageContentEncoding(object): def test_simple(self): r = tresp() - assert r.content == b"message" + assert r.raw_content == b"message" assert "content-encoding" not in r.headers - assert r.encode("gzip") + r.encode("gzip") assert r.headers["content-encoding"] - assert r.content != b"message" - with decoded(r): - assert "content-encoding" not in r.headers - assert r.content == b"message" - assert r.headers["content-encoding"] - assert r.content != b"message" + assert r.raw_content != b"message" + assert r.content == b"message" + assert r.raw_content != b"message" + + r.raw_content = b"foo" + with mock.patch("netlib.encoding.decode") as e: + assert r.content + assert e.call_count == 1 + e.reset_mock() + assert r.content + assert e.call_count == 0 def test_modify(self): r = tresp() assert "content-encoding" not in r.headers - assert r.encode("gzip") + r.encode("gzip") + + r.content = b"foo" + assert r.raw_content != b"foo" + r.decode() + assert r.raw_content == b"foo" - with decoded(r): + r.encode("identity") + with mock.patch("netlib.encoding.encode") as e: r.content = b"foo" + assert e.call_count == 0 + r.content = b"bar" + assert e.call_count == 1 - assert r.content != b"foo" - r.decode() - assert r.content == b"foo" + with tutils.raises(TypeError): + r.content = u"foo" def test_unknown_ce(self): r = tresp() r.headers["content-encoding"] = "zopfli" - r.content = b"foo" - with decoded(r): - assert r.headers["content-encoding"] - assert r.content == b"foo" + r.raw_content = b"foo" + with tutils.raises(ValueError): + assert r.content assert r.headers["content-encoding"] - assert r.content == b"foo" + assert r.get_content(strict=False) == b"foo" def test_cannot_decode(self): r = tresp() - assert r.encode("gzip") - r.content = b"foo" - with decoded(r): - assert r.headers["content-encoding"] - assert r.content == b"foo" + r.encode("gzip") + r.raw_content = b"foo" + with tutils.raises(ValueError): + assert r.content assert r.headers["content-encoding"] - assert r.content != b"foo" - r.decode() + assert r.get_content(strict=False) == b"foo" + + with tutils.raises(ValueError): + r.decode() + assert r.raw_content == b"foo" + assert "content-encoding" in r.headers + + r.decode(strict=False) assert r.content == b"foo" + assert "content-encoding" not in r.headers + + def test_none(self): + r = tresp(content=None) + assert r.content is None + r.content = b"foo" + assert r.content is not None + r.content = None + assert r.content is None def test_cannot_encode(self): r = tresp() - assert r.encode("gzip") - with decoded(r): - r.content = None + r.encode("gzip") + r.content = None + assert r.headers["content-encoding"] + assert r.raw_content is None + r.headers["content-encoding"] = "zopfli" + r.content = b"foo" assert "content-encoding" not in r.headers - assert r.content is None + assert r.raw_content == b"foo" + + with tutils.raises(ValueError): + r.encode("zopfli") + assert r.raw_content == b"foo" + assert "content-encoding" not in r.headers + + +class TestMessageText(object): + def test_simple(self): + r = tresp(content=b'\xfc') + assert r.raw_content == b"\xfc" + assert r.content == b"\xfc" + assert r.text == u"ü" + + r.encode("gzip") + assert r.text == u"ü" + r.decode() + assert r.text == u"ü" + + r.headers["content-type"] = "text/html; charset=latin1" + r.content = b"\xc3\xbc" + assert r.text == u"ü" + r.headers["content-type"] = "text/html; charset=utf8" + assert r.text == u"ü" + + r.encode("identity") + r.raw_content = b"foo" + with mock.patch("netlib.encoding.decode") as e: + assert r.text + assert e.call_count == 2 + e.reset_mock() + assert r.text + assert e.call_count == 0 + + def test_guess_json(self): + r = tresp(content=b'"\xc3\xbc"') + r.headers["content-type"] = "application/json" + assert r.text == u'"ü"' + + def test_none(self): + r = tresp(content=None) + assert r.text is None + r.text = u"foo" + assert r.text is not None + r.text = None + assert r.text is None + + def test_modify(self): + r = tresp() + + r.text = u"ü" + assert r.raw_content == b"\xfc" + + r.headers["content-type"] = "text/html; charset=utf8" + r.text = u"ü" + assert r.raw_content == b"\xc3\xbc" + assert r.headers["content-length"] == "2" + + r.encode("identity") + with mock.patch("netlib.encoding.encode") as e: + e.return_value = b"" + r.text = u"ü" + assert e.call_count == 0 + r.text = u"ä" + assert e.call_count == 2 + + def test_unknown_ce(self): + r = tresp() + r.headers["content-type"] = "text/html; charset=wtf" + r.raw_content = b"foo" + with tutils.raises(ValueError): + assert r.text == u"foo" + assert r.get_text(strict=False) == u"foo" + + def test_cannot_decode(self): + r = tresp() + r.headers["content-type"] = "text/html; charset=utf8" + r.raw_content = b"\xFF" + with tutils.raises(ValueError): + assert r.text + + assert r.get_text(strict=False) == u'\ufffd' if six.PY2 else '\udcff' + + def test_cannot_encode(self): + r = tresp() + r.content = None + assert "content-type" not in r.headers + assert r.raw_content is None + + r.headers["content-type"] = "text/html; charset=latin1; foo=bar" + r.text = u"☃" + assert r.headers["content-type"] == "text/html; charset=utf-8; foo=bar" + assert r.raw_content == b'\xe2\x98\x83' + + r.headers["content-type"] = "gibberish" + r.text = u"☃" + assert r.headers["content-type"] == "text/plain; charset=utf-8" + assert r.raw_content == b'\xe2\x98\x83' + + del r.headers["content-type"] + r.text = u"☃" + assert r.headers["content-type"] == "text/plain; charset=utf-8" + assert r.raw_content == b'\xe2\x98\x83' + + r.headers["content-type"] = "text/html; charset=latin1" + r.text = u'\udcff' + assert r.headers["content-type"] == "text/html; charset=utf-8" + assert r.raw_content == b'\xed\xb3\xbf' if six.PY2 else b"\xFF" diff --git a/test/netlib/test_encoding.py b/test/netlib/test_encoding.py index 0ff1aad1..de10fc48 100644 --- a/test/netlib/test_encoding.py +++ b/test/netlib/test_encoding.py @@ -1,37 +1,39 @@ -from netlib import encoding +from netlib import encoding, tutils def test_identity(): - assert b"string" == encoding.decode("identity", b"string") - assert b"string" == encoding.encode("identity", b"string") - assert not encoding.encode("nonexistent", b"string") - assert not encoding.decode("nonexistent encoding", b"string") + assert b"string" == encoding.decode(b"string", "identity") + assert b"string" == encoding.encode(b"string", "identity") + with tutils.raises(ValueError): + encoding.encode(b"string", "nonexistent encoding") def test_gzip(): assert b"string" == encoding.decode( - "gzip", encoding.encode( - "gzip", - b"string" - ) + b"string", + "gzip" + ), + "gzip" ) - assert encoding.decode("gzip", b"bogus") is None + with tutils.raises(ValueError): + encoding.decode(b"bogus", "gzip") def test_deflate(): assert b"string" == encoding.decode( - "deflate", encoding.encode( - "deflate", - b"string" - ) + b"string", + "deflate" + ), + "deflate" ) assert b"string" == encoding.decode( - "deflate", encoding.encode( - "deflate", - b"string" - )[2:-4] + b"string", + "deflate" + )[2:-4], + "deflate" ) - assert encoding.decode("deflate", b"bogus") is None + with tutils.raises(ValueError): + encoding.decode(b"bogus", "deflate") diff --git a/test/netlib/test_strutils.py b/test/netlib/test_strutils.py index 68bfdb94..7c3eacc6 100644 --- a/test/netlib/test_strutils.py +++ b/test/netlib/test_strutils.py @@ -38,8 +38,9 @@ def test_escape_control_characters(): u'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.' ) - with tutils.raises(ValueError): - strutils.escape_control_characters(b"foo") + if not six.PY2: + with tutils.raises(ValueError): + strutils.escape_control_characters(b"foo") def test_bytes_to_escaped_str(): |