diff options
Diffstat (limited to 'test')
57 files changed, 1395 insertions, 956 deletions
diff --git a/test/mitmproxy/builtins/__init__.py b/test/mitmproxy/builtins/__init__.py new file mode 100644 index 00000000..e69de29b --- /dev/null +++ b/test/mitmproxy/builtins/__init__.py diff --git a/test/mitmproxy/builtins/test_anticache.py b/test/mitmproxy/builtins/test_anticache.py new file mode 100644 index 00000000..127e1c1a --- /dev/null +++ b/test/mitmproxy/builtins/test_anticache.py @@ -0,0 +1,23 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import anticache +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy.flow import options + + +class TestAntiCache(mastertest.MasterTest): + def test_simple(self): + s = state.State() + m = master.FlowMaster(options.Options(anticache = True), None, s) + sa = anticache.AntiCache() + m.addons.add(sa) + + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + + f = tutils.tflow(resp=True) + f.request.headers["if-modified-since"] = "test" + f.request.headers["if-none-match"] = "test" + self.invoke(m, "request", f) + assert "if-modified-since" not in f.request.headers + assert "if-none-match" not in f.request.headers diff --git a/test/mitmproxy/builtins/test_anticomp.py b/test/mitmproxy/builtins/test_anticomp.py new file mode 100644 index 00000000..601e56c8 --- /dev/null +++ b/test/mitmproxy/builtins/test_anticomp.py @@ -0,0 +1,22 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import anticomp +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy.flow import options + + +class TestAntiComp(mastertest.MasterTest): + def test_simple(self): + s = state.State() + m = master.FlowMaster(options.Options(anticomp = True), None, s) + sa = anticomp.AntiComp() + m.addons.add(sa) + + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + + f = tutils.tflow(resp=True) + + f.request.headers["Accept-Encoding"] = "foobar" + self.invoke(m, "request", f) + assert f.request.headers["Accept-Encoding"] == "identity" diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py new file mode 100644 index 00000000..c9616249 --- /dev/null +++ b/test/mitmproxy/builtins/test_script.py @@ -0,0 +1,191 @@ +import time + +from mitmproxy.builtins import script +from mitmproxy import exceptions +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy.flow import options + +from .. import tutils, mastertest + + +class TestParseCommand: + def test_empty_command(self): + with tutils.raises(exceptions.AddonError): + script.parse_command("") + + with tutils.raises(exceptions.AddonError): + script.parse_command(" ") + + def test_no_script_file(self): + with tutils.raises("not found"): + script.parse_command("notfound") + + with tutils.tmpdir() as dir: + with tutils.raises("not a file"): + script.parse_command(dir) + + def test_parse_args(self): + with tutils.chdir(tutils.test_data.dirname): + assert script.parse_command("data/addonscripts/recorder.py") == ("data/addonscripts/recorder.py", []) + assert script.parse_command("data/addonscripts/recorder.py foo bar") == ("data/addonscripts/recorder.py", ["foo", "bar"]) + assert script.parse_command("data/addonscripts/recorder.py 'foo bar'") == ("data/addonscripts/recorder.py", ["foo bar"]) + + @tutils.skip_not_windows + def test_parse_windows(self): + with tutils.chdir(tutils.test_data.dirname): + assert script.parse_command( + "data\\addonscripts\\recorder.py" + ) == ("data\\addonscripts\\recorder.py", []) + assert script.parse_command( + "data\\addonscripts\\recorder.py 'foo \\ bar'" + ) == ("data\\addonscripts\\recorder.py", ['foo \\ bar']) + + +def test_load_script(): + ns = script.load_script( + tutils.test_data.path( + "data/addonscripts/recorder.py" + ), [] + ) + assert ns["configure"] + + +class TestScript(mastertest.MasterTest): + def test_simple(self): + s = state.State() + m = master.FlowMaster(options.Options(), None, s) + sc = script.Script( + tutils.test_data.path( + "data/addonscripts/recorder.py" + ) + ) + m.addons.add(sc) + assert sc.ns["call_log"] == [ + ("solo", "start", (), {}), + ("solo", "configure", (options.Options(),), {}) + ] + + sc.ns["call_log"] = [] + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + + recf = sc.ns["call_log"][0] + assert recf[1] == "request" + + def test_reload(self): + s = state.State() + m = mastertest.RecordingMaster(options.Options(), None, s) + with tutils.tmpdir(): + with open("foo.py", "w"): + pass + sc = script.Script("foo.py") + m.addons.add(sc) + + for _ in range(100): + with open("foo.py", "a") as f: + f.write(".") + m.addons.invoke_with_context(sc, "tick") + time.sleep(0.1) + if m.event_log: + return + raise AssertionError("Change event not detected.") + + def test_exception(self): + s = state.State() + m = mastertest.RecordingMaster(options.Options(), None, s) + sc = script.Script( + tutils.test_data.path("data/addonscripts/error.py") + ) + m.addons.add(sc) + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + assert m.event_log[0][0] == "error" + + def test_duplicate_flow(self): + s = state.State() + fm = master.FlowMaster(None, None, s) + fm.addons.add( + script.Script( + tutils.test_data.path("data/addonscripts/duplicate_flow.py") + ) + ) + f = tutils.tflow() + fm.request(f) + assert fm.state.flow_count() == 2 + assert not fm.state.view[0].request.is_replay + assert fm.state.view[1].request.is_replay + + +class TestScriptLoader(mastertest.MasterTest): + def test_simple(self): + s = state.State() + o = options.Options(scripts=[]) + m = master.FlowMaster(o, None, s) + sc = script.ScriptLoader() + m.addons.add(sc) + assert len(m.addons) == 1 + o.update( + scripts = [ + tutils.test_data.path("data/addonscripts/recorder.py") + ] + ) + assert len(m.addons) == 2 + o.update(scripts = []) + assert len(m.addons) == 1 + + def test_dupes(self): + s = state.State() + o = options.Options(scripts=["one", "one"]) + m = master.FlowMaster(o, None, s) + sc = script.ScriptLoader() + tutils.raises(exceptions.OptionsError, m.addons.add, sc) + + def test_order(self): + rec = tutils.test_data.path("data/addonscripts/recorder.py") + + s = state.State() + o = options.Options( + scripts = [ + "%s %s" % (rec, "a"), + "%s %s" % (rec, "b"), + "%s %s" % (rec, "c"), + ] + ) + m = mastertest.RecordingMaster(o, None, s) + sc = script.ScriptLoader() + m.addons.add(sc) + + debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"] + assert debug == [ + ('debug', 'a start'), ('debug', 'a configure'), + ('debug', 'b start'), ('debug', 'b configure'), + ('debug', 'c start'), ('debug', 'c configure') + ] + m.event_log[:] = [] + + o.scripts = [ + "%s %s" % (rec, "c"), + "%s %s" % (rec, "a"), + "%s %s" % (rec, "b"), + ] + debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"] + assert debug == [ + ('debug', 'c configure'), + ('debug', 'a configure'), + ('debug', 'b configure'), + ] + m.event_log[:] = [] + + o.scripts = [ + "%s %s" % (rec, "x"), + "%s %s" % (rec, "a"), + ] + debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"] + assert debug == [ + ('debug', 'c done'), + ('debug', 'b done'), + ('debug', 'x start'), + ('debug', 'x configure'), + ('debug', 'a configure'), + ] diff --git a/test/mitmproxy/builtins/test_stickyauth.py b/test/mitmproxy/builtins/test_stickyauth.py new file mode 100644 index 00000000..1e617402 --- /dev/null +++ b/test/mitmproxy/builtins/test_stickyauth.py @@ -0,0 +1,23 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import stickyauth +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy.flow import options + + +class TestStickyAuth(mastertest.MasterTest): + def test_simple(self): + s = state.State() + m = master.FlowMaster(options.Options(stickyauth = ".*"), None, s) + sa = stickyauth.StickyAuth() + m.addons.add(sa) + + f = tutils.tflow(resp=True) + f.request.headers["authorization"] = "foo" + self.invoke(m, "request", f) + + assert "address" in sa.hosts + + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + assert f.request.headers["authorization"] == "foo" diff --git a/test/mitmproxy/builtins/test_stickycookie.py b/test/mitmproxy/builtins/test_stickycookie.py new file mode 100644 index 00000000..b8d703bd --- /dev/null +++ b/test/mitmproxy/builtins/test_stickycookie.py @@ -0,0 +1,131 @@ +from .. import tutils, mastertest +from mitmproxy.builtins import stickycookie +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy.flow import options +from netlib import tutils as ntutils + + +def test_domain_match(): + assert stickycookie.domain_match("www.google.com", ".google.com") + assert stickycookie.domain_match("google.com", ".google.com") + + +class TestStickyCookie(mastertest.MasterTest): + def mk(self): + s = state.State() + m = master.FlowMaster(options.Options(stickycookie = ".*"), None, s) + sc = stickycookie.StickyCookie() + m.addons.add(sc) + return s, m, sc + + def test_config(self): + sc = stickycookie.StickyCookie() + tutils.raises( + "invalid filter", + sc.configure, + options.Options(stickycookie = "~b") + ) + + def test_simple(self): + s, m, sc = self.mk() + m.addons.add(sc) + + f = tutils.tflow(resp=True) + f.response.headers["set-cookie"] = "foo=bar" + self.invoke(m, "request", f) + + f.reply.acked = False + self.invoke(m, "response", f) + + assert sc.jar + assert "cookie" not in f.request.headers + + f = f.copy() + f.reply.acked = False + self.invoke(m, "request", f) + assert f.request.headers["cookie"] == "foo=bar" + + def _response(self, s, m, sc, cookie, host): + f = tutils.tflow(req=ntutils.treq(host=host, port=80), resp=True) + f.response.headers["Set-Cookie"] = cookie + self.invoke(m, "response", f) + return f + + def test_response(self): + s, m, sc = self.mk() + + c = "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " \ + "Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; " + + self._response(s, m, sc, c, "host") + assert not sc.jar.keys() + + self._response(s, m, sc, c, "www.google.com") + assert sc.jar.keys() + + sc.jar.clear() + self._response( + s, m, sc, "SSID=mooo", "www.google.com" + ) + assert list(sc.jar.keys())[0] == ('www.google.com', 80, '/') + + def test_response_multiple(self): + s, m, sc = self.mk() + + # Test setting of multiple cookies + c1 = "somecookie=test; Path=/" + c2 = "othercookie=helloworld; Path=/" + f = self._response(s, m, sc, c1, "www.google.com") + f.response.headers["Set-Cookie"] = c2 + self.invoke(m, "response", f) + googlekey = list(sc.jar.keys())[0] + assert len(sc.jar[googlekey].keys()) == 2 + + def test_response_weird(self): + s, m, sc = self.mk() + + # Test setting of weird cookie keys + f = tutils.tflow(req=ntutils.treq(host="www.google.com", port=80), resp=True) + cs = [ + "foo/bar=hello", + "foo:bar=world", + "foo@bar=fizz", + "foo,bar=buzz", + ] + for c in cs: + f.response.headers["Set-Cookie"] = c + self.invoke(m, "response", f) + googlekey = list(sc.jar.keys())[0] + assert len(sc.jar[googlekey].keys()) == len(cs) + + def test_response_overwrite(self): + s, m, sc = self.mk() + + # Test overwriting of a cookie value + c1 = "somecookie=helloworld; Path=/" + c2 = "somecookie=newvalue; Path=/" + f = self._response(s, m, sc, c1, "www.google.com") + f.response.headers["Set-Cookie"] = c2 + self.invoke(m, "response", f) + googlekey = list(sc.jar.keys())[0] + assert len(sc.jar[googlekey].keys()) == 1 + assert list(sc.jar[googlekey]["somecookie"].items())[0][1] == "newvalue" + + def test_response_delete(self): + s, m, sc = self.mk() + + # Test that a cookie is be deleted + # by setting the expire time in the past + f = self._response(s, m, sc, "duffer=zafar; Path=/", "www.google.com") + f.response.headers["Set-Cookie"] = "duffer=; Expires=Thu, 01-Jan-1970 00:00:00 GMT" + self.invoke(m, "response", f) + assert not sc.jar.keys() + + def test_request(self): + s, m, sc = self.mk() + + f = self._response(s, m, sc, "SSID=mooo", "www.google.com") + assert "cookie" not in f.request.headers + self.invoke(m, "request", f) + assert "cookie" in f.request.headers diff --git a/test/mitmproxy/builtins/test_stream.py b/test/mitmproxy/builtins/test_stream.py new file mode 100644 index 00000000..edaa41d2 --- /dev/null +++ b/test/mitmproxy/builtins/test_stream.py @@ -0,0 +1,46 @@ +from __future__ import absolute_import, print_function, division + +from .. import tutils, mastertest + +import os.path + +from mitmproxy.builtins import stream +from mitmproxy.flow import master, FlowReader +from mitmproxy.flow import state +from mitmproxy.flow import options + + +class TestStream(mastertest.MasterTest): + def test_stream(self): + with tutils.tmpdir() as tdir: + p = os.path.join(tdir, "foo") + + def r(): + r = FlowReader(open(p, "rb")) + return list(r.stream()) + + s = state.State() + m = master.FlowMaster( + options.Options( + outfile = (p, "wb") + ), + None, + s + ) + sa = stream.Stream() + + m.addons.add(sa) + f = tutils.tflow(resp=True) + self.invoke(m, "request", f) + self.invoke(m, "response", f) + m.addons.remove(sa) + + assert r()[0].response + + m.options.outfile = (p, "ab") + + m.addons.add(sa) + f = tutils.tflow() + self.invoke(m, "request", f) + m.addons.remove(sa) + assert not r()[1].response diff --git a/test/mitmproxy/console/test_master.py b/test/mitmproxy/console/test_master.py index 33261c28..b84e4c1c 100644 --- a/test/mitmproxy/console/test_master.py +++ b/test/mitmproxy/console/test_master.py @@ -111,12 +111,14 @@ def test_options(): class TestMaster(mastertest.MasterTest): - def mkmaster(self, filt, **options): - o = console.master.Options(filtstr=filt, **options) + def mkmaster(self, **options): + if "verbosity" not in options: + options["verbosity"] = 0 + o = console.master.Options(**options) return console.master.ConsoleMaster(None, o) def test_basic(self): - m = self.mkmaster(None) + m = self.mkmaster() for i in (1, 2, 3): - self.dummy_cycle(m, 1, "") + self.dummy_cycle(m, 1, b"") assert len(m.state.flows) == i diff --git a/test/mitmproxy/data/scripts/concurrent_decorator.py b/test/mitmproxy/data/addonscripts/concurrent_decorator.py index e017f605..a56c2af1 100644 --- a/test/mitmproxy/data/scripts/concurrent_decorator.py +++ b/test/mitmproxy/data/addonscripts/concurrent_decorator.py @@ -1,7 +1,6 @@ import time from mitmproxy.script import concurrent - @concurrent -def request(context, flow): +def request(flow): time.sleep(0.1) diff --git a/test/mitmproxy/data/scripts/concurrent_decorator_err.py b/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py index 349e5dd6..756869c8 100644 --- a/test/mitmproxy/data/scripts/concurrent_decorator_err.py +++ b/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py @@ -2,5 +2,5 @@ from mitmproxy.script import concurrent @concurrent -def start(context): +def start(): pass diff --git a/test/mitmproxy/data/addonscripts/duplicate_flow.py b/test/mitmproxy/data/addonscripts/duplicate_flow.py new file mode 100644 index 00000000..b466423c --- /dev/null +++ b/test/mitmproxy/data/addonscripts/duplicate_flow.py @@ -0,0 +1,6 @@ +from mitmproxy import ctx + + +def request(flow): + f = ctx.master.duplicate_flow(flow) + ctx.master.replay_request(f, block=True) diff --git a/test/mitmproxy/data/addonscripts/error.py b/test/mitmproxy/data/addonscripts/error.py new file mode 100644 index 00000000..8ece9fce --- /dev/null +++ b/test/mitmproxy/data/addonscripts/error.py @@ -0,0 +1,7 @@ + +def mkerr(): + raise ValueError("Error!") + + +def request(flow): + mkerr() diff --git a/test/mitmproxy/data/addonscripts/recorder.py b/test/mitmproxy/data/addonscripts/recorder.py new file mode 100644 index 00000000..b6ac8d89 --- /dev/null +++ b/test/mitmproxy/data/addonscripts/recorder.py @@ -0,0 +1,25 @@ +from mitmproxy import controller +from mitmproxy import ctx +import sys + +call_log = [] + +if len(sys.argv) > 1: + name = sys.argv[1] +else: + name = "solo" + +# Keep a log of all possible event calls +evts = list(controller.Events) + ["configure"] +for i in evts: + def mkprox(): + evt = i + + def prox(*args, **kwargs): + lg = (name, evt, args, kwargs) + if evt != "log": + ctx.log.info(str(lg)) + call_log.append(lg) + ctx.log.debug("%s %s" % (name, evt)) + return prox + globals()[i] = mkprox() diff --git a/test/mitmproxy/data/scripts/stream_modify.py b/test/mitmproxy/data/addonscripts/stream_modify.py index 8221b0dd..bc616342 100644 --- a/test/mitmproxy/data/scripts/stream_modify.py +++ b/test/mitmproxy/data/addonscripts/stream_modify.py @@ -1,7 +1,8 @@ + def modify(chunks): for chunk in chunks: yield chunk.replace(b"foo", b"bar") -def responseheaders(context, flow): +def responseheaders(flow): flow.response.stream = modify diff --git a/test/mitmproxy/data/scripts/tcp_stream_modify.py b/test/mitmproxy/data/addonscripts/tcp_stream_modify.py index 0965beba..af4ccf7e 100644 --- a/test/mitmproxy/data/scripts/tcp_stream_modify.py +++ b/test/mitmproxy/data/addonscripts/tcp_stream_modify.py @@ -1,4 +1,5 @@ -def tcp_message(ctx, flow): + +def tcp_message(flow): message = flow.messages[-1] if not message.from_client: message.content = message.content.replace(b"foo", b"bar") diff --git a/test/mitmproxy/data/dumpfile-010 b/test/mitmproxy/data/dumpfile-010 Binary files differnew file mode 100644 index 00000000..435795bf --- /dev/null +++ b/test/mitmproxy/data/dumpfile-010 diff --git a/test/mitmproxy/data/dumpfile-011 b/test/mitmproxy/data/dumpfile-011 Binary files differnew file mode 100644 index 00000000..2534ad89 --- /dev/null +++ b/test/mitmproxy/data/dumpfile-011 diff --git a/test/mitmproxy/data/dumpfile-012 b/test/mitmproxy/data/dumpfile-012 deleted file mode 100644 index 49c2350d..00000000 --- a/test/mitmproxy/data/dumpfile-012 +++ /dev/null @@ -1,35 +0,0 @@ -4092:8:response,491:11:httpversion,8:1:1#1:1#]13:timestamp_end,14:1449080668.874^3:msg,12:Not Modified,15:timestamp_start,14:1449080668.863^7:headers,330:35:13:Cache-Control,14:max-age=604800,]40:4:Date,29:Wed, 02 Dec 2015 18:24:32 GMT,]32:4:Etag,21:"359670651+gzip+gzip",]43:7:Expires,29:Wed, 09 Dec 2015 18:24:32 GMT,]50:13:Last-Modified,29:Fri, 09 Aug 2013 23:54:35 GMT,]27:6:Server,14:ECS (lga/1312),]26:4:Vary,15:Accept-Encoding,]16:7:X-Cache,3:HIT,]25:17:x-ec-custom-error,1:1,]]7:content,0:,4:code,3:304#}4:type,4:http,2:id,36:d209a4fc-8e12-43cb-9250-b0b052d2caf8,5:error,0:~7:version,9:1:0#2:12#]11:client_conn,208:15:ssl_established,4:true!10:clientcert,0:~13:timestamp_end,0:~19:timestamp_ssl_setup,14:1449080668.754^7:address,53:7:address,20:9:127.0.0.1,5:58199#]8:use_ipv6,5:false!}15:timestamp_start,14:1449080666.523^}11:server_conn,2479:15:ssl_established,4:true!14:source_address,57:7:address,24:12:10.67.56.236,5:58201#]8:use_ipv6,5:false!}13:timestamp_end,0:~7:address,54:7:address,21:11:example.com,3:443#]8:use_ipv6,5:false!}15:timestamp_start,14:1449080668.046^3:sni,11:example.com,4:cert,2122:-----BEGIN CERTIFICATE----- -MIIF8jCCBNqgAwIBAgIQDmTF+8I2reFLFyrrQceMsDANBgkqhkiG9w0BAQsFADBw -MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMS8wLQYDVQQDEyZEaWdpQ2VydCBTSEEyIEhpZ2ggQXNz -dXJhbmNlIFNlcnZlciBDQTAeFw0xNTExMDMwMDAwMDBaFw0xODExMjgxMjAwMDBa -MIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxML -TG9zIEFuZ2VsZXMxPDA6BgNVBAoTM0ludGVybmV0IENvcnBvcmF0aW9uIGZvciBB -c3NpZ25lZCBOYW1lcyBhbmQgTnVtYmVyczETMBEGA1UECxMKVGVjaG5vbG9neTEY -MBYGA1UEAxMPd3d3LmV4YW1wbGUub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAs0CWL2FjPiXBl61lRfvvE0KzLJmG9LWAC3bcBjgsH6NiVVo2dt6u -Xfzi5bTm7F3K7srfUBYkLO78mraM9qizrHoIeyofrV/n+pZZJauQsPjCPxMEJnRo -D8Z4KpWKX0LyDu1SputoI4nlQ/htEhtiQnuoBfNZxF7WxcxGwEsZuS1KcXIkHl5V -RJOreKFHTaXcB1qcZ/QRaBIv0yhxvK1yBTwWddT4cli6GfHcCe3xGMaSL328Fgs3 -jYrvG29PueB6VJi/tbbPu6qTfwp/H1brqdjh29U52Bhb0fJkM9DWxCP/Cattcc7a -z8EXnCO+LK8vkhw/kAiJWPKx4RBvgy73nwIDAQABo4ICUDCCAkwwHwYDVR0jBBgw -FoAUUWj/kK8CB3U8zNllZGKiErhZcjswHQYDVR0OBBYEFKZPYB4fLdHn8SOgKpUW -5Oia6m5IMIGBBgNVHREEejB4gg93d3cuZXhhbXBsZS5vcmeCC2V4YW1wbGUuY29t -ggtleGFtcGxlLmVkdYILZXhhbXBsZS5uZXSCC2V4YW1wbGUub3Jngg93d3cuZXhh -bXBsZS5jb22CD3d3dy5leGFtcGxlLmVkdYIPd3d3LmV4YW1wbGUubmV0MA4GA1Ud -DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0f -BG4wbDA0oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItaGEtc2Vy -dmVyLWc0LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTIt -aGEtc2VydmVyLWc0LmNybDBMBgNVHSAERTBDMDcGCWCGSAGG/WwBATAqMCgGCCsG -AQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAgGBmeBDAECAjCB -gwYIKwYBBQUHAQEEdzB1MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2Vy -dC5jb20wTQYIKwYBBQUHMAKGQWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9E -aWdpQ2VydFNIQTJIaWdoQXNzdXJhbmNlU2VydmVyQ0EuY3J0MAwGA1UdEwEB/wQC -MAAwDQYJKoZIhvcNAQELBQADggEBAISomhGn2L0LJn5SJHuyVZ3qMIlRCIdvqe0Q -6ls+C8ctRwRO3UU3x8q8OH+2ahxlQmpzdC5al4XQzJLiLjiJ2Q1p+hub8MFiMmVP -PZjb2tZm2ipWVuMRM+zgpRVM6nVJ9F3vFfUSHOb4/JsEIUvPY+d8/Krc+kPQwLvy -ieqRbcuFjmqfyPmUv1U9QoI4TQikpw7TZU0zYZANP4C/gj4Ry48/znmUaRvy2kvI -l7gRQ21qJTK5suoiYoYNo3J9T+pXPGU7Lydz/HwW+w0DpArtAaukI8aNX4ohFUKS -wDSiIIWIWJiJGbEeIO0TIFwEVWTOnbNl/faPXpk5IRXicapqiII= ------END CERTIFICATE----- -,19:timestamp_ssl_setup,14:1449080668.358^5:state,0:]19:timestamp_tcp_setup,14:1449080668.177^}11:intercepted,5:false!7:request,727:9:is_replay,5:false!4:port,3:443#6:scheme,5:https,6:method,3:GET,4:path,1:/,8:form_out,8:relative,11:httpversion,8:1:1#1:1#]4:host,11:example.com,7:headers,460:22:4:Host,11:example.com,]91:10:User-Agent,73:Mozilla/5.0 (Windows NT 10.0; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0,]76:6:Accept,63:text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8,]46:15:Accept-Language,23:de,en-US;q=0.7,en;q=0.3,]36:15:Accept-Encoding,13:gzip, deflate,]28:10:Connection,10:keep-alive,]54:17:If-Modified-Since,29:Fri, 09 Aug 2013 23:54:35 GMT,]42:13:If-None-Match,21:"359670651+gzip+gzip",]29:13:Cache-Control,9:max-age=0,]]7:content,0:,7:form_in,8:relative,15:timestamp_start,14:1449080668.754^13:timestamp_end,14:1449080668.757^}}
\ No newline at end of file diff --git a/test/mitmproxy/data/dumpfile-013 b/test/mitmproxy/data/dumpfile-013 deleted file mode 100644 index ede06f23..00000000 --- a/test/mitmproxy/data/dumpfile-013 +++ /dev/null @@ -1,35 +0,0 @@ -4092:8:response,491:11:httpversion,8:1:1#1:1#]13:timestamp_end,14:1449080668.874^3:msg,12:Not Modified,15:timestamp_start,14:1449080668.863^7:headers,330:35:13:Cache-Control,14:max-age=604800,]40:4:Date,29:Wed, 02 Dec 2015 18:24:32 GMT,]32:4:Etag,21:"359670651+gzip+gzip",]43:7:Expires,29:Wed, 09 Dec 2015 18:24:32 GMT,]50:13:Last-Modified,29:Fri, 09 Aug 2013 23:54:35 GMT,]27:6:Server,14:ECS (lga/1312),]26:4:Vary,15:Accept-Encoding,]16:7:X-Cache,3:HIT,]25:17:x-ec-custom-error,1:1,]]7:content,0:,4:code,3:304#}4:type,4:http,2:id,36:d209a4fc-8e12-43cb-9250-b0b052d2caf8,5:error,0:~7:version,9:1:0#2:13#]11:client_conn,208:15:ssl_established,4:true!10:clientcert,0:~13:timestamp_end,0:~19:timestamp_ssl_setup,14:1449080668.754^7:address,53:7:address,20:9:127.0.0.1,5:58199#]8:use_ipv6,5:false!}15:timestamp_start,14:1449080666.523^}11:server_conn,2479:15:ssl_established,4:true!14:source_address,57:7:address,24:12:10.67.56.236,5:58201#]8:use_ipv6,5:false!}13:timestamp_end,0:~7:address,54:7:address,21:11:example.com,3:443#]8:use_ipv6,5:false!}15:timestamp_start,14:1449080668.046^3:sni,11:example.com,4:cert,2122:-----BEGIN CERTIFICATE----- -MIIF8jCCBNqgAwIBAgIQDmTF+8I2reFLFyrrQceMsDANBgkqhkiG9w0BAQsFADBw -MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 -d3cuZGlnaWNlcnQuY29tMS8wLQYDVQQDEyZEaWdpQ2VydCBTSEEyIEhpZ2ggQXNz -dXJhbmNlIFNlcnZlciBDQTAeFw0xNTExMDMwMDAwMDBaFw0xODExMjgxMjAwMDBa -MIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxML -TG9zIEFuZ2VsZXMxPDA6BgNVBAoTM0ludGVybmV0IENvcnBvcmF0aW9uIGZvciBB -c3NpZ25lZCBOYW1lcyBhbmQgTnVtYmVyczETMBEGA1UECxMKVGVjaG5vbG9neTEY -MBYGA1UEAxMPd3d3LmV4YW1wbGUub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A -MIIBCgKCAQEAs0CWL2FjPiXBl61lRfvvE0KzLJmG9LWAC3bcBjgsH6NiVVo2dt6u -Xfzi5bTm7F3K7srfUBYkLO78mraM9qizrHoIeyofrV/n+pZZJauQsPjCPxMEJnRo -D8Z4KpWKX0LyDu1SputoI4nlQ/htEhtiQnuoBfNZxF7WxcxGwEsZuS1KcXIkHl5V -RJOreKFHTaXcB1qcZ/QRaBIv0yhxvK1yBTwWddT4cli6GfHcCe3xGMaSL328Fgs3 -jYrvG29PueB6VJi/tbbPu6qTfwp/H1brqdjh29U52Bhb0fJkM9DWxCP/Cattcc7a -z8EXnCO+LK8vkhw/kAiJWPKx4RBvgy73nwIDAQABo4ICUDCCAkwwHwYDVR0jBBgw -FoAUUWj/kK8CB3U8zNllZGKiErhZcjswHQYDVR0OBBYEFKZPYB4fLdHn8SOgKpUW -5Oia6m5IMIGBBgNVHREEejB4gg93d3cuZXhhbXBsZS5vcmeCC2V4YW1wbGUuY29t -ggtleGFtcGxlLmVkdYILZXhhbXBsZS5uZXSCC2V4YW1wbGUub3Jngg93d3cuZXhh -bXBsZS5jb22CD3d3dy5leGFtcGxlLmVkdYIPd3d3LmV4YW1wbGUubmV0MA4GA1Ud -DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0f -BG4wbDA0oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItaGEtc2Vy -dmVyLWc0LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTIt -aGEtc2VydmVyLWc0LmNybDBMBgNVHSAERTBDMDcGCWCGSAGG/WwBATAqMCgGCCsG -AQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAgGBmeBDAECAjCB -gwYIKwYBBQUHAQEEdzB1MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2Vy -dC5jb20wTQYIKwYBBQUHMAKGQWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9E -aWdpQ2VydFNIQTJIaWdoQXNzdXJhbmNlU2VydmVyQ0EuY3J0MAwGA1UdEwEB/wQC -MAAwDQYJKoZIhvcNAQELBQADggEBAISomhGn2L0LJn5SJHuyVZ3qMIlRCIdvqe0Q -6ls+C8ctRwRO3UU3x8q8OH+2ahxlQmpzdC5al4XQzJLiLjiJ2Q1p+hub8MFiMmVP -PZjb2tZm2ipWVuMRM+zgpRVM6nVJ9F3vFfUSHOb4/JsEIUvPY+d8/Krc+kPQwLvy -ieqRbcuFjmqfyPmUv1U9QoI4TQikpw7TZU0zYZANP4C/gj4Ry48/znmUaRvy2kvI -l7gRQ21qJTK5suoiYoYNo3J9T+pXPGU7Lydz/HwW+w0DpArtAaukI8aNX4ohFUKS -wDSiIIWIWJiJGbEeIO0TIFwEVWTOnbNl/faPXpk5IRXicapqiII= ------END CERTIFICATE----- -,19:timestamp_ssl_setup,14:1449080668.358^5:state,0:]19:timestamp_tcp_setup,14:1449080668.177^}11:intercepted,5:false!7:request,727:9:is_replay,5:false!4:port,3:443#6:scheme,5:https,6:method,3:GET,4:path,1:/,8:form_out,8:relative,11:httpversion,8:1:1#1:1#]4:host,11:example.com,7:headers,460:22:4:Host,11:example.com,]91:10:User-Agent,73:Mozilla/5.0 (Windows NT 10.0; WOW64; rv:41.0) Gecko/20100101 Firefox/41.0,]76:6:Accept,63:text/html,application/xhtml+xml,application/xml;q=0.9,*/*;q=0.8,]46:15:Accept-Language,23:de,en-US;q=0.7,en;q=0.3,]36:15:Accept-Encoding,13:gzip, deflate,]28:10:Connection,10:keep-alive,]54:17:If-Modified-Since,29:Fri, 09 Aug 2013 23:54:35 GMT,]42:13:If-None-Match,21:"359670651+gzip+gzip",]29:13:Cache-Control,9:max-age=0,]]7:content,0:,7:form_in,8:relative,15:timestamp_start,14:1449080668.754^13:timestamp_end,14:1449080668.757^}}
\ No newline at end of file diff --git a/test/mitmproxy/data/scripts/a.py b/test/mitmproxy/data/scripts/a.py deleted file mode 100644 index 33dbaa64..00000000 --- a/test/mitmproxy/data/scripts/a.py +++ /dev/null @@ -1,20 +0,0 @@ -import sys - -from a_helper import parser - -var = 0 - - -def start(ctx): - global var - var = parser.parse_args(sys.argv[1:]).var - - -def here(ctx): - global var - var += 1 - return var - - -def errargs(): - pass diff --git a/test/mitmproxy/data/scripts/a_helper.py b/test/mitmproxy/data/scripts/a_helper.py deleted file mode 100644 index e1f1c649..00000000 --- a/test/mitmproxy/data/scripts/a_helper.py +++ /dev/null @@ -1,4 +0,0 @@ -import argparse - -parser = argparse.ArgumentParser() -parser.add_argument('--var', type=int) diff --git a/test/mitmproxy/data/scripts/all.py b/test/mitmproxy/data/scripts/all.py index dad2aade..bf8e93ec 100644 --- a/test/mitmproxy/data/scripts/all.py +++ b/test/mitmproxy/data/scripts/all.py @@ -1,36 +1,37 @@ +import mitmproxy log = [] -def clientconnect(ctx, cc): - ctx.log("XCLIENTCONNECT") +def clientconnect(cc): + mitmproxy.ctx.log("XCLIENTCONNECT") log.append("clientconnect") -def serverconnect(ctx, cc): - ctx.log("XSERVERCONNECT") +def serverconnect(cc): + mitmproxy.ctx.log("XSERVERCONNECT") log.append("serverconnect") -def request(ctx, f): - ctx.log("XREQUEST") +def request(f): + mitmproxy.ctx.log("XREQUEST") log.append("request") -def response(ctx, f): - ctx.log("XRESPONSE") +def response(f): + mitmproxy.ctx.log("XRESPONSE") log.append("response") -def responseheaders(ctx, f): - ctx.log("XRESPONSEHEADERS") +def responseheaders(f): + mitmproxy.ctx.log("XRESPONSEHEADERS") log.append("responseheaders") -def clientdisconnect(ctx, cc): - ctx.log("XCLIENTDISCONNECT") +def clientdisconnect(cc): + mitmproxy.ctx.log("XCLIENTDISCONNECT") log.append("clientdisconnect") -def error(ctx, cc): - ctx.log("XERROR") +def error(cc): + mitmproxy.ctx.log("XERROR") log.append("error") diff --git a/test/mitmproxy/data/scripts/duplicate_flow.py b/test/mitmproxy/data/scripts/duplicate_flow.py deleted file mode 100644 index e13af786..00000000 --- a/test/mitmproxy/data/scripts/duplicate_flow.py +++ /dev/null @@ -1,4 +0,0 @@ - -def request(ctx, f): - f = ctx.duplicate_flow(f) - ctx.replay_request(f) diff --git a/test/mitmproxy/data/scripts/loaderr.py b/test/mitmproxy/data/scripts/loaderr.py deleted file mode 100644 index 8dc4d56d..00000000 --- a/test/mitmproxy/data/scripts/loaderr.py +++ /dev/null @@ -1,3 +0,0 @@ - - -a = x diff --git a/test/mitmproxy/data/scripts/reqerr.py b/test/mitmproxy/data/scripts/reqerr.py deleted file mode 100644 index e7c503a8..00000000 --- a/test/mitmproxy/data/scripts/reqerr.py +++ /dev/null @@ -1,2 +0,0 @@ -def request(ctx, r): - raise ValueError diff --git a/test/mitmproxy/data/scripts/starterr.py b/test/mitmproxy/data/scripts/starterr.py deleted file mode 100644 index 82d773bd..00000000 --- a/test/mitmproxy/data/scripts/starterr.py +++ /dev/null @@ -1,3 +0,0 @@ - -def start(ctx): - raise ValueError() diff --git a/test/mitmproxy/data/scripts/syntaxerr.py b/test/mitmproxy/data/scripts/syntaxerr.py deleted file mode 100644 index 219d6b84..00000000 --- a/test/mitmproxy/data/scripts/syntaxerr.py +++ /dev/null @@ -1,3 +0,0 @@ - - -a + diff --git a/test/mitmproxy/data/scripts/unloaderr.py b/test/mitmproxy/data/scripts/unloaderr.py deleted file mode 100644 index fba02734..00000000 --- a/test/mitmproxy/data/scripts/unloaderr.py +++ /dev/null @@ -1,2 +0,0 @@ -def done(ctx): - raise RuntimeError() diff --git a/test/mitmproxy/data/test_flow_export/python_post_json.py b/test/mitmproxy/data/test_flow_export/python_post_json.py index 6c1b9740..5ef110f3 100644 --- a/test/mitmproxy/data/test_flow_export/python_post_json.py +++ b/test/mitmproxy/data/test_flow_export/python_post_json.py @@ -8,8 +8,8 @@ headers = { json = { - u'email': u'example@example.com', - u'name': u'example', + 'email': 'example@example.com', + 'name': 'example', } diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index 9e726a32..d1fe8cb4 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -3,24 +3,30 @@ import mock from . import tutils import netlib.tutils -from mitmproxy import flow, proxy, models +from mitmproxy.flow import master +from mitmproxy import flow, proxy, models, controller class MasterTest: + def invoke(self, master, handler, message): + with master.handlecontext(): + func = getattr(master, handler) + func(message) + message.reply = controller.DummyReply() + def cycle(self, master, content): f = tutils.tflow(req=netlib.tutils.treq(content=content)) l = proxy.Log("connect") l.reply = mock.MagicMock() master.log(l) - master.clientconnect(f.client_conn) - master.serverconnect(f.server_conn) - master.request(f) + self.invoke(master, "clientconnect", f.client_conn) + self.invoke(master, "clientconnect", f.client_conn) + self.invoke(master, "serverconnect", f.server_conn) + self.invoke(master, "request", f) if not f.error: f.response = models.HTTPResponse.wrap(netlib.tutils.tresp(content=content)) - f.reply.acked = False - f = master.response(f) - f.client_conn.reply.acked = False - master.clientdisconnect(f.client_conn) + self.invoke(master, "response", f) + self.invoke(master, "clientdisconnect", f) return f def dummy_cycle(self, master, n, content): @@ -34,3 +40,12 @@ class MasterTest: t = tutils.tflow(resp=True) fw.add(t) f.close() + + +class RecordingMaster(master.FlowMaster): + def __init__(self, *args, **kwargs): + master.FlowMaster.__init__(self, *args, **kwargs) + self.event_log = [] + + def add_log(self, e, level): + self.event_log.append((level, e)) diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index 62541f3f..080746e8 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -1,28 +1,46 @@ -from mitmproxy.script import Script -from test.mitmproxy import tutils +from test.mitmproxy import tutils, mastertest from mitmproxy import controller +from mitmproxy.builtins import script +from mitmproxy import options +from mitmproxy.flow import master +from mitmproxy.flow import state import time class Thing: def __init__(self): self.reply = controller.DummyReply() + self.live = True -@tutils.skip_appveyor -def test_concurrent(): - with Script(tutils.test_data.path("data/scripts/concurrent_decorator.py"), None) as s: - f1, f2 = Thing(), Thing() - s.run("request", f1) - s.run("request", f2) +class TestConcurrent(mastertest.MasterTest): + @tutils.skip_appveyor + def test_concurrent(self): + s = state.State() + m = master.FlowMaster(options.Options(), None, s) + sc = script.Script( + tutils.test_data.path( + "data/addonscripts/concurrent_decorator.py" + ) + ) + m.addons.add(sc) + f1, f2 = tutils.tflow(), tutils.tflow() + self.invoke(m, "request", f1) + self.invoke(m, "request", f2) start = time.time() while time.time() - start < 5: if f1.reply.acked and f2.reply.acked: return raise ValueError("Script never acked") - -def test_concurrent_err(): - s = Script(tutils.test_data.path("data/scripts/concurrent_decorator_err.py"), None) - with tutils.raises("Concurrent decorator not supported for 'start' method"): - s.load() + def test_concurrent_err(self): + s = state.State() + m = mastertest.RecordingMaster(options.Options(), None, s) + sc = script.Script( + tutils.test_data.path( + "data/addonscripts/concurrent_decorator_err.py" + ) + ) + with m.handlecontext(): + sc.start() + assert "decorator not supported" in m.event_log[0][1] diff --git a/test/mitmproxy/script/test_reloader.py b/test/mitmproxy/script/test_reloader.py deleted file mode 100644 index 0345f6ed..00000000 --- a/test/mitmproxy/script/test_reloader.py +++ /dev/null @@ -1,34 +0,0 @@ -import mock -from mitmproxy.script.reloader import watch, unwatch -from test.mitmproxy import tutils -from threading import Event - - -def test_simple(): - with tutils.tmpdir(): - with open("foo.py", "w"): - pass - - script = mock.Mock() - script.filename = "foo.py" - - e = Event() - - def _onchange(): - e.set() - - watch(script, _onchange) - with tutils.raises("already observed"): - watch(script, _onchange) - - # Some reloaders don't register a change directly after watching, because they first need to initialize. - # To test if watching works at all, we do repeated writes every 100ms. - for _ in range(100): - with open("foo.py", "a") as f: - f.write(".") - if e.wait(0.1): - break - else: - raise AssertionError("No change detected.") - - unwatch(script) diff --git a/test/mitmproxy/script/test_script.py b/test/mitmproxy/script/test_script.py deleted file mode 100644 index fe98fab5..00000000 --- a/test/mitmproxy/script/test_script.py +++ /dev/null @@ -1,83 +0,0 @@ -from mitmproxy.script import Script -from mitmproxy.exceptions import ScriptException -from test.mitmproxy import tutils - - -class TestParseCommand: - def test_empty_command(self): - with tutils.raises(ScriptException): - Script.parse_command("") - - with tutils.raises(ScriptException): - Script.parse_command(" ") - - def test_no_script_file(self): - with tutils.raises("not found"): - Script.parse_command("notfound") - - with tutils.tmpdir() as dir: - with tutils.raises("not a file"): - Script.parse_command(dir) - - def test_parse_args(self): - with tutils.chdir(tutils.test_data.dirname): - assert Script.parse_command("data/scripts/a.py") == ["data/scripts/a.py"] - assert Script.parse_command("data/scripts/a.py foo bar") == ["data/scripts/a.py", "foo", "bar"] - assert Script.parse_command("data/scripts/a.py 'foo bar'") == ["data/scripts/a.py", "foo bar"] - - @tutils.skip_not_windows - def test_parse_windows(self): - with tutils.chdir(tutils.test_data.dirname): - assert Script.parse_command("data\\scripts\\a.py") == ["data\\scripts\\a.py"] - assert Script.parse_command("data\\scripts\\a.py 'foo \\ bar'") == ["data\\scripts\\a.py", 'foo \\ bar'] - - -def test_simple(): - with tutils.chdir(tutils.test_data.path("data/scripts")): - s = Script("a.py --var 42", None) - assert s.filename == "a.py" - assert s.ns is None - - s.load() - assert s.ns["var"] == 42 - - s.run("here") - assert s.ns["var"] == 43 - - s.unload() - assert s.ns is None - - with tutils.raises(ScriptException): - s.run("here") - - with Script("a.py --var 42", None) as s: - s.run("here") - - -def test_script_exception(): - with tutils.chdir(tutils.test_data.path("data/scripts")): - s = Script("syntaxerr.py", None) - with tutils.raises(ScriptException): - s.load() - - s = Script("starterr.py", None) - with tutils.raises(ScriptException): - s.load() - - s = Script("a.py", None) - s.load() - with tutils.raises(ScriptException): - s.load() - - s = Script("a.py", None) - with tutils.raises(ScriptException): - s.run("here") - - with tutils.raises(ScriptException): - with Script("reqerr.py", None) as s: - s.run("request", None) - - s = Script("unloaderr.py", None) - s.load() - with tutils.raises(ScriptException): - s.unload() diff --git a/test/mitmproxy/test_addons.py b/test/mitmproxy/test_addons.py new file mode 100644 index 00000000..1861d4ac --- /dev/null +++ b/test/mitmproxy/test_addons.py @@ -0,0 +1,20 @@ +from __future__ import absolute_import, print_function, division +from mitmproxy import addons +from mitmproxy import controller +from mitmproxy import options + + +class TAddon: + def __init__(self, name): + self.name = name + + def __repr__(self): + return "Addon(%s)" % self.name + + +def test_simple(): + m = controller.Master(options.Options()) + a = addons.Addons(m) + a.add(TAddon("one")) + assert a.has_addon("one") + assert not a.has_addon("two") diff --git a/test/mitmproxy/test_contentview.py b/test/mitmproxy/test_contentview.py index 7037745d..2db9ab40 100644 --- a/test/mitmproxy/test_contentview.py +++ b/test/mitmproxy/test_contentview.py @@ -200,6 +200,13 @@ Larry ) assert "Raw" in r[0] + r = cv.get_content_view( + cv.get("Auto"), + b"[1, 2, 3]", + headers=Headers(content_type="application/vnd.api+json") + ) + assert r[0] == "JSON" + tutils.raises( ContentViewException, cv.get_content_view, diff --git a/test/mitmproxy/test_contrib_tnetstring.py b/test/mitmproxy/test_contrib_tnetstring.py index 17654ad9..05c4a7c9 100644 --- a/test/mitmproxy/test_contrib_tnetstring.py +++ b/test/mitmproxy/test_contrib_tnetstring.py @@ -15,7 +15,9 @@ FORMAT_EXAMPLES = { {b'hello': [12345678901, b'this', True, None, b'\x00\x00\x00\x00']}, b'5:12345#': 12345, b'12:this is cool,': b'this is cool', + b'19:this is unicode \xe2\x98\x85;': u'this is unicode \u2605', b'0:,': b'', + b'0:;': u'', b'0:~': None, b'4:true!': True, b'5:false!': False, @@ -43,7 +45,7 @@ def get_random_object(random=random, depth=0): d = {} for _ in range(n): n = random.randint(0, 100) - k = bytes([random.randint(32, 126) for _ in range(n)]) + k = str([random.randint(32, 126) for _ in range(n)]) d[k] = get_random_object(random, depth + 1) return d else: @@ -78,12 +80,6 @@ class Test_Format(unittest.TestCase): self.assertEqual(v, tnetstring.loads(tnetstring.dumps(v))) self.assertEqual((v, b""), tnetstring.pop(tnetstring.dumps(v))) - def test_unicode_handling(self): - with self.assertRaises(ValueError): - tnetstring.dumps(u"hello") - self.assertEqual(tnetstring.dumps(u"hello".encode()), b"5:hello,") - self.assertEqual(type(tnetstring.loads(b"5:hello,")), bytes) - def test_roundtrip_format_unicode(self): for _ in range(500): v = get_random_object() diff --git a/test/mitmproxy/test_controller.py b/test/mitmproxy/test_controller.py index 5a68e15b..6d4b8fe6 100644 --- a/test/mitmproxy/test_controller.py +++ b/test/mitmproxy/test_controller.py @@ -25,7 +25,7 @@ class TestMaster(object): # Speed up test super(DummyMaster, self).tick(0) - m = DummyMaster() + m = DummyMaster(None) assert not m.should_exit.is_set() msg = TMsg() msg.reply = controller.DummyReply() @@ -34,7 +34,7 @@ class TestMaster(object): assert m.should_exit.is_set() def test_server_simple(self): - m = controller.Master() + m = controller.Master(None) s = DummyServer(None) m.add_server(s) m.start() diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index 234490f8..c94630a9 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -4,31 +4,33 @@ from mitmproxy.exceptions import ContentViewException import netlib.tutils -from mitmproxy import dump, flow, models +from mitmproxy import dump, flow, models, exceptions from . import tutils, mastertest import mock def test_strfuncs(): - o = dump.Options() + o = dump.Options( + tfile = StringIO(), + flow_detail = 0, + ) m = dump.DumpMaster(None, o) - m.outfile = StringIO() m.o.flow_detail = 0 m.echo_flow(tutils.tflow()) - assert not m.outfile.getvalue() + assert not o.tfile.getvalue() m.o.flow_detail = 4 m.echo_flow(tutils.tflow()) - assert m.outfile.getvalue() + assert o.tfile.getvalue() - m.outfile = StringIO() + o.tfile = StringIO() m.echo_flow(tutils.tflow(resp=True)) - assert "<<" in m.outfile.getvalue() + assert "<<" in o.tfile.getvalue() - m.outfile = StringIO() + o.tfile = StringIO() m.echo_flow(tutils.tflow(err=True)) - assert "<<" in m.outfile.getvalue() + assert "<<" in o.tfile.getvalue() flow = tutils.tflow() flow.request = netlib.tutils.treq() @@ -40,7 +42,7 @@ def test_strfuncs(): flow.response.status_code = 300 m.echo_flow(flow) - flow = tutils.tflow(resp=netlib.tutils.tresp(content="{")) + flow = tutils.tflow(resp=netlib.tutils.tresp(content=b"{")) flow.response.headers["content-type"] = "application/json" flow.response.status_code = 400 m.echo_flow(flow) @@ -50,78 +52,102 @@ def test_strfuncs(): def test_contentview(get_content_view): get_content_view.side_effect = ContentViewException(""), ("x", iter([])) - o = dump.Options(flow_detail=4, verbosity=3) - m = dump.DumpMaster(None, o, StringIO()) + o = dump.Options( + flow_detail=4, + verbosity=3, + tfile=StringIO(), + ) + m = dump.DumpMaster(None, o) m.echo_flow(tutils.tflow()) - assert "Content viewer failed" in m.outfile.getvalue() + assert "Content viewer failed" in m.options.tfile.getvalue() class TestDumpMaster(mastertest.MasterTest): def dummy_cycle(self, master, n, content): mastertest.MasterTest.dummy_cycle(self, master, n, content) - return master.outfile.getvalue() + return master.options.tfile.getvalue() def mkmaster(self, filt, **options): - cs = StringIO() - o = dump.Options(filtstr=filt, **options) - return dump.DumpMaster(None, o, outfile=cs) + if "verbosity" not in options: + options["verbosity"] = 0 + if "flow_detail" not in options: + options["flow_detail"] = 0 + o = dump.Options( + filtstr=filt, + tfile=StringIO(), + **options + ) + return dump.DumpMaster(None, o) def test_basic(self): for i in (1, 2, 3): - assert "GET" in self.dummy_cycle(self.mkmaster("~s", flow_detail=i), 1, "") assert "GET" in self.dummy_cycle( self.mkmaster("~s", flow_detail=i), 1, - "\x00\x00\x00" + b"" + ) + assert "GET" in self.dummy_cycle( + self.mkmaster("~s", flow_detail=i), + 1, + b"\x00\x00\x00" ) assert "GET" in self.dummy_cycle( self.mkmaster("~s", flow_detail=i), - 1, "ascii" + 1, + b"ascii" ) def test_error(self): - cs = StringIO() - o = dump.Options(flow_detail=1) - m = dump.DumpMaster(None, o, outfile=cs) + o = dump.Options( + tfile=StringIO(), + flow_detail=1 + ) + m = dump.DumpMaster(None, o) f = tutils.tflow(err=True) m.request(f) assert m.error(f) - assert "error" in cs.getvalue() + assert "error" in o.tfile.getvalue() def test_missing_content(self): - cs = StringIO() - o = dump.Options(flow_detail=3) - m = dump.DumpMaster(None, o, outfile=cs) + o = dump.Options( + flow_detail=3, + tfile=StringIO(), + ) + m = dump.DumpMaster(None, o) f = tutils.tflow() f.request.content = None m.request(f) f.response = models.HTTPResponse.wrap(netlib.tutils.tresp()) f.response.content = None m.response(f) - assert "content missing" in cs.getvalue() + assert "content missing" in o.tfile.getvalue() def test_replay(self): - cs = StringIO() - o = dump.Options(server_replay=["nonexistent"], kill=True) - tutils.raises(dump.DumpError, dump.DumpMaster, None, o, outfile=cs) + tutils.raises(dump.DumpError, dump.DumpMaster, None, o) with tutils.tmpdir() as t: p = os.path.join(t, "rep") self.flowfile(p) o = dump.Options(server_replay=[p], kill=True) - m = dump.DumpMaster(None, o, outfile=cs) + o.verbosity = 0 + o.flow_detail = 0 + m = dump.DumpMaster(None, o) - self.cycle(m, "content") - self.cycle(m, "content") + self.cycle(m, b"content") + self.cycle(m, b"content") o = dump.Options(server_replay=[p], kill=False) - m = dump.DumpMaster(None, o, outfile=cs) - self.cycle(m, "nonexistent") + o.verbosity = 0 + o.flow_detail = 0 + m = dump.DumpMaster(None, o) + self.cycle(m, b"nonexistent") o = dump.Options(client_replay=[p], kill=False) - m = dump.DumpMaster(None, o, outfile=cs) + o.verbosity = 0 + o.flow_detail = 0 + m = dump.DumpMaster(None, o) def test_read(self): with tutils.tmpdir() as t: @@ -129,7 +155,7 @@ class TestDumpMaster(mastertest.MasterTest): self.flowfile(p) assert "GET" in self.dummy_cycle( self.mkmaster(None, flow_detail=1, rfile=p), - 0, "", + 0, b"", ) tutils.raises( @@ -147,7 +173,7 @@ class TestDumpMaster(mastertest.MasterTest): def test_filter(self): assert "GET" not in self.dummy_cycle( - self.mkmaster("~u foo", verbosity=1), 1, "" + self.mkmaster("~u foo", verbosity=1), 1, b"" ) def test_app(self): @@ -157,24 +183,32 @@ class TestDumpMaster(mastertest.MasterTest): assert len(m.apps.apps) == 1 def test_replacements(self): - cs = StringIO() - o = dump.Options(replacements=[(".*", "content", "foo")]) - m = dump.DumpMaster(None, o, outfile=cs) - f = self.cycle(m, "content") - assert f.request.content == "foo" + o = dump.Options( + replacements=[(".*", "content", "foo")], + tfile = StringIO(), + ) + o.verbosity = 0 + o.flow_detail = 0 + m = dump.DumpMaster(None, o) + f = self.cycle(m, b"content") + assert f.request.content == b"foo" def test_setheader(self): - cs = StringIO() - o = dump.Options(setheaders=[(".*", "one", "two")]) - m = dump.DumpMaster(None, o, outfile=cs) - f = self.cycle(m, "content") + o = dump.Options( + setheaders=[(".*", "one", "two")], + tfile=StringIO() + ) + o.verbosity = 0 + o.flow_detail = 0 + m = dump.DumpMaster(None, o) + f = self.cycle(m, b"content") assert f.request.headers["one"] == "two" def test_write(self): with tutils.tmpdir() as d: p = os.path.join(d, "a") self.dummy_cycle( - self.mkmaster(None, outfile=(p, "wb"), verbosity=0), 1, "" + self.mkmaster(None, outfile=(p, "wb"), verbosity=0), 1, b"" ) assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 1 @@ -183,17 +217,17 @@ class TestDumpMaster(mastertest.MasterTest): p = os.path.join(d, "a.append") self.dummy_cycle( self.mkmaster(None, outfile=(p, "wb"), verbosity=0), - 1, "" + 1, b"" ) self.dummy_cycle( self.mkmaster(None, outfile=(p, "ab"), verbosity=0), - 1, "" + 1, b"" ) assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 2 def test_write_err(self): tutils.raises( - dump.DumpError, + exceptions.OptionsError, self.mkmaster, None, outfile = ("nonexistentdir/foo", "wb") ) @@ -201,9 +235,10 @@ class TestDumpMaster(mastertest.MasterTest): ret = self.dummy_cycle( self.mkmaster( None, - scripts=[tutils.test_data.path("data/scripts/all.py")], verbosity=1 + scripts=[tutils.test_data.path("data/scripts/all.py")], + verbosity=2 ), - 1, "", + 1, b"", ) assert "XCLIENTCONNECT" in ret assert "XSERVERCONNECT" in ret @@ -211,12 +246,12 @@ class TestDumpMaster(mastertest.MasterTest): assert "XRESPONSE" in ret assert "XCLIENTDISCONNECT" in ret tutils.raises( - dump.DumpError, + exceptions.AddonError, self.mkmaster, None, scripts=["nonexistent"] ) tutils.raises( - dump.DumpError, + exceptions.AddonError, self.mkmaster, None, scripts=["starterr.py"] ) @@ -224,11 +259,11 @@ class TestDumpMaster(mastertest.MasterTest): def test_stickycookie(self): self.dummy_cycle( self.mkmaster(None, stickycookie = ".*"), - 1, "" + 1, b"" ) def test_stickyauth(self): self.dummy_cycle( self.mkmaster(None, stickyauth = ".*"), - 1, "" + 1, b"" ) diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index 22d3c425..0ec85f52 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -1,155 +1,126 @@ -import glob import json -import os -from contextlib import contextmanager -from mitmproxy import script -from mitmproxy.proxy import config +import six +import sys +import os.path +from mitmproxy.flow import master +from mitmproxy.flow import state +from mitmproxy import options +from mitmproxy import contentviews +from mitmproxy.builtins import script import netlib.utils from netlib import tutils as netutils from netlib.http import Headers -from . import tservers, tutils +from . import tutils, mastertest -example_dir = netlib.utils.Data(__name__).path("../../examples") +example_dir = netlib.utils.Data(__name__).push("../../examples") -class DummyContext(object): - """Emulate script.ScriptContext() functionality.""" +class ScriptError(Exception): + pass - contentview = None - def log(self, *args, **kwargs): - pass +class RaiseMaster(master.FlowMaster): + def add_log(self, e, level): + if level in ("warn", "error"): + raise ScriptError(e) - def add_contentview(self, view_obj): - self.contentview = view_obj - def remove_contentview(self, view_obj): - self.contentview = None +def tscript(cmd, args=""): + cmd = example_dir.path(cmd) + " " + args + m = RaiseMaster(options.Options(), None, state.State()) + sc = script.Script(cmd) + m.addons.add(sc) + return m, sc -@contextmanager -def example(command): - command = os.path.join(example_dir, command) - ctx = DummyContext() - with script.Script(command, ctx) as s: - yield s +class TestScripts(mastertest.MasterTest): + def test_add_header(self): + m, _ = tscript("add_header.py") + f = tutils.tflow(resp=netutils.tresp()) + self.invoke(m, "response", f) + assert f.response.headers["newheader"] == "foo" - -def test_load_scripts(): - scripts = glob.glob("%s/*.py" % example_dir) - - tmaster = tservers.TestMaster(config.ProxyConfig()) - - for f in scripts: - if "har_extractor" in f: - continue - if "flowwriter" in f: - f += " -" - if "iframe_injector" in f: - f += " foo" # one argument required - if "filt" in f: - f += " ~a" - if "modify_response_body" in f: - f += " foo bar" # two arguments required - - s = script.Script(f, script.ScriptContext(tmaster)) - try: - s.load() - except Exception as v: - if "ImportError" not in str(v): - raise - else: - s.unload() - - -def test_add_header(): - flow = tutils.tflow(resp=netutils.tresp()) - with example("add_header.py") as ex: - ex.run("response", flow) - assert flow.response.headers["newheader"] == "foo" - - -def test_custom_contentviews(): - with example("custom_contentviews.py") as ex: - pig = ex.ctx.contentview + def test_custom_contentviews(self): + m, sc = tscript("custom_contentviews.py") + pig = contentviews.get("pig_latin_HTML") _, fmt = pig(b"<html>test!</html>") assert any(b'esttay!' in val[0][1] for val in fmt) assert not pig(b"gobbledygook") + def test_iframe_injector(self): + with tutils.raises(ScriptError): + tscript("iframe_injector.py") -def test_iframe_injector(): - with tutils.raises(script.ScriptException): - with example("iframe_injector.py") as ex: - pass - - flow = tutils.tflow(resp=netutils.tresp(content="<html>mitmproxy</html>")) - with example("iframe_injector.py http://example.org/evil_iframe") as ex: - ex.run("response", flow) + m, sc = tscript("iframe_injector.py", "http://example.org/evil_iframe") + flow = tutils.tflow(resp=netutils.tresp(content=b"<html>mitmproxy</html>")) + self.invoke(m, "response", flow) content = flow.response.content - assert 'iframe' in content and 'evil_iframe' in content - - -def test_modify_form(): - form_header = Headers(content_type="application/x-www-form-urlencoded") - flow = tutils.tflow(req=netutils.treq(headers=form_header)) - with example("modify_form.py") as ex: - ex.run("request", flow) - assert flow.request.urlencoded_form["mitmproxy"] == "rocks" - - flow.request.headers["content-type"] = "" - ex.run("request", flow) - assert list(flow.request.urlencoded_form.items()) == [("foo", "bar")] - - -def test_modify_querystring(): - flow = tutils.tflow(req=netutils.treq(path=b"/search?q=term")) - with example("modify_querystring.py") as ex: - ex.run("request", flow) - assert flow.request.query["mitmproxy"] == "rocks" - - flow.request.path = "/" - ex.run("request", flow) - assert flow.request.query["mitmproxy"] == "rocks" - - -def test_modify_response_body(): - with tutils.raises(script.ScriptException): - with example("modify_response_body.py"): - assert True - - flow = tutils.tflow(resp=netutils.tresp(content="I <3 mitmproxy")) - with example("modify_response_body.py mitmproxy rocks") as ex: - assert ex.ctx.old == "mitmproxy" and ex.ctx.new == "rocks" - ex.run("response", flow) - assert flow.response.content == "I <3 rocks" - - -def test_redirect_requests(): - flow = tutils.tflow(req=netutils.treq(host=b"example.org")) - with example("redirect_requests.py") as ex: - ex.run("request", flow) - assert flow.request.host == "mitmproxy.org" - - -def test_har_extractor(): - with tutils.raises(script.ScriptException): - with example("har_extractor.py"): - pass - - times = dict( - timestamp_start=746203272, - timestamp_end=746203272, - ) - - flow = tutils.tflow( - req=netutils.treq(**times), - resp=netutils.tresp(**times) - ) - - with example("har_extractor.py -") as ex: - ex.run("response", flow) - - with open(tutils.test_data.path("data/har_extractor.har")) as fp: - test_data = json.load(fp) - assert json.loads(ex.ctx.HARLog.json()) == test_data["test_response"] + assert b'iframe' in content and b'evil_iframe' in content + + def test_modify_form(self): + m, sc = tscript("modify_form.py") + + form_header = Headers(content_type="application/x-www-form-urlencoded") + f = tutils.tflow(req=netutils.treq(headers=form_header)) + self.invoke(m, "request", f) + + assert f.request.urlencoded_form[b"mitmproxy"] == b"rocks" + + f.request.headers["content-type"] = "" + self.invoke(m, "request", f) + assert list(f.request.urlencoded_form.items()) == [(b"foo", b"bar")] + + def test_modify_querystring(self): + m, sc = tscript("modify_querystring.py") + f = tutils.tflow(req=netutils.treq(path="/search?q=term")) + + self.invoke(m, "request", f) + assert f.request.query["mitmproxy"] == "rocks" + + f.request.path = "/" + self.invoke(m, "request", f) + assert f.request.query["mitmproxy"] == "rocks" + + def test_modify_response_body(self): + with tutils.raises(ScriptError): + tscript("modify_response_body.py") + + m, sc = tscript("modify_response_body.py", "mitmproxy rocks") + f = tutils.tflow(resp=netutils.tresp(content=b"I <3 mitmproxy")) + self.invoke(m, "response", f) + assert f.response.content == b"I <3 rocks" + + def test_redirect_requests(self): + m, sc = tscript("redirect_requests.py") + f = tutils.tflow(req=netutils.treq(host="example.org")) + self.invoke(m, "request", f) + assert f.request.host == "mitmproxy.org" + + def test_har_extractor(self): + if sys.version_info >= (3, 0): + with tutils.raises("does not work on Python 3"): + tscript("har_extractor.py") + return + + with tutils.raises(ScriptError): + tscript("har_extractor.py") + + with tutils.tmpdir() as tdir: + times = dict( + timestamp_start=746203272, + timestamp_end=746203272, + ) + + path = os.path.join(tdir, "file") + m, sc = tscript("har_extractor.py", six.moves.shlex_quote(path)) + f = tutils.tflow( + req=netutils.treq(**times), + resp=netutils.tresp(**times) + ) + self.invoke(m, "response", f) + m.addons.remove(sc) + + with open(path, "rb") as f: + test_data = json.load(f) + assert len(test_data["log"]["pages"]) == 1 diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 5753e728..0bdcc038 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -1,13 +1,11 @@ -import os.path -from six.moves import cStringIO as StringIO - import mock +import io import netlib.utils from netlib.http import Headers from mitmproxy import filt, controller, flow from mitmproxy.contrib import tnetstring -from mitmproxy.exceptions import FlowReadException, ScriptException +from mitmproxy.exceptions import FlowReadException from mitmproxy.models import Error from mitmproxy.models import Flow from mitmproxy.models import HTTPFlow @@ -40,94 +38,12 @@ def test_app_registry(): assert ar.get(r) -class TestStickyCookieState: - - def _response(self, cookie, host): - s = flow.StickyCookieState(filt.parse(".*")) - f = tutils.tflow(req=netlib.tutils.treq(host=host, port=80), resp=True) - f.response.headers["Set-Cookie"] = cookie - s.handle_response(f) - return s, f - - def test_domain_match(self): - s = flow.StickyCookieState(filt.parse(".*")) - assert s.domain_match("www.google.com", ".google.com") - assert s.domain_match("google.com", ".google.com") - - def test_response(self): - c = "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " \ - "Expires=Wed, 13-Jan-2021 22:23:01 GMT; Secure; " - - s, f = self._response(c, "host") - assert not s.jar.keys() - - s, f = self._response(c, "www.google.com") - assert s.jar.keys() - - s, f = self._response("SSID=mooo", "www.google.com") - assert s.jar.keys()[0] == ('www.google.com', 80, '/') - - # Test setting of multiple cookies - c1 = "somecookie=test; Path=/" - c2 = "othercookie=helloworld; Path=/" - s, f = self._response(c1, "www.google.com") - f.response.headers["Set-Cookie"] = c2 - s.handle_response(f) - googlekey = s.jar.keys()[0] - assert len(s.jar[googlekey].keys()) == 2 - - # Test setting of weird cookie keys - s = flow.StickyCookieState(filt.parse(".*")) - f = tutils.tflow(req=netlib.tutils.treq(host="www.google.com", port=80), resp=True) - cs = [ - "foo/bar=hello", - "foo:bar=world", - "foo@bar=fizz", - "foo,bar=buzz", - ] - for c in cs: - f.response.headers["Set-Cookie"] = c - s.handle_response(f) - googlekey = s.jar.keys()[0] - assert len(s.jar[googlekey].keys()) == len(cs) - - # Test overwriting of a cookie value - c1 = "somecookie=helloworld; Path=/" - c2 = "somecookie=newvalue; Path=/" - s, f = self._response(c1, "www.google.com") - f.response.headers["Set-Cookie"] = c2 - s.handle_response(f) - googlekey = s.jar.keys()[0] - assert len(s.jar[googlekey].keys()) == 1 - assert s.jar[googlekey]["somecookie"].items()[0][1] == "newvalue" - - def test_request(self): - s, f = self._response("SSID=mooo", "www.google.com") - assert "cookie" not in f.request.headers - s.handle_request(f) - assert "cookie" in f.request.headers - - -class TestStickyAuthState: - - def test_response(self): - s = flow.StickyAuthState(filt.parse(".*")) - f = tutils.tflow(resp=True) - f.request.headers["authorization"] = "foo" - s.handle_request(f) - assert "address" in s.hosts - - f = tutils.tflow(resp=True) - s.handle_request(f) - assert f.request.headers["authorization"] == "foo" - - class TestClientPlaybackState: def test_tick(self): first = tutils.tflow() s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.start_client_playback([first, tutils.tflow()], True) c = fm.client_playback c.testing = True @@ -264,26 +180,26 @@ class TestServerPlaybackState: "param1", "param2"], False) r = tutils.tflow(resp=True) r.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - r.request.content = "paramx=x¶m1=1" + r.request.content = b"paramx=x¶m1=1" r2 = tutils.tflow(resp=True) r2.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - r2.request.content = "paramx=x¶m1=1" + r2.request.content = b"paramx=x¶m1=1" # same parameters assert s._hash(r) == s._hash(r2) # ignored parameters != - r2.request.content = "paramx=x¶m1=2" + r2.request.content = b"paramx=x¶m1=2" assert s._hash(r) == s._hash(r2) # missing parameter - r2.request.content = "paramx=x" + r2.request.content = b"paramx=x" assert s._hash(r) == s._hash(r2) # ignorable parameter added - r2.request.content = "paramx=x¶m1=2" + r2.request.content = b"paramx=x¶m1=2" assert s._hash(r) == s._hash(r2) # not ignorable parameter changed - r2.request.content = "paramx=y¶m1=1" + r2.request.content = b"paramx=y¶m1=1" assert not s._hash(r) == s._hash(r2) # not ignorable parameter missing - r2.request.content = "param1=1" + r2.request.content = b"param1=1" assert not s._hash(r) == s._hash(r2) def test_ignore_payload_params_other_content_type(self): @@ -292,14 +208,14 @@ class TestServerPlaybackState: "param1", "param2"], False) r = tutils.tflow(resp=True) r.request.headers["Content-Type"] = "application/json" - r.request.content = '{"param1":"1"}' + r.request.content = b'{"param1":"1"}' r2 = tutils.tflow(resp=True) r2.request.headers["Content-Type"] = "application/json" - r2.request.content = '{"param1":"1"}' + r2.request.content = b'{"param1":"1"}' # same content assert s._hash(r) == s._hash(r2) # distint content (note only x-www-form-urlencoded payload is analysed) - r2.request.content = '{"param1":"2"}' + r2.request.content = b'{"param1":"2"}' assert not s._hash(r) == s._hash(r2) def test_ignore_payload_wins_over_params(self): @@ -309,10 +225,10 @@ class TestServerPlaybackState: "param1", "param2"], False) r = tutils.tflow(resp=True) r.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - r.request.content = "paramx=y" + r.request.content = b"paramx=y" r2 = tutils.tflow(resp=True) r2.request.headers["Content-Type"] = "application/x-www-form-urlencoded" - r2.request.content = "paramx=x" + r2.request.content = b"paramx=x" # same parameters assert s._hash(r) == s._hash(r2) @@ -329,10 +245,10 @@ class TestServerPlaybackState: r = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True) - r.request.content = "foo" - r2.request.content = "foo" + r.request.content = b"foo" + r2.request.content = b"foo" assert s._hash(r) == s._hash(r2) - r2.request.content = "bar" + r2.request.content = b"bar" assert not s._hash(r) == s._hash(r2) # now ignoring content @@ -347,12 +263,12 @@ class TestServerPlaybackState: False) r = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True) - r.request.content = "foo" - r2.request.content = "foo" + r.request.content = b"foo" + r2.request.content = b"foo" assert s._hash(r) == s._hash(r2) - r2.request.content = "bar" + r2.request.content = b"bar" assert s._hash(r) == s._hash(r2) - r2.request.content = "" + r2.request.content = b"" assert s._hash(r) == s._hash(r2) r2.request.content = None assert s._hash(r) == s._hash(r2) @@ -420,13 +336,13 @@ class TestFlow(object): def test_backup(self): f = tutils.tflow() f.response = HTTPResponse.wrap(netlib.tutils.tresp()) - f.request.content = "foo" + f.request.content = b"foo" assert not f.modified() f.backup() - f.request.content = "bar" + f.request.content = b"bar" assert f.modified() f.revert() - assert f.request.content == "foo" + assert f.request.content == b"foo" def test_backup_idempotence(self): f = tutils.tflow(resp=True) @@ -458,7 +374,7 @@ class TestFlow(object): def test_kill(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow() f.intercept(mock.Mock()) f.kill(fm) @@ -467,7 +383,7 @@ class TestFlow(object): def test_killall(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow() f.intercept(fm) @@ -486,8 +402,8 @@ class TestFlow(object): def test_replace_unicode(self): f = tutils.tflow(resp=True) - f.response.content = "\xc2foo" - f.replace("foo", u"bar") + f.response.content = b"\xc2foo" + f.replace(b"foo", u"bar") def test_replace_no_content(self): f = tutils.tflow() @@ -497,34 +413,34 @@ class TestFlow(object): def test_replace(self): f = tutils.tflow(resp=True) f.request.headers["foo"] = "foo" - f.request.content = "afoob" + f.request.content = b"afoob" f.response.headers["foo"] = "foo" - f.response.content = "afoob" + f.response.content = b"afoob" assert f.replace("foo", "bar") == 6 assert f.request.headers["bar"] == "bar" - assert f.request.content == "abarb" + assert f.request.content == b"abarb" assert f.response.headers["bar"] == "bar" - assert f.response.content == "abarb" + assert f.response.content == b"abarb" def test_replace_encoded(self): f = tutils.tflow(resp=True) - f.request.content = "afoob" + f.request.content = b"afoob" f.request.encode("gzip") - f.response.content = "afoob" + f.response.content = b"afoob" f.response.encode("gzip") f.replace("foo", "bar") - assert f.request.raw_content != "abarb" + assert f.request.raw_content != b"abarb" f.request.decode() - assert f.request.raw_content == "abarb" + assert f.request.raw_content == b"abarb" - assert f.response.raw_content != "abarb" + assert f.response.raw_content != b"abarb" f.response.decode() - assert f.response.raw_content == "abarb" + assert f.response.raw_content == b"abarb" class TestState: @@ -667,7 +583,7 @@ class TestState: class TestSerialize: def _treader(self): - sio = StringIO() + sio = io.BytesIO() w = flow.FlowWriter(sio) for i in range(3): f = tutils.tflow(resp=True) @@ -684,9 +600,9 @@ class TestSerialize: return flow.FlowReader(sio) def test_roundtrip(self): - sio = StringIO() + sio = io.BytesIO() f = tutils.tflow() - f.request.content = "".join(chr(i) for i in range(255)) + f.request.content = bytes(bytearray(range(256))) w = flow.FlowWriter(sio) w.add(f) @@ -702,7 +618,7 @@ class TestSerialize: def test_load_flows(self): r = self._treader() s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.load_flows(r) assert len(s.flows) == 6 @@ -713,12 +629,12 @@ class TestSerialize: mode="reverse", upstream_server=("https", ("use-this-domain", 80)) ) - fm = flow.FlowMaster(DummyServer(conf), s) + fm = flow.FlowMaster(None, DummyServer(conf), s) fm.load_flows(r) assert s.flows[0].request.host == "use-this-domain" def test_filter(self): - sio = StringIO() + sio = io.BytesIO() fl = filt.parse("~c 200") w = flow.FilteredFlowWriter(sio, fl) @@ -735,8 +651,8 @@ class TestSerialize: assert len(list(r.stream())) def test_error(self): - sio = StringIO() - sio.write("bogus") + sio = io.BytesIO() + sio.write(b"bogus") sio.seek(0) r = flow.FlowReader(sio) tutils.raises(FlowReadException, list, r.stream()) @@ -748,7 +664,7 @@ class TestSerialize: f = tutils.tflow() d = f.get_state() d["version"] = (0, 0) - sio = StringIO() + sio = io.BytesIO() tnetstring.dump(d, sio) sio.seek(0) @@ -758,32 +674,17 @@ class TestSerialize: class TestFlowMaster: - def test_load_script(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - - fm.load_script(tutils.test_data.path("data/scripts/a.py")) - fm.load_script(tutils.test_data.path("data/scripts/a.py")) - fm.unload_scripts() - with tutils.raises(ScriptException): - fm.load_script("nonexistent") - try: - fm.load_script(tutils.test_data.path("data/scripts/starterr.py")) - except ScriptException as e: - assert "ValueError" in str(e) - assert len(fm.scripts) == 0 - def test_getset_ignore(self): p = mock.Mock() p.config.check_ignore = HostMatcher() - fm = flow.FlowMaster(p, flow.State()) + fm = flow.FlowMaster(None, p, flow.State()) assert not fm.get_ignore_filter() fm.set_ignore_filter(["^apple\.com:", ":443$"]) assert fm.get_ignore_filter() def test_replay(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow(resp=True) f.request.content = None assert "missing" in fm.replay_request(f) @@ -792,55 +693,11 @@ class TestFlowMaster: assert "intercepting" in fm.replay_request(f) f.live = True - assert "live" in fm.replay_request(f, run_scripthooks=True) - - def test_script_reqerr(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - fm.load_script(tutils.test_data.path("data/scripts/reqerr.py")) - f = tutils.tflow() - fm.clientconnect(f.client_conn) - assert fm.request(f) - - def test_script(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - fm.load_script(tutils.test_data.path("data/scripts/all.py")) - f = tutils.tflow(resp=True) - - f.client_conn.acked = False - fm.clientconnect(f.client_conn) - assert fm.scripts[0].ns["log"][-1] == "clientconnect" - f.server_conn.acked = False - fm.serverconnect(f.server_conn) - assert fm.scripts[0].ns["log"][-1] == "serverconnect" - f.reply.acked = False - fm.request(f) - assert fm.scripts[0].ns["log"][-1] == "request" - f.reply.acked = False - fm.response(f) - assert fm.scripts[0].ns["log"][-1] == "response" - # load second script - fm.load_script(tutils.test_data.path("data/scripts/all.py")) - assert len(fm.scripts) == 2 - f.server_conn.reply.acked = False - fm.clientdisconnect(f.server_conn) - assert fm.scripts[0].ns["log"][-1] == "clientdisconnect" - assert fm.scripts[1].ns["log"][-1] == "clientdisconnect" - - # unload first script - fm.unload_scripts() - assert len(fm.scripts) == 0 - fm.load_script(tutils.test_data.path("data/scripts/all.py")) - - f.error = tutils.terr() - f.reply.acked = False - fm.error(f) - assert fm.scripts[0].ns["log"][-1] == "error" + assert "live" in fm.replay_request(f) def test_duplicate_flow(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) f = tutils.tflow(resp=True) fm.load_flow(f) assert s.flow_count() == 1 @@ -851,14 +708,12 @@ class TestFlowMaster: def test_create_flow(self): s = flow.State() - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) assert fm.create_request("GET", "http", "example.com", 80, "/") def test_all(self): s = flow.State() - fm = flow.FlowMaster(None, s) - fm.anticache = True - fm.anticomp = True + fm = flow.FlowMaster(None, None, s) f = tutils.tflow(req=None) fm.clientconnect(f.client_conn) f.request = HTTPRequest.wrap(netlib.tutils.treq()) @@ -875,7 +730,6 @@ class TestFlowMaster: f.error.reply = controller.DummyReply() fm.error(f) - fm.load_script(tutils.test_data.path("data/scripts/a.py")) fm.shutdown() def test_client_playback(self): @@ -883,7 +737,7 @@ class TestFlowMaster: f = tutils.tflow(resp=True) pb = [tutils.tflow(resp=True), f] - fm = flow.FlowMaster(DummyServer(ProxyConfig()), s) + fm = flow.FlowMaster(None, DummyServer(ProxyConfig()), s) assert not fm.start_server_playback( pb, False, @@ -911,7 +765,7 @@ class TestFlowMaster: f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) pb = [f] - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.refresh_server_playback = True assert not fm.do_server_playback(tutils.tflow()) @@ -938,7 +792,7 @@ class TestFlowMaster: None, False) r = tutils.tflow() - r.request.content = "gibble" + r.request.content = b"gibble" assert not fm.do_server_playback(r) assert fm.do_server_playback(tutils.tflow()) @@ -953,7 +807,7 @@ class TestFlowMaster: f = tutils.tflow() f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=f.request)) pb = [f] - fm = flow.FlowMaster(None, s) + fm = flow.FlowMaster(None, None, s) fm.refresh_server_playback = True fm.start_server_playback( pb, @@ -971,74 +825,6 @@ class TestFlowMaster: fm.process_new_request(f) assert "killed" in f.error.msg - def test_stickycookie(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - assert "Invalid" in fm.set_stickycookie("~h") - fm.set_stickycookie(".*") - assert fm.stickycookie_state - fm.set_stickycookie(None) - assert not fm.stickycookie_state - - fm.set_stickycookie(".*") - f = tutils.tflow(resp=True) - f.response.headers["set-cookie"] = "foo=bar" - fm.request(f) - f.reply.acked = False - fm.response(f) - assert fm.stickycookie_state.jar - assert "cookie" not in f.request.headers - f = f.copy() - f.reply.acked = False - fm.request(f) - assert f.request.headers["cookie"] == "foo=bar" - - def test_stickyauth(self): - s = flow.State() - fm = flow.FlowMaster(None, s) - assert "Invalid" in fm.set_stickyauth("~h") - fm.set_stickyauth(".*") - assert fm.stickyauth_state - fm.set_stickyauth(None) - assert not fm.stickyauth_state - - fm.set_stickyauth(".*") - f = tutils.tflow(resp=True) - f.request.headers["authorization"] = "foo" - fm.request(f) - - f = tutils.tflow(resp=True) - assert fm.stickyauth_state.hosts - assert "authorization" not in f.request.headers - fm.request(f) - assert f.request.headers["authorization"] == "foo" - - def test_stream(self): - with tutils.tmpdir() as tdir: - p = os.path.join(tdir, "foo") - - def r(): - r = flow.FlowReader(open(p, "rb")) - return list(r.stream()) - - s = flow.State() - fm = flow.FlowMaster(None, s) - f = tutils.tflow(resp=True) - - fm.start_stream(file(p, "ab"), None) - fm.request(f) - fm.response(f) - fm.stop_stream() - - assert r()[0].response - - f = tutils.tflow() - fm.start_stream(file(p, "ab"), None) - fm.request(f) - fm.shutdown() - - assert not r()[1].response - class TestRequest: @@ -1073,23 +859,14 @@ class TestRequest: assert r.url == "https://address:22/path" assert r.pretty_url == "https://foo.com:22/path" - def test_anticache(self): - r = HTTPRequest.wrap(netlib.tutils.treq()) - r.headers = Headers() - r.headers["if-modified-since"] = "test" - r.headers["if-none-match"] = "test" - r.anticache() - assert "if-modified-since" not in r.headers - assert "if-none-match" not in r.headers - def test_replace(self): r = HTTPRequest.wrap(netlib.tutils.treq()) r.path = "path/foo" r.headers["Foo"] = "fOo" - r.content = "afoob" + r.content = b"afoob" assert r.replace("foo(?i)", "boo") == 4 assert r.path == "path/boo" - assert "foo" not in r.content + assert b"foo" not in r.content assert r.headers["boo"] == "boo" def test_constrain_encoding(self): @@ -1119,9 +896,9 @@ class TestResponse: def test_replace(self): r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["Foo"] = "fOo" - r.content = "afoob" + r.content = b"afoob" assert r.replace("foo(?i)", "boo") == 3 - assert "foo" not in r.content + assert b"foo" not in r.content assert r.headers["boo"] == "boo" def test_get_content_type(self): @@ -1198,24 +975,24 @@ def test_replacehooks(): assert h.count() == 0 f = tutils.tflow() - f.request.content = "foo" + f.request.content = b"foo" h.add("~s", "foo", "bar") h.run(f) - assert f.request.content == "foo" + assert f.request.content == b"foo" f = tutils.tflow(resp=True) - f.request.content = "foo" - f.response.content = "foo" + f.request.content = b"foo" + f.response.content = b"foo" h.run(f) - assert f.response.content == "bar" - assert f.request.content == "foo" + assert f.response.content == b"bar" + assert f.request.content == b"foo" f = tutils.tflow() h.clear() h.add("~q", "foo", "bar") - f.request.content = "foo" + f.request.content = b"foo" h.run(f) - assert f.request.content == "bar" + assert f.request.content == b"bar" assert not h.add("~", "foo", "bar") assert not h.add("foo", "*", "bar") @@ -1247,10 +1024,10 @@ def test_setheaders(): assert h.count() == 0 f = tutils.tflow() - f.request.content = "foo" + f.request.content = b"foo" h.add("~s", "foo", "bar") h.run(f) - assert f.request.content == "foo" + assert f.request.content == b"foo" h.clear() h.add("~s", "one", "two") diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py index 9a263b1b..e6d65e40 100644 --- a/test/mitmproxy/test_flow_export.py +++ b/test/mitmproxy/test_flow_export.py @@ -21,15 +21,15 @@ def python_equals(testdata, text): def req_get(): - return netlib.tutils.treq(method='GET', content='', path=b"/path?a=foo&a=bar&b=baz") + return netlib.tutils.treq(method=b'GET', content=b'', path=b"/path?a=foo&a=bar&b=baz") def req_post(): - return netlib.tutils.treq(method='POST', headers=()) + return netlib.tutils.treq(method=b'POST', headers=()) def req_patch(): - return netlib.tutils.treq(method='PATCH', path=b"/path?query=param") + return netlib.tutils.treq(method=b'PATCH', path=b"/path?query=param") class TestExportCurlCommand(): @@ -60,7 +60,7 @@ class TestExportPythonCode(): def test_post_json(self): p = req_post() - p.content = '{"name": "example", "email": "example@example.com"}' + p.content = b'{"name": "example", "email": "example@example.com"}' p.headers = Headers(content_type="application/json") flow = tutils.tflow(req=p) python_equals("data/test_flow_export/python_post_json.py", export.python_code(flow)) @@ -112,7 +112,7 @@ class TestExportLocustCode(): def test_post(self): p = req_post() - p.content = '''content''' + p.content = b'content' p.headers = '' flow = tutils.tflow(req=p) python_equals("data/test_flow_export/locust_post.py", export.locust_code(flow)) @@ -142,14 +142,14 @@ class TestIsJson(): def test_json_type(self): headers = Headers(content_type="application/json") - assert export.is_json(headers, "foobar") is False + assert export.is_json(headers, b"foobar") is False def test_valid(self): headers = Headers(content_type="application/foobar") - j = export.is_json(headers, '{"name": "example", "email": "example@example.com"}') + j = export.is_json(headers, b'{"name": "example", "email": "example@example.com"}') assert j is False def test_valid2(self): headers = Headers(content_type="application/json") - j = export.is_json(headers, '{"name": "example", "email": "example@example.com"}') + j = export.is_json(headers, b'{"name": "example", "email": "example@example.com"}') assert isinstance(j, dict) diff --git a/test/mitmproxy/test_flow_format_compat.py b/test/mitmproxy/test_flow_format_compat.py index b2cef88d..cc80db81 100644 --- a/test/mitmproxy/test_flow_format_compat.py +++ b/test/mitmproxy/test_flow_format_compat.py @@ -4,7 +4,7 @@ from . import tutils def test_load(): - with open(tutils.test_data.path("data/dumpfile-013"), "rb") as f: + with open(tutils.test_data.path("data/dumpfile-011"), "rb") as f: flow_reader = FlowReader(f) flows = list(flow_reader.stream()) assert len(flows) == 1 @@ -12,7 +12,7 @@ def test_load(): def test_cannot_convert(): - with open(tutils.test_data.path("data/dumpfile-012"), "rb") as f: + with open(tutils.test_data.path("data/dumpfile-010"), "rb") as f: flow_reader = FlowReader(f) with tutils.raises(FlowReadException): list(flow_reader.stream()) diff --git a/test/mitmproxy/test_options.py b/test/mitmproxy/test_options.py new file mode 100644 index 00000000..cdb0d765 --- /dev/null +++ b/test/mitmproxy/test_options.py @@ -0,0 +1,89 @@ +from __future__ import absolute_import, print_function, division +import copy + +from mitmproxy import options +from mitmproxy import exceptions +from netlib import tutils + + +class TO(options.Options): + def __init__(self, one=None, two=None): + self.one = one + self.two = two + super(TO, self).__init__() + + +def test_options(): + o = TO(two="three") + assert o.one is None + assert o.two == "three" + o.one = "one" + assert o.one == "one" + + with tutils.raises(TypeError): + TO(nonexistent = "value") + with tutils.raises("no such option"): + o.nonexistent = "value" + with tutils.raises("no such option"): + o.update(nonexistent = "value") + + rec = [] + + def sub(opts): + rec.append(copy.copy(opts)) + + o.changed.connect(sub) + + o.one = "ninety" + assert len(rec) == 1 + assert rec[-1].one == "ninety" + + o.update(one="oink") + assert len(rec) == 2 + assert rec[-1].one == "oink" + + +def test_setter(): + o = TO(two="three") + f = o.setter("two") + f("xxx") + assert o.two == "xxx" + with tutils.raises("no such option"): + o.setter("nonexistent") + + +def test_rollback(): + o = TO(one="two") + + rec = [] + + def sub(opts): + rec.append(copy.copy(opts)) + + recerr = [] + + def errsub(opts, **kwargs): + recerr.append(kwargs) + + def err(opts): + if opts.one == "ten": + raise exceptions.OptionsError() + + o.changed.connect(sub) + o.changed.connect(err) + o.errored.connect(errsub) + + o.one = "ten" + assert isinstance(recerr[0]["exc"], exceptions.OptionsError) + assert o.one == "two" + assert len(rec) == 2 + assert rec[0].one == "ten" + assert rec[1].one == "two" + + +def test_repr(): + assert repr(TO()) == "test.mitmproxy.test_options.TO({'one': None, 'two': None})" + assert repr(TO(one='x' * 60)) == """test.mitmproxy.test_options.TO({ + 'one': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', + 'two': None +})""" diff --git a/test/mitmproxy/test_protocol_http2.py b/test/mitmproxy/test_protocol_http2.py index 6e021b2c..a100ac2d 100644 --- a/test/mitmproxy/test_protocol_http2.py +++ b/test/mitmproxy/test_protocol_http2.py @@ -3,9 +3,10 @@ from __future__ import (absolute_import, print_function, division) import pytest -import traceback import os import tempfile +import traceback + import h2 from mitmproxy.proxy.config import ProxyConfig @@ -46,6 +47,11 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): self.wfile.write(h2_conn.data_to_send()) self.wfile.flush() + if 'h2_server_settings' in self.kwargs: + h2_conn.update_settings(self.kwargs['h2_server_settings']) + self.wfile.write(h2_conn.data_to_send()) + self.wfile.flush() + done = False while not done: try: @@ -54,7 +60,10 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): except HttpException: print(traceback.format_exc()) assert False + except netlib.exceptions.TcpDisconnect: + break except: + print(traceback.format_exc()) break self.wfile.write(h2_conn.data_to_send()) self.wfile.flush() @@ -64,8 +73,11 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): if not self.server.handle_server_event(event, h2_conn, self.rfile, self.wfile): done = True break + except netlib.exceptions.TcpDisconnect: + done = True except: done = True + print(traceback.format_exc()) break def handle_server_event(self, h2_conn, rfile, wfile): @@ -132,11 +144,22 @@ class _Http2TestBase(object): return client, h2_conn - def _send_request(self, wfile, h2_conn, stream_id=1, headers=[], body=b''): + def _send_request(self, + wfile, + h2_conn, + stream_id=1, + headers=[], + body=b'', + priority_exclusive=None, + priority_depends_on=None, + priority_weight=None): h2_conn.send_headers( stream_id=stream_id, headers=headers, end_stream=(len(body) == 0), + priority_exclusive=priority_exclusive, + priority_depends_on=priority_depends_on, + priority_weight=priority_weight, ) if body: h2_conn.send_data(stream_id, body) @@ -145,8 +168,7 @@ class _Http2TestBase(object): wfile.flush() -@requires_alpn -class TestSimple(_Http2TestBase, _Http2ServerBase): +class _Http2Test(_Http2TestBase, _Http2ServerBase): @classmethod def setup_class(self): @@ -158,14 +180,19 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): _Http2TestBase.teardown_class() _Http2ServerBase.teardown_class() + +@requires_alpn +class TestSimple(_Http2Test): + request_body_buffer = b'' + @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): return False elif isinstance(event, h2.events.RequestReceived): - assert ('client-foo', 'client-bar-1') in event.headers - assert ('client-foo', 'client-bar-2') in event.headers - + assert (b'client-foo', b'client-bar-1') in event.headers + assert (b'client-foo', b'client-bar-2') in event.headers + elif isinstance(event, h2.events.StreamEnded): import warnings with warnings.catch_warnings(): # Ignore UnicodeWarning: @@ -181,23 +208,30 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): ('föo', 'bär'), ('X-Stream-ID', str(event.stream_id)), ]) - h2_conn.send_data(event.stream_id, b'foobar') + h2_conn.send_data(event.stream_id, b'response body') h2_conn.end_stream(event.stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() + elif isinstance(event, h2.events.DataReceived): + self.request_body_buffer += event.data return True def test_simple(self): + response_body_buffer = b'' client, h2_conn = self._setup_connection() - self._send_request(client.wfile, h2_conn, headers=[ - (':authority', "127.0.0.1:%s" % self.server.server.address.port), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/'), - ('ClIeNt-FoO', 'client-bar-1'), - ('ClIeNt-FoO', 'client-bar-2'), - ], body=b'my request body echoed back to me') + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ('ClIeNt-FoO', 'client-bar-1'), + ('ClIeNt-FoO', 'client-bar-2'), + ], + body=b'request body') done = False while not done: @@ -212,7 +246,9 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): client.wfile.flush() for event in events: - if isinstance(event, h2.events.StreamEnded): + if isinstance(event, h2.events.DataReceived): + response_body_buffer += event.data + elif isinstance(event, h2.events.StreamEnded): done = True h2_conn.close_connection() @@ -223,41 +259,136 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): assert self.master.state.flows[0].response.status_code == 200 assert self.master.state.flows[0].response.headers['server-foo'] == 'server-bar' assert self.master.state.flows[0].response.headers['föo'] == 'bär' - assert self.master.state.flows[0].response.body == b'foobar' + assert self.master.state.flows[0].response.body == b'response body' + assert self.request_body_buffer == b'request body' + assert response_body_buffer == b'response body' @requires_alpn -class TestWithBodies(_Http2TestBase, _Http2ServerBase): - tmp_data_buffer_foobar = b'' +class TestRequestWithPriority(_Http2Test): @classmethod - def setup_class(self): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class() + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.RequestReceived): + import warnings + with warnings.catch_warnings(): + # Ignore UnicodeWarning: + # h2/utilities.py:64: UnicodeWarning: Unicode equal comparison + # failed to convert both arguments to Unicode - interpreting + # them as being unequal. + # elif header[0] in (b'cookie', u'cookie') and len(header[1]) < 20: - @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() + warnings.simplefilter("ignore") + + headers = [(':status', '200')] + if event.priority_updated: + headers.append(('priority_exclusive', event.priority_updated.exclusive)) + headers.append(('priority_depends_on', event.priority_updated.depends_on)) + headers.append(('priority_weight', event.priority_updated.weight)) + h2_conn.send_headers(event.stream_id, headers) + h2_conn.end_stream(event.stream_id) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return True + + def test_request_with_priority(self): + client, h2_conn = self._setup_connection() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + priority_exclusive = True, + priority_depends_on = 42424242, + priority_weight = 42, + ) + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 1 + assert self.master.state.flows[0].response.headers['priority_exclusive'] == 'True' + assert self.master.state.flows[0].response.headers['priority_depends_on'] == '42424242' + assert self.master.state.flows[0].response.headers['priority_weight'] == '42' + + def test_request_without_priority(self): + client, h2_conn = self._setup_connection() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + ) + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 1 + assert 'priority_exclusive' not in self.master.state.flows[0].response.headers + assert 'priority_depends_on' not in self.master.state.flows[0].response.headers + assert 'priority_weight' not in self.master.state.flows[0].response.headers + + +@requires_alpn +class TestStreamResetFromServer(_Http2Test): @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): if isinstance(event, h2.events.ConnectionTerminated): return False - if isinstance(event, h2.events.DataReceived): - self.tmp_data_buffer_foobar += event.data - elif isinstance(event, h2.events.StreamEnded): - h2_conn.send_headers(1, [ - (':status', '200'), - ]) - h2_conn.send_data(1, self.tmp_data_buffer_foobar) - h2_conn.end_stream(1) + elif isinstance(event, h2.events.RequestReceived): + h2_conn.reset_stream(event.stream_id, 0x8) wfile.write(h2_conn.data_to_send()) wfile.flush() - return True - def test_with_bodies(self): + def test_request_with_priority(self): client, h2_conn = self._setup_connection() self._send_request( @@ -269,7 +400,6 @@ class TestWithBodies(_Http2TestBase, _Http2ServerBase): (':scheme', 'https'), (':path', '/'), ], - body=b'foobar with request body', ) done = False @@ -285,28 +415,68 @@ class TestWithBodies(_Http2TestBase, _Http2ServerBase): client.wfile.flush() for event in events: - if isinstance(event, h2.events.StreamEnded): + if isinstance(event, h2.events.StreamReset): done = True h2_conn.close_connection() client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() - assert self.master.state.flows[0].response.body == b'foobar with request body' + assert len(self.master.state.flows) == 1 + assert self.master.state.flows[0].response is None @requires_alpn -class TestPushPromise(_Http2TestBase, _Http2ServerBase): +class TestBodySizeLimit(_Http2Test): @classmethod - def setup_class(self): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class() + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + return True - @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() + def test_body_size_limit(self): + self.config.body_size_limit = 20 + + client, h2_conn = self._setup_connection() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + body=b'very long body over 20 characters long', + ) + + done = False + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + except HttpException: + print(traceback.format_exc()) + assert False + + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamReset): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == 0 + + +@requires_alpn +class TestPushPromise(_Http2Test): @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): @@ -459,17 +629,7 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): @requires_alpn -class TestConnectionLost(_Http2TestBase, _Http2ServerBase): - - @classmethod - def setup_class(self): - _Http2TestBase.setup_class() - _Http2ServerBase.setup_class() - - @classmethod - def teardown_class(self): - _Http2TestBase.teardown_class() - _Http2ServerBase.teardown_class() +class TestConnectionLost(_Http2Test): @classmethod def handle_server_event(self, event, h2_conn, rfile, wfile): @@ -508,3 +668,105 @@ class TestConnectionLost(_Http2TestBase, _Http2ServerBase): if len(self.master.state.flows) == 1: assert self.master.state.flows[0].response is None + + +@requires_alpn +class TestMaxConcurrentStreams(_Http2Test): + + @classmethod + def setup_class(self): + _Http2TestBase.setup_class() + _Http2ServerBase.setup_class(h2_server_settings={h2.settings.MAX_CONCURRENT_STREAMS: 2}) + + @classmethod + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.RequestReceived): + h2_conn.send_headers(event.stream_id, [ + (':status', '200'), + ('X-Stream-ID', str(event.stream_id)), + ]) + h2_conn.send_data(event.stream_id, 'Stream-ID {}'.format(event.stream_id).encode()) + h2_conn.end_stream(event.stream_id) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return True + + def test_max_concurrent_streams(self): + client, h2_conn = self._setup_connection() + new_streams = [1, 3, 5, 7, 9, 11] + for id in new_streams: + # this will exceed MAX_CONCURRENT_STREAMS on the server connection + # and cause mitmproxy to throttle stream creation to the server + self._send_request(client.wfile, h2_conn, stream_id=id, headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ('X-Stream-ID', str(id)), + ]) + + ended_streams = 0 + while ended_streams != len(new_streams): + try: + header, body = framereader.http2_read_raw_frame(client.rfile) + events = h2_conn.receive_data(b''.join([header, body])) + except: + break + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + ended_streams += 1 + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert len(self.master.state.flows) == len(new_streams) + for flow in self.master.state.flows: + assert flow.response.status_code == 200 + assert b"Stream-ID " in flow.response.body + + +@requires_alpn +class TestConnectionTerminated(_Http2Test): + + @classmethod + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.RequestReceived): + h2_conn.close_connection(error_code=5, last_stream_id=42, additional_data=b'foobar') + wfile.write(h2_conn.data_to_send()) + wfile.flush() + return True + + def test_connection_terminated(self): + client, h2_conn = self._setup_connection() + + self._send_request(client.wfile, h2_conn, headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ]) + + done = False + connection_terminated_event = None + while not done: + try: + raw = b''.join(framereader.http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) + for event in events: + if isinstance(event, h2.events.ConnectionTerminated): + connection_terminated_event = event + done = True + except: + break + + assert len(self.master.state.flows) == 1 + assert connection_terminated_event is not None + assert connection_terminated_event.error_code == 5 + assert connection_terminated_event.last_stream_id == 42 + assert connection_terminated_event.additional_data == b'foobar' diff --git a/test/mitmproxy/test_script.py b/test/mitmproxy/test_script.py deleted file mode 100644 index 81994780..00000000 --- a/test/mitmproxy/test_script.py +++ /dev/null @@ -1,13 +0,0 @@ -from mitmproxy import flow -from . import tutils - - -def test_duplicate_flow(): - s = flow.State() - fm = flow.FlowMaster(None, s) - fm.load_script(tutils.test_data.path("data/scripts/duplicate_flow.py")) - f = tutils.tflow() - fm.request(f) - assert fm.state.flow_count() == 2 - assert not fm.state.view[0].request.is_replay - assert fm.state.view[1].request.is_replay diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 1bbef975..a5196dae 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -1,6 +1,7 @@ import os import socket import time +import types from OpenSSL import SSL from netlib.exceptions import HttpReadDisconnect, HttpException from netlib.tcp import Address @@ -12,6 +13,7 @@ from netlib.http import authentication, http1 from netlib.tutils import raises from pathod import pathoc, pathod +from mitmproxy.builtins import script from mitmproxy import controller from mitmproxy.proxy.config import HostMatcher from mitmproxy.models import Error, HTTPResponse, HTTPFlow @@ -100,10 +102,10 @@ class CommonMixin: if not self.ssl: return - f = self.pathod("304", sni=b"testserver.com") + f = self.pathod("304", sni="testserver.com") assert f.status_code == 304 log = self.server.last_log() - assert log["request"]["sni"] == b"testserver.com" + assert log["request"]["sni"] == "testserver.com" class TcpMixin: @@ -286,10 +288,13 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin): self.master.set_stream_large_bodies(None) def test_stream_modify(self): - self.master.load_script(tutils.test_data.path("data/scripts/stream_modify.py")) + s = script.Script( + tutils.test_data.path("data/addonscripts/stream_modify.py") + ) + self.master.addons.add(s) d = self.pathod('200:b"foo"') assert d.content == b"bar" - self.master.unload_scripts() + self.master.addons.remove(s) class TestHTTPAuth(tservers.HTTPProxyTest): @@ -498,7 +503,7 @@ class TestHttps2Http(tservers.ReverseProxyTest): assert p.request("get:'/p/200'").status_code == 200 def test_sni(self): - p = self.pathoc(ssl=True, sni=b"example.com") + p = self.pathoc(ssl=True, sni="example.com") assert p.request("get:'/p/200'").status_code == 200 assert all("Error in handle_sni" not in msg for msg in self.proxy.tlog) @@ -511,15 +516,15 @@ class TestTransparent(tservers.TransparentProxyTest, CommonMixin, TcpMixin): ssl = False def test_tcp_stream_modify(self): - self.master.load_script(tutils.test_data.path("data/scripts/tcp_stream_modify.py")) - + s = script.Script( + tutils.test_data.path("data/addonscripts/tcp_stream_modify.py") + ) + self.master.addons.add(s) self._tcpproxy_on() d = self.pathod('200:b"foo"') self._tcpproxy_off() - assert d.content == b"bar" - - self.master.unload_scripts() + self.master.addons.remove(s) class TestTransparentSSL(tservers.TransparentProxyTest, CommonMixin, TcpMixin): @@ -945,7 +950,7 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest): f.reply.kill() return _func(f) - setattr(master, attr, handler) + setattr(master, attr, types.MethodType(handler, master)) kill_requests( self.chain[1].tmaster, diff --git a/test/mitmproxy/test_web_master.py b/test/mitmproxy/test_web_master.py index 98f53c93..2ab440ce 100644 --- a/test/mitmproxy/test_web_master.py +++ b/test/mitmproxy/test_web_master.py @@ -3,15 +3,12 @@ from . import mastertest class TestWebMaster(mastertest.MasterTest): - def mkmaster(self, filt, **options): - o = master.Options( - filtstr=filt, - **options - ) + def mkmaster(self, **options): + o = master.Options(**options) return master.WebMaster(None, o) def test_basic(self): - m = self.mkmaster(None) + m = self.mkmaster() for i in (1, 2, 3): - self.dummy_cycle(m, 1, "") + self.dummy_cycle(m, 1, b"") assert len(m.state.flows) == i diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 6d8730f5..9a66984b 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -11,7 +11,6 @@ import pathod.pathoc from mitmproxy import flow, controller from mitmproxy.cmdline import APP_HOST, APP_PORT - testapp = flask.Flask(__name__) @@ -35,7 +34,7 @@ class TestMaster(flow.FlowMaster): config.port = 0 s = ProxyServer(config) state = flow.State() - flow.FlowMaster.__init__(self, s, state) + flow.FlowMaster.__init__(self, None, s, state) self.apps.add(testapp, "testapp", 80) self.apps.add(errapp, "errapp", 80) self.clear_log() @@ -43,7 +42,7 @@ class TestMaster(flow.FlowMaster): def clear_log(self): self.tlog = [] - def add_event(self, message, level=None): + def add_log(self, message, level=None): self.tlog.append(message) diff --git a/test/netlib/http/http1/test_assemble.py b/test/netlib/http/http1/test_assemble.py index 50d29384..841ea58a 100644 --- a/test/netlib/http/http1/test_assemble.py +++ b/test/netlib/http/http1/test_assemble.py @@ -24,7 +24,7 @@ def test_assemble_request(): def test_assemble_request_head(): - c = assemble_request_head(treq(content="foo")) + c = assemble_request_head(treq(content=b"foo")) assert b"GET" in c assert b"qvalue" in c assert b"content-length" in c diff --git a/test/netlib/http/test_cookies.py b/test/netlib/http/test_cookies.py index 83b85656..17e21b94 100644 --- a/test/netlib/http/test_cookies.py +++ b/test/netlib/http/test_cookies.py @@ -245,3 +245,24 @@ def test_refresh_cookie(): assert cookies.refresh_set_cookie_header(c, 0) c = "foo/bar=bla" assert cookies.refresh_set_cookie_header(c, 0) + + +def test_is_expired(): + CA = cookies.CookieAttrs + + # A cookie can be expired + # by setting the expire time in the past + assert cookies.is_expired(CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT")])) + + # or by setting Max-Age to 0 + assert cookies.is_expired(CA([("Max-Age", "0")])) + + # or both + assert cookies.is_expired(CA([("Expires", "Thu, 01-Jan-1970 00:00:00 GMT"), ("Max-Age", "0")])) + + assert not cookies.is_expired(CA([("Expires", "Thu, 24-Aug-2063 00:00:00 GMT")])) + assert not cookies.is_expired(CA([("Max-Age", "1")])) + assert not cookies.is_expired(CA([("Expires", "Thu, 15-Jul-2068 00:00:00 GMT"), ("Max-Age", "1")])) + + assert not cookies.is_expired(CA([("Max-Age", "nan")])) + assert not cookies.is_expired(CA([("Expires", "false")])) diff --git a/test/netlib/http/test_message.py b/test/netlib/http/test_message.py index 8b178e04..deebd6f2 100644 --- a/test/netlib/http/test_message.py +++ b/test/netlib/http/test_message.py @@ -10,8 +10,8 @@ from netlib import http, tutils def _test_passthrough_attr(message, attr): assert getattr(message, attr) == getattr(message.data, attr) - setattr(message, attr, "foo") - assert getattr(message.data, attr) == "foo" + setattr(message, attr, b"foo") + assert getattr(message.data, attr) == b"foo" def _test_decoded_attr(message, attr): @@ -233,7 +233,7 @@ class TestMessageText(object): def test_none(self): r = tresp(content=None) assert r.text is None - r.text = b"foo" + r.text = u"foo" assert r.text is not None r.text = None assert r.text is None diff --git a/test/netlib/http/test_request.py b/test/netlib/http/test_request.py index c03db339..f3cd8b71 100644 --- a/test/netlib/http/test_request.py +++ b/test/netlib/http/test_request.py @@ -248,20 +248,20 @@ class TestRequestUtils(object): assert "gzip" in request.headers["Accept-Encoding"] def test_get_urlencoded_form(self): - request = treq(content="foobar=baz") + request = treq(content=b"foobar=baz") assert not request.urlencoded_form request.headers["Content-Type"] = "application/x-www-form-urlencoded" - assert list(request.urlencoded_form.items()) == [("foobar", "baz")] + assert list(request.urlencoded_form.items()) == [(b"foobar", b"baz")] def test_set_urlencoded_form(self): request = treq() - request.urlencoded_form = [('foo', 'bar'), ('rab', 'oof')] + request.urlencoded_form = [(b'foo', b'bar'), (b'rab', b'oof')] assert request.headers["Content-Type"] == "application/x-www-form-urlencoded" assert request.content def test_get_multipart_form(self): - request = treq(content="foobar") + request = treq(content=b"foobar") assert not request.multipart_form request.headers["Content-Type"] = "multipart/form-data" diff --git a/test/netlib/test_strutils.py b/test/netlib/test_strutils.py index 16e5d0b3..7c3eacc6 100644 --- a/test/netlib/test_strutils.py +++ b/test/netlib/test_strutils.py @@ -1,9 +1,15 @@ -# coding=utf-8 import six from netlib import strutils, tutils +def test_always_bytes(): + assert strutils.always_bytes(bytes(bytearray(range(256)))) == bytes(bytearray(range(256))) + assert strutils.always_bytes("foo") == b"foo" + with tutils.raises(ValueError): + strutils.always_bytes(u"\u2605", "ascii") + + def test_native(): with tutils.raises(TypeError): strutils.native(42) @@ -15,18 +21,26 @@ def test_native(): assert strutils.native(b"foo") == u"foo" -def test_clean_bin(): - assert strutils.clean_bin(b"one") == u"one" - assert strutils.clean_bin(b"\00ne") == u".ne" - assert strutils.clean_bin(b"\nne") == u"\nne" - assert strutils.clean_bin(b"\nne", False) == u".ne" - assert strutils.clean_bin(u"\u2605".encode("utf8")) == u"..." - - assert strutils.clean_bin(u"one") == u"one" - assert strutils.clean_bin(u"\00ne") == u".ne" - assert strutils.clean_bin(u"\nne") == u"\nne" - assert strutils.clean_bin(u"\nne", False) == u".ne" - assert strutils.clean_bin(u"\u2605") == u"\u2605" +def test_escape_control_characters(): + assert strutils.escape_control_characters(u"one") == u"one" + assert strutils.escape_control_characters(u"\00ne") == u".ne" + assert strutils.escape_control_characters(u"\nne") == u"\nne" + assert strutils.escape_control_characters(u"\nne", False) == u".ne" + assert strutils.escape_control_characters(u"\u2605") == u"\u2605" + assert ( + strutils.escape_control_characters(bytes(bytearray(range(128))).decode()) == + u'.........\t\n..\r.................. !"#$%&\'()*+,-./0123456789:;<' + u'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.' + ) + assert ( + strutils.escape_control_characters(bytes(bytearray(range(128))).decode(), False) == + u'................................ !"#$%&\'()*+,-./0123456789:;<' + u'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.' + ) + + if not six.PY2: + with tutils.raises(ValueError): + strutils.escape_control_characters(b"foo") def test_bytes_to_escaped_str(): @@ -37,6 +51,14 @@ def test_bytes_to_escaped_str(): assert strutils.bytes_to_escaped_str(b"'") == r"\'" assert strutils.bytes_to_escaped_str(b'"') == r'"' + assert strutils.bytes_to_escaped_str(b"\r\n\t") == "\\r\\n\\t" + assert strutils.bytes_to_escaped_str(b"\r\n\t", True) == "\r\n\t" + + assert strutils.bytes_to_escaped_str(b"\n", True) == "\n" + assert strutils.bytes_to_escaped_str(b"\\n", True) == "\\ \\ n".replace(" ", "") + assert strutils.bytes_to_escaped_str(b"\\\n", True) == "\\ \\ \n".replace(" ", "") + assert strutils.bytes_to_escaped_str(b"\\\\n", True) == "\\ \\ \\ \\ n".replace(" ", "") + with tutils.raises(ValueError): strutils.bytes_to_escaped_str(u"such unicode") @@ -45,10 +67,9 @@ def test_escaped_str_to_bytes(): assert strutils.escaped_str_to_bytes("foo") == b"foo" assert strutils.escaped_str_to_bytes("\x08") == b"\b" assert strutils.escaped_str_to_bytes("&!?=\\\\)") == br"&!?=\)" - assert strutils.escaped_str_to_bytes("ü") == b'\xc3\xbc' assert strutils.escaped_str_to_bytes(u"\\x08") == b"\b" assert strutils.escaped_str_to_bytes(u"&!?=\\\\)") == br"&!?=\)" - assert strutils.escaped_str_to_bytes(u"ü") == b'\xc3\xbc' + assert strutils.escaped_str_to_bytes(u"\u00fc") == b'\xc3\xbc' if six.PY2: with tutils.raises(ValueError): @@ -58,17 +79,15 @@ def test_escaped_str_to_bytes(): strutils.escaped_str_to_bytes(b"very byte") -def test_isBin(): - assert not strutils.isBin("testing\n\r") - assert strutils.isBin("testing\x01") - assert strutils.isBin("testing\x0e") - assert strutils.isBin("testing\x7f") +def test_is_mostly_bin(): + assert not strutils.is_mostly_bin(b"foo\xFF") + assert strutils.is_mostly_bin(b"foo" + b"\xFF" * 10) -def test_isXml(): - assert not strutils.isXML("foo") - assert strutils.isXML("<foo") - assert strutils.isXML(" \n<foo") +def test_is_xml(): + assert not strutils.is_xml(b"foo") + assert strutils.is_xml(b"<foo") + assert strutils.is_xml(b" \n<foo") def test_clean_hanging_newline(): diff --git a/test/netlib/test_tcp.py b/test/netlib/test_tcp.py index 590bcc01..273427d5 100644 --- a/test/netlib/test_tcp.py +++ b/test/netlib/test_tcp.py @@ -169,7 +169,7 @@ class TestServerSSL(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni=b"foo.com", options=SSL.OP_ALL) + c.convert_to_ssl(sni="foo.com", options=SSL.OP_ALL) testval = b"echo!\n" c.wfile.write(testval) c.wfile.flush() @@ -179,7 +179,7 @@ class TestServerSSL(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): assert not c.get_current_cipher() - c.convert_to_ssl(sni=b"foo.com") + c.convert_to_ssl(sni="foo.com") ret = c.get_current_cipher() assert ret assert "AES" in ret[0] @@ -195,7 +195,7 @@ class TestSSLv3Only(tservers.ServerTestBase): def test_failure(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - tutils.raises(TlsException, c.convert_to_ssl, sni=b"foo.com") + tutils.raises(TlsException, c.convert_to_ssl, sni="foo.com") class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): @@ -238,7 +238,7 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): with c.connect(): with tutils.raises(InvalidCertificateException): c.convert_to_ssl( - sni=b"example.mitmproxy.org", + sni="example.mitmproxy.org", verify_options=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("data/verificationcerts/trusted-root.crt") ) @@ -272,7 +272,7 @@ class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): with c.connect(): with tutils.raises(InvalidCertificateException): c.convert_to_ssl( - sni=b"mitmproxy.org", + sni="mitmproxy.org", verify_options=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("data/verificationcerts/trusted-root.crt") ) @@ -291,7 +291,7 @@ class TestSSLUpstreamCertVerificationWValidCertChain(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): c.convert_to_ssl( - sni=b"example.mitmproxy.org", + sni="example.mitmproxy.org", verify_options=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("data/verificationcerts/trusted-root.crt") ) @@ -307,7 +307,7 @@ class TestSSLUpstreamCertVerificationWValidCertChain(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): c.convert_to_ssl( - sni=b"example.mitmproxy.org", + sni="example.mitmproxy.org", verify_options=SSL.VERIFY_PEER, ca_path=tutils.test_data.path("data/verificationcerts/") ) @@ -371,8 +371,8 @@ class TestSNI(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni=b"foo.com") - assert c.sni == b"foo.com" + c.convert_to_ssl(sni="foo.com") + assert c.sni == "foo.com" assert c.rfile.readline() == b"foo.com" @@ -385,7 +385,7 @@ class TestServerCipherList(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni=b"foo.com") + c.convert_to_ssl(sni="foo.com") assert c.rfile.readline() == b"['RC4-SHA']" @@ -405,7 +405,7 @@ class TestServerCurrentCipher(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni=b"foo.com") + c.convert_to_ssl(sni="foo.com") assert b"RC4-SHA" in c.rfile.readline() @@ -418,7 +418,7 @@ class TestServerCipherListError(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - tutils.raises("handshake error", c.convert_to_ssl, sni=b"foo.com") + tutils.raises("handshake error", c.convert_to_ssl, sni="foo.com") class TestClientCipherListError(tservers.ServerTestBase): @@ -433,7 +433,7 @@ class TestClientCipherListError(tservers.ServerTestBase): tutils.raises( "cipher specification", c.convert_to_ssl, - sni=b"foo.com", + sni="foo.com", cipher_list="bogus" ) diff --git a/test/netlib/tservers.py b/test/netlib/tservers.py index 803aaa72..666f97ac 100644 --- a/test/netlib/tservers.py +++ b/test/netlib/tservers.py @@ -24,7 +24,7 @@ class _ServerThread(threading.Thread): class _TServer(tcp.TCPServer): - def __init__(self, ssl, q, handler_klass, addr): + def __init__(self, ssl, q, handler_klass, addr, **kwargs): """ ssl: A dictionary of SSL parameters: @@ -42,6 +42,8 @@ class _TServer(tcp.TCPServer): self.q = q self.handler_klass = handler_klass + if self.handler_klass is not None: + self.handler_klass.kwargs = kwargs self.last_handler = None def handle_client_connection(self, request, client_address): @@ -89,16 +91,16 @@ class ServerTestBase(object): addr = ("localhost", 0) @classmethod - def setup_class(cls): + def setup_class(cls, **kwargs): cls.q = queue.Queue() - s = cls.makeserver() + s = cls.makeserver(**kwargs) cls.port = s.address.port cls.server = _ServerThread(s) cls.server.start() @classmethod - def makeserver(cls): - return _TServer(cls.ssl, cls.q, cls.handler, cls.addr) + def makeserver(cls, **kwargs): + return _TServer(cls.ssl, cls.q, cls.handler, cls.addr, **kwargs) @classmethod def teardown_class(cls): diff --git a/test/pathod/test_pathoc.py b/test/pathod/test_pathoc.py index 28f9f0f8..361a863b 100644 --- a/test/pathod/test_pathoc.py +++ b/test/pathod/test_pathoc.py @@ -54,10 +54,10 @@ class TestDaemonSSL(PathocTestDaemon): def test_sni(self): self.tval( ["get:/p/200"], - sni=b"foobar.com" + sni="foobar.com" ) log = self.d.log() - assert log[0]["request"]["sni"] == b"foobar.com" + assert log[0]["request"]["sni"] == "foobar.com" def test_showssl(self): assert "certificate chain" in self.tval(["get:/p/200"], showssl=True) diff --git a/test/pathod/test_protocols_http2.py b/test/pathod/test_protocols_http2.py index e42c2858..8d7efc82 100644 --- a/test/pathod/test_protocols_http2.py +++ b/test/pathod/test_protocols_http2.py @@ -367,37 +367,6 @@ class TestReadRequestAbsolute(netlib_tservers.ServerTestBase): assert req.port == 22 -class TestReadRequestConnect(netlib_tservers.ServerTestBase): - class handler(tcp.BaseHandler): - def handle(self): - self.wfile.write( - codecs.decode('00001b0105000000014287bdab4e9c17b7ff44871c92585422e08541871c92585422e085', 'hex_codec')) - self.wfile.write( - codecs.decode('00001d0105000000014287bdab4e9c17b7ff44882f91d35d055c87a741882f91d35d055c87a7', 'hex_codec')) - self.wfile.flush() - - ssl = True - - def test_connect(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - with c.connect(): - c.convert_to_ssl() - protocol = HTTP2StateProtocol(c, is_server=True) - protocol.connection_preface_performed = True - - req = protocol.read_request(NotImplemented) - assert req.first_line_format == "authority" - assert req.method == "CONNECT" - assert req.host == "address" - assert req.port == 22 - - req = protocol.read_request(NotImplemented) - assert req.first_line_format == "authority" - assert req.method == "CONNECT" - assert req.host == "example.com" - assert req.port == 443 - - class TestReadResponse(netlib_tservers.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): |