diff options
43 files changed, 348 insertions, 327 deletions
diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py index 9d0e7069..ac561073 100644 --- a/mitmproxy/addonmanager.py +++ b/mitmproxy/addonmanager.py @@ -140,6 +140,7 @@ class AddonManager: """ for i in self.chain: self.remove(i) + self.lookup = {} def get(self, name): """ diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index 51341f2f..decd0759 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -1,7 +1,7 @@ +import asyncio import os import importlib.util import importlib.machinery -import time import sys import types import typing @@ -59,13 +59,15 @@ def script_error_handler(path, exc, msg="", tb=False): ctx.log.error(log_msg) +ReloadInterval = 1 + + class Script: """ An addon that manages a single script. """ - ReloadInterval = 2 - def __init__(self, path): + def __init__(self, path: str, reload: bool) -> None: self.name = "scriptmanager:" + path self.path = path self.fullpath = os.path.expanduser( @@ -73,45 +75,57 @@ class Script: ) self.ns = None - self.last_load = 0 - self.last_mtime = 0 if not os.path.isfile(self.fullpath): raise exceptions.OptionsError('No such script') + self.reloadtask = None + if reload: + self.reloadtask = asyncio.ensure_future(self.watcher()) + else: + self.loadscript() + + def done(self): + if self.reloadtask: + self.reloadtask.cancel() + @property def addons(self): return [self.ns] if self.ns else [] - def tick(self): - if time.time() - self.last_load > self.ReloadInterval: + def loadscript(self): + ctx.log.info("Loading script %s" % self.path) + if self.ns: + ctx.master.addons.remove(self.ns) + self.ns = None + with addonmanager.safecall(): + ns = load_script(self.fullpath) + ctx.master.addons.register(ns) + self.ns = ns + if self.ns: + # We're already running, so we have to explicitly register and + # configure the addon + ctx.master.addons.invoke_addon(self.ns, "running") + ctx.master.addons.invoke_addon( + self.ns, + "configure", + ctx.options.keys() + ) + + async def watcher(self): + last_mtime = 0 + while True: try: mtime = os.stat(self.fullpath).st_mtime except FileNotFoundError: + ctx.log.info("Removing script %s" % self.path) scripts = list(ctx.options.scripts) scripts.remove(self.path) ctx.options.update(scripts=scripts) return - - if mtime > self.last_mtime: - ctx.log.info("Loading script: %s" % self.path) - if self.ns: - ctx.master.addons.remove(self.ns) - self.ns = None - with addonmanager.safecall(): - ns = load_script(self.fullpath) - ctx.master.addons.register(ns) - self.ns = ns - if self.ns: - # We're already running, so we have to explicitly register and - # configure the addon - ctx.master.addons.invoke_addon(self.ns, "running") - ctx.master.addons.invoke_addon( - self.ns, - "configure", - ctx.options.keys() - ) - self.last_load = time.time() - self.last_mtime = mtime + if mtime > last_mtime: + self.loadscript() + last_mtime = mtime + await asyncio.sleep(ReloadInterval) class ScriptLoader: @@ -125,9 +139,7 @@ class ScriptLoader: def load(self, loader): loader.add_option( "scripts", typing.Sequence[str], [], - """ - Execute a script. - """ + "Execute a script." ) def running(self): @@ -141,12 +153,7 @@ class ScriptLoader: simulated. """ try: - s = Script(path) - l = addonmanager.Loader(ctx.master) - ctx.master.addons.invoke_addon(s, "load", l) - ctx.master.addons.invoke_addon(s, "configure", ctx.options.keys()) - # Script is loaded on the first tick - ctx.master.addons.invoke_addon(s, "tick") + s = Script(path, False) for f in flows: for evt, arg in eventsequence.iterate(f): ctx.master.addons.invoke_addon(s, evt, arg) @@ -161,7 +168,7 @@ class ScriptLoader: for a in self.addons[:]: if a.path not in ctx.options.scripts: - ctx.log.info("Un-loading script: %s" % a.name) + ctx.log.info("Un-loading script: %s" % a.path) ctx.master.addons.remove(a) self.addons.remove(a) @@ -181,7 +188,7 @@ class ScriptLoader: if s in current: ordered.append(current[s]) else: - sc = Script(s) + sc = Script(s, True) ordered.append(sc) newscripts.append(sc) diff --git a/mitmproxy/master.py b/mitmproxy/master.py index bbbd07d0..8eb01600 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -94,19 +94,14 @@ class Master: exc = None try: loop() - except Exception as e: + except Exception as e: # pragma: no cover exc = traceback.format_exc() finally: - if not self.should_exit.is_set(): + if not self.should_exit.is_set(): # pragma: no cover self.shutdown() - pending = asyncio.Task.all_tasks() loop = asyncio.get_event_loop() - try: - loop.run_until_complete(asyncio.gather(*pending)) - except Exception as e: - # When we exit with an error, shutdown might not happen cleanly, - # and we can get exceptions here caused by pending Futures. - pass + for p in asyncio.Task.all_tasks(): + p.cancel() loop.close() if exc: # pragma: no cover @@ -122,6 +117,7 @@ class Master: self.run_loop(loop.run_forever) async def _shutdown(self): + self.should_exit.set() if self.server: self.server.shutdown() loop = asyncio.get_event_loop() diff --git a/mitmproxy/test/taddons.py b/mitmproxy/test/taddons.py index 73e57456..0505f9f7 100644 --- a/mitmproxy/test/taddons.py +++ b/mitmproxy/test/taddons.py @@ -35,7 +35,7 @@ class RecordingMaster(mitmproxy.master.Master): for i in self.logs: print("%s: %s" % (i.level, i.msg), file=outf) - def _has_log(self, txt, level=None): + def has_log(self, txt, level=None): for i in self.logs: if level and i.level != level: continue @@ -45,7 +45,7 @@ class RecordingMaster(mitmproxy.master.Master): async def await_log(self, txt, level=None): for i in range(20): - if self._has_log(txt, level): + if self.has_log(txt, level): return True else: await asyncio.sleep(0.1) @@ -123,11 +123,7 @@ class context: """ Loads a script from path, and returns the enclosed addon. """ - sc = script.Script(path) - loader = addonmanager.Loader(self.master) - self.master.addons.invoke_addon(sc, "load", loader) - self.configure(sc) - self.master.addons.invoke_addon(sc, "tick") + sc = script.Script(path, False) return sc.addons[0] if sc.addons else None def invoke(self, addon, event, *args, **kwargs): diff --git a/mitmproxy/test/tutils.py b/mitmproxy/test/tutils.py index d5b52bbe..09f2fcc0 100644 --- a/mitmproxy/test/tutils.py +++ b/mitmproxy/test/tutils.py @@ -1,13 +1,9 @@ from io import BytesIO -from mitmproxy.utils import data from mitmproxy.net import tcp from mitmproxy.net import http -test_data = data.Data(__name__).push("../../test/") - - def treader(bytes): """ Construct a tcp.Read object from bytes. diff --git a/test/conftest.py b/test/conftest.py index 27918cf9..7c7dec4a 100644 --- a/test/conftest.py +++ b/test/conftest.py @@ -1,6 +1,8 @@ import os import socket +from mitmproxy.utils import data + import pytest pytest_plugins = ('test.full_coverage_plugin',) @@ -33,3 +35,8 @@ skip_no_ipv6 = pytest.mark.skipif( no_ipv6, reason='Host has no IPv6 support' ) + + +@pytest.fixture() +def tdata(): + return data.Data(__name__) diff --git a/test/examples/test_examples.py b/test/examples/test_examples.py index 4c1631ce..255dbf71 100644 --- a/test/examples/test_examples.py +++ b/test/examples/test_examples.py @@ -6,27 +6,25 @@ from mitmproxy.net.http import Headers from ..mitmproxy import tservers -example_dir = tutils.test_data.push("../examples") - class TestScripts(tservers.MasterTest): - def test_add_header(self): + def test_add_header(self, tdata): with taddons.context() as tctx: - a = tctx.script(example_dir.path("simple/add_header.py")) + a = tctx.script(tdata.path("../examples/simple/add_header.py")) f = tflow.tflow(resp=tutils.tresp()) a.response(f) assert f.response.headers["newheader"] == "foo" - def test_custom_contentviews(self): + def test_custom_contentviews(self, tdata): with taddons.context() as tctx: - tctx.script(example_dir.path("simple/custom_contentview.py")) + tctx.script(tdata.path("../examples/simple/custom_contentview.py")) swapcase = contentviews.get("swapcase") _, fmt = swapcase(b"<html>Test!</html>") assert any(b'tEST!' in val[0][1] for val in fmt) - def test_iframe_injector(self): + def test_iframe_injector(self, tdata): with taddons.context() as tctx: - sc = tctx.script(example_dir.path("simple/modify_body_inject_iframe.py")) + sc = tctx.script(tdata.path("../examples/simple/modify_body_inject_iframe.py")) tctx.configure( sc, iframe = "http://example.org/evil_iframe" @@ -38,9 +36,9 @@ class TestScripts(tservers.MasterTest): content = f.response.content assert b'iframe' in content and b'evil_iframe' in content - def test_modify_form(self): + def test_modify_form(self, tdata): with taddons.context() as tctx: - sc = tctx.script(example_dir.path("simple/modify_form.py")) + sc = tctx.script(tdata.path("../examples/simple/modify_form.py")) form_header = Headers(content_type="application/x-www-form-urlencoded") f = tflow.tflow(req=tutils.treq(headers=form_header)) @@ -52,9 +50,9 @@ class TestScripts(tservers.MasterTest): sc.request(f) assert list(f.request.urlencoded_form.items()) == [("foo", "bar")] - def test_modify_querystring(self): + def test_modify_querystring(self, tdata): with taddons.context() as tctx: - sc = tctx.script(example_dir.path("simple/modify_querystring.py")) + sc = tctx.script(tdata.path("../examples/simple/modify_querystring.py")) f = tflow.tflow(req=tutils.treq(path="/search?q=term")) sc.request(f) @@ -64,23 +62,23 @@ class TestScripts(tservers.MasterTest): sc.request(f) assert f.request.query["mitmproxy"] == "rocks" - def test_redirect_requests(self): + def test_redirect_requests(self, tdata): with taddons.context() as tctx: - sc = tctx.script(example_dir.path("simple/redirect_requests.py")) + sc = tctx.script(tdata.path("../examples/simple/redirect_requests.py")) f = tflow.tflow(req=tutils.treq(host="example.org")) sc.request(f) assert f.request.host == "mitmproxy.org" - def test_send_reply_from_proxy(self): + def test_send_reply_from_proxy(self, tdata): with taddons.context() as tctx: - sc = tctx.script(example_dir.path("simple/send_reply_from_proxy.py")) + sc = tctx.script(tdata.path("../examples/simple/send_reply_from_proxy.py")) f = tflow.tflow(req=tutils.treq(host="example.com", port=80)) sc.request(f) assert f.response.content == b"Hello World" - def test_dns_spoofing(self): + def test_dns_spoofing(self, tdata): with taddons.context() as tctx: - sc = tctx.script(example_dir.path("complex/dns_spoofing.py")) + sc = tctx.script(tdata.path("../examples/complex/dns_spoofing.py")) original_host = "example.com" diff --git a/test/examples/test_har_dump.py b/test/examples/test_har_dump.py index 11cd5c29..7eb4f5f9 100644 --- a/test/examples/test_har_dump.py +++ b/test/examples/test_har_dump.py @@ -5,8 +5,6 @@ from mitmproxy.test import tutils from mitmproxy.test import taddons from mitmproxy.net.http import cookies -example_dir = tutils.test_data.push("../examples") - class TestHARDump: def flow(self, resp_content=b'message'): @@ -21,9 +19,9 @@ class TestHARDump: resp=tutils.tresp(content=resp_content, **times) ) - def test_simple(self, tmpdir): + def test_simple(self, tmpdir, tdata): with taddons.context() as tctx: - a = tctx.script(example_dir.path("complex/har_dump.py")) + a = tctx.script(tdata.path("../examples/complex/har_dump.py")) path = str(tmpdir.join("somefile")) tctx.configure(a, hardump=path) tctx.invoke(a, "response", self.flow()) @@ -32,9 +30,9 @@ class TestHARDump: har = json.load(inp) assert len(har["log"]["entries"]) == 1 - def test_base64(self, tmpdir): + def test_base64(self, tmpdir, tdata): with taddons.context() as tctx: - a = tctx.script(example_dir.path("complex/har_dump.py")) + a = tctx.script(tdata.path("../examples/complex/har_dump.py")) path = str(tmpdir.join("somefile")) tctx.configure(a, hardump=path) @@ -46,9 +44,9 @@ class TestHARDump: har = json.load(inp) assert har["log"]["entries"][0]["response"]["content"]["encoding"] == "base64" - def test_format_cookies(self): + def test_format_cookies(self, tdata): with taddons.context() as tctx: - a = tctx.script(example_dir.path("complex/har_dump.py")) + a = tctx.script(tdata.path("../examples/complex/har_dump.py")) CA = cookies.CookieAttrs @@ -65,9 +63,9 @@ class TestHARDump: f = a.format_cookies([("n", "v", CA([("expires", "Mon, 24-Aug-2037 00:00:00 GMT")]))])[0] assert f['expires'] - def test_binary(self, tmpdir): + def test_binary(self, tmpdir, tdata): with taddons.context() as tctx: - a = tctx.script(example_dir.path("complex/har_dump.py")) + a = tctx.script(tdata.path("../examples/complex/har_dump.py")) path = str(tmpdir.join("somefile")) tctx.configure(a, hardump=path) diff --git a/test/mitmproxy/addons/test_core.py b/test/mitmproxy/addons/test_core.py index 3c674b3f..59875c2b 100644 --- a/test/mitmproxy/addons/test_core.py +++ b/test/mitmproxy/addons/test_core.py @@ -3,7 +3,6 @@ from unittest import mock from mitmproxy.addons import core from mitmproxy.test import taddons from mitmproxy.test import tflow -from mitmproxy.test import tutils from mitmproxy import exceptions import pytest @@ -198,13 +197,13 @@ def test_validation_modes(m): tctx.configure(sa, mode = "reverse:") -def test_client_certs(): +def test_client_certs(tdata): sa = core.Core() with taddons.context() as tctx: # Folders should work. - tctx.configure(sa, client_certs = tutils.test_data.path("mitmproxy/data/clientcert")) + tctx.configure(sa, client_certs = tdata.path("mitmproxy/data/clientcert")) # Files, too. - tctx.configure(sa, client_certs = tutils.test_data.path("mitmproxy/data/clientcert/client.pem")) + tctx.configure(sa, client_certs = tdata.path("mitmproxy/data/clientcert/client.pem")) with pytest.raises(exceptions.OptionsError, match="certificate path does not exist"): tctx.configure(sa, client_certs = "invalid") diff --git a/test/mitmproxy/addons/test_cut.py b/test/mitmproxy/addons/test_cut.py index 35375393..5a733d07 100644 --- a/test/mitmproxy/addons/test_cut.py +++ b/test/mitmproxy/addons/test_cut.py @@ -5,13 +5,12 @@ from mitmproxy import exceptions from mitmproxy import certs from mitmproxy.test import taddons from mitmproxy.test import tflow -from mitmproxy.test import tutils import pytest import pyperclip from unittest import mock -def test_extract(): +def test_extract(tdata): tf = tflow.tflow(resp=True) tests = [ ["request.method", "GET"], @@ -54,7 +53,7 @@ def test_extract(): ret = cut.extract(spec, tf) assert spec and ret == expected - with open(tutils.test_data.path("mitmproxy/net/data/text_cert"), "rb") as f: + with open(tdata.path("mitmproxy/net/data/text_cert"), "rb") as f: d = f.read() c1 = certs.Cert.from_pem(d) tf.server_conn.cert = c1 diff --git a/test/mitmproxy/addons/test_onboarding.py b/test/mitmproxy/addons/test_onboarding.py index 0d99b1ff..a942062f 100644 --- a/test/mitmproxy/addons/test_onboarding.py +++ b/test/mitmproxy/addons/test_onboarding.py @@ -4,23 +4,21 @@ from mitmproxy.addons import onboarding from mitmproxy.test import taddons from .. import tservers -import asyncio -import tornado.platform.asyncio -asyncio.set_event_loop_policy(tornado.platform.asyncio.AnyThreadEventLoopPolicy()) - class TestApp(tservers.HTTPProxyTest): def addons(self): return [onboarding.Onboarding()] - def test_basic(self): + @pytest.mark.asyncio + async def test_basic(self): ob = onboarding.Onboarding() with taddons.context(ob) as tctx: tctx.configure(ob) assert self.app("/").status_code == 200 @pytest.mark.parametrize("ext", ["pem", "p12"]) - def test_cert(self, ext): + @pytest.mark.asyncio + async def test_cert(self, ext): ob = onboarding.Onboarding() with taddons.context(ob) as tctx: tctx.configure(ob) @@ -29,7 +27,8 @@ class TestApp(tservers.HTTPProxyTest): assert resp.content @pytest.mark.parametrize("ext", ["pem", "p12"]) - def test_head(self, ext): + @pytest.mark.asyncio + async def test_head(self, ext): ob = onboarding.Onboarding() with taddons.context(ob) as tctx: tctx.configure(ob) diff --git a/test/mitmproxy/addons/test_proxyauth.py b/test/mitmproxy/addons/test_proxyauth.py index 7816dd18..f12bf33f 100644 --- a/test/mitmproxy/addons/test_proxyauth.py +++ b/test/mitmproxy/addons/test_proxyauth.py @@ -7,7 +7,6 @@ from mitmproxy import exceptions from mitmproxy.addons import proxyauth from mitmproxy.test import taddons from mitmproxy.test import tflow -from mitmproxy.test import tutils class TestMkauth: @@ -73,7 +72,7 @@ class TestProxyAuth: assert resp.status_code == expected_status_code assert expected_header in resp.headers.keys() - def test_check(self): + def test_check(self, tdata): up = proxyauth.ProxyAuth() with taddons.context(up) as ctx: ctx.configure(up, proxyauth="any", mode="regular") @@ -102,7 +101,7 @@ class TestProxyAuth: ctx.configure( up, - proxyauth="@" + tutils.test_data.path( + proxyauth="@" + tdata.path( "mitmproxy/net/data/htpasswd" ) ) @@ -163,7 +162,7 @@ class TestProxyAuth: assert not f.response assert not f.request.headers.get("Authorization") - def test_configure(self): + def test_configure(self, tdata): up = proxyauth.ProxyAuth() with taddons.context(up) as ctx: with pytest.raises(exceptions.OptionsError): @@ -199,14 +198,14 @@ class TestProxyAuth: with pytest.raises(exceptions.OptionsError): ctx.configure( up, - proxyauth= "@" + tutils.test_data.path("mitmproxy/net/data/server.crt") + proxyauth= "@" + tdata.path("mitmproxy/net/data/server.crt") ) with pytest.raises(exceptions.OptionsError): ctx.configure(up, proxyauth="@nonexistent") ctx.configure( up, - proxyauth= "@" + tutils.test_data.path( + proxyauth= "@" + tdata.path( "mitmproxy/net/data/htpasswd" ) ) diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index 96e19841..c358f019 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -1,3 +1,4 @@ +import asyncio import os import sys import traceback @@ -9,14 +10,17 @@ from mitmproxy import exceptions from mitmproxy.addons import script from mitmproxy.test import taddons from mitmproxy.test import tflow -from mitmproxy.test import tutils + + +# We want this to be speedy for testing +script.ReloadInterval = 0.1 @pytest.mark.asyncio -async def test_load_script(): +async def test_load_script(tdata): with taddons.context() as tctx: ns = script.load_script( - tutils.test_data.path( + tdata.path( "mitmproxy/data/addonscripts/recorder/recorder.py" ) ) @@ -28,27 +32,27 @@ async def test_load_script(): assert await tctx.master.await_log("No such file or directory") script.load_script( - tutils.test_data.path( + tdata.path( "mitmproxy/data/addonscripts/recorder/error.py" ) ) assert await tctx.master.await_log("invalid syntax") -def test_load_fullname(): +def test_load_fullname(tdata): """ Test that loading two scripts at locations a/foo.py and b/foo.py works. This only succeeds if they get assigned different basenames. """ ns = script.load_script( - tutils.test_data.path( + tdata.path( "mitmproxy/data/addonscripts/addon.py" ) ) assert ns.addons ns2 = script.load_script( - tutils.test_data.path( + tdata.path( "mitmproxy/data/addonscripts/same_filename/addon.py" ) ) @@ -57,11 +61,11 @@ def test_load_fullname(): @pytest.mark.asyncio -async def test_script_print_stdout(): +async def test_script_print_stdout(tdata): with taddons.context() as tctx: with addonmanager.safecall(): ns = script.load_script( - tutils.test_data.path("mitmproxy/data/addonscripts/print.py") + tdata.path("mitmproxy/data/addonscripts/print.py") ) ns.load(addonmanager.Loader(tctx.master)) assert await tctx.master.await_log("stdoutprint") @@ -71,31 +75,33 @@ class TestScript: def test_notfound(self): with taddons.context(): with pytest.raises(exceptions.OptionsError): - script.Script("nonexistent") + script.Script("nonexistent", False) - def test_quotes_around_filename(self): + def test_quotes_around_filename(self, tdata): """ Test that a script specified as '"foo.py"' works to support the calling convention of mitmproxy 2.0, as e.g. used by Cuckoo Sandbox. """ - path = tutils.test_data.path("mitmproxy/data/addonscripts/recorder/recorder.py") + path = tdata.path("mitmproxy/data/addonscripts/recorder/recorder.py") s = script.Script( - '"{}"'.format(path) + '"{}"'.format(path), + False ) assert '"' not in s.fullpath - def test_simple(self): + @pytest.mark.asyncio + async def test_simple(self, tdata): with taddons.context() as tctx: sc = script.Script( - tutils.test_data.path( + tdata.path( "mitmproxy/data/addonscripts/recorder/recorder.py" - ) + ), + True, ) tctx.master.addons.add(sc) tctx.configure(sc) - sc.tick() - + await tctx.master.await_log("recorder running") rec = tctx.master.addons.get("recorder") assert rec.call_log[0][0:2] == ("recorder", "load") @@ -112,25 +118,29 @@ class TestScript: f = tmpdir.join("foo.py") f.ensure(file=True) f.write("\n") - sc = script.Script(str(f)) + sc = script.Script(str(f), True) tctx.configure(sc) - sc.tick() assert await tctx.master.await_log("Loading") - tctx.master.clear() - sc.last_load, sc.last_mtime = 0, 0 - sc.tick() - assert await tctx.master.await_log("Loading") + tctx.master.clear() + for i in range(20): + f.write("\n") + if tctx.master.has_log("Loading"): + break + await asyncio.sleep(0.1) + else: + raise AssertionError("No reload seen") @pytest.mark.asyncio - async def test_exception(self): + async def test_exception(self, tdata): with taddons.context() as tctx: sc = script.Script( - tutils.test_data.path("mitmproxy/data/addonscripts/error.py") + tdata.path("mitmproxy/data/addonscripts/error.py"), + True, ) tctx.master.addons.add(sc) + await tctx.master.await_log("error running") tctx.configure(sc) - sc.tick() f = tflow.tflow(resp=True) tctx.master.addons.trigger("request", f) @@ -138,16 +148,17 @@ class TestScript: assert await tctx.master.await_log("ValueError: Error!") assert await tctx.master.await_log("error.py") - def test_addon(self): + @pytest.mark.asyncio + async def test_addon(self, tdata): with taddons.context() as tctx: sc = script.Script( - tutils.test_data.path( + tdata.path( "mitmproxy/data/addonscripts/addon.py" - ) + ), + True ) tctx.master.addons.add(sc) - tctx.configure(sc) - sc.tick() + await tctx.master.await_log("addon running") assert sc.ns.event_log == [ 'scriptload', 'addonload', 'scriptconfigure', 'addonconfigure' ] @@ -173,8 +184,8 @@ class TestCutTraceback: class TestScriptLoader: @pytest.mark.asyncio - async def test_script_run(self): - rp = tutils.test_data.path( + async def test_script_run(self, tdata): + rp = tdata.path( "mitmproxy/data/addonscripts/recorder/recorder.py" ) sc = script.ScriptLoader() @@ -184,7 +195,6 @@ class TestScriptLoader: debug = [i.msg for i in tctx.master.logs if i.level == "debug"] assert debug == [ 'recorder load', 'recorder running', 'recorder configure', - 'recorder tick', 'recorder requestheaders', 'recorder request', 'recorder responseheaders', 'recorder response' ] @@ -196,7 +206,7 @@ class TestScriptLoader: sc.script_run([tflow.tflow(resp=True)], "/") assert await tctx.master.await_log("/: No such script") - def test_simple(self): + def test_simple(self, tdata): sc = script.ScriptLoader() with taddons.context(loadcore=False) as tctx: tctx.master.addons.add(sc) @@ -204,7 +214,7 @@ class TestScriptLoader: assert len(tctx.master.addons) == 1 tctx.master.options.update( scripts = [ - tutils.test_data.path( + tdata.path( "mitmproxy/data/addonscripts/recorder/recorder.py" ) ] @@ -224,25 +234,29 @@ class TestScriptLoader: scripts = ["one", "one"] ) - def test_script_deletion(self): - tdir = tutils.test_data.path("mitmproxy/data/addonscripts/") + @pytest.mark.asyncio + async def test_script_deletion(self, tdata): + tdir = tdata.path("mitmproxy/data/addonscripts/") with open(tdir + "/dummy.py", 'w') as f: f.write("\n") + with taddons.context() as tctx: sl = script.ScriptLoader() tctx.master.addons.add(sl) - tctx.configure(sl, scripts=[tutils.test_data.path("mitmproxy/data/addonscripts/dummy.py")]) + tctx.configure(sl, scripts=[tdata.path("mitmproxy/data/addonscripts/dummy.py")]) + await tctx.master.await_log("Loading") - os.remove(tutils.test_data.path("mitmproxy/data/addonscripts/dummy.py")) - tctx.invoke(sl, "tick") + os.remove(tdata.path("mitmproxy/data/addonscripts/dummy.py")) + + await tctx.master.await_log("Removing") assert not tctx.options.scripts assert not sl.addons - def test_load_err(self): + def test_load_err(self, tdata): sc = script.ScriptLoader() with taddons.context(sc, loadcore=False) as tctx: tctx.configure(sc, scripts=[ - tutils.test_data.path("mitmproxy/data/addonscripts/load_error.py") + tdata.path("mitmproxy/data/addonscripts/load_error.py") ]) try: tctx.invoke(sc, "tick") @@ -266,8 +280,8 @@ class TestScriptLoader: assert await tctx.master.await_log("NoneType") @pytest.mark.asyncio - async def test_order(self): - rec = tutils.test_data.path("mitmproxy/data/addonscripts/recorder") + async def test_order(self, tdata): + rec = tdata.path("mitmproxy/data/addonscripts/recorder") sc = script.ScriptLoader() sc.is_running = True with taddons.context() as tctx: @@ -286,17 +300,14 @@ class TestScriptLoader: 'a load', 'a running', 'a configure', - 'a tick', 'b load', 'b running', 'b configure', - 'b tick', 'c load', 'c running', 'c configure', - 'c tick', ] tctx.master.clear() @@ -317,7 +328,7 @@ class TestScriptLoader: 'b configure', ] - tctx.master.logs = [] + tctx.master.clear() tctx.configure( sc, scripts = [ @@ -325,9 +336,7 @@ class TestScriptLoader: "%s/a.py" % rec, ] ) - tctx.master.addons.invoke_addon(sc, "tick") - await tctx.master.await_log("a tick") - + await tctx.master.await_log("Loading") debug = [i.msg for i in tctx.master.logs if i.level == "debug"] assert debug == [ 'c done', @@ -336,6 +345,4 @@ class TestScriptLoader: 'e load', 'e running', 'e configure', - 'e tick', - 'a tick', ] diff --git a/test/mitmproxy/contentviews/image/test_image_parser.py b/test/mitmproxy/contentviews/image/test_image_parser.py index fdc72165..481d821f 100644 --- a/test/mitmproxy/contentviews/image/test_image_parser.py +++ b/test/mitmproxy/contentviews/image/test_image_parser.py @@ -1,7 +1,6 @@ import pytest from mitmproxy.contentviews.image import image_parser -from mitmproxy.test import tutils @pytest.mark.parametrize("filename, metadata", { @@ -71,8 +70,8 @@ from mitmproxy.test import tutils ('date:modify', '2012-07-11T14:04:52-07:00') ], }.items()) -def test_parse_png(filename, metadata): - with open(tutils.test_data.path(filename), "rb") as f: +def test_parse_png(filename, metadata, tdata): + with open(tdata.path(filename), "rb") as f: assert metadata == image_parser.parse_png(f.read()) @@ -101,8 +100,8 @@ def test_parse_png(filename, metadata): ('background', '0') ], }.items()) -def test_parse_gif(filename, metadata): - with open(tutils.test_data.path(filename), 'rb') as f: +def test_parse_gif(filename, metadata, tdata): + with open(tdata.path(filename), 'rb') as f: assert metadata == image_parser.parse_gif(f.read()) @@ -164,8 +163,8 @@ def test_parse_gif(filename, metadata): ('Size', '750 x 1055 px') ], }.items()) -def test_parse_jpeg(filename, metadata): - with open(tutils.test_data.path(filename), 'rb') as f: +def test_parse_jpeg(filename, metadata, tdata): + with open(tdata.path(filename), 'rb') as f: assert metadata == image_parser.parse_jpeg(f.read()) @@ -187,6 +186,6 @@ def test_parse_jpeg(filename, metadata): ) ] }.items()) -def test_ico(filename, metadata): - with open(tutils.test_data.path(filename), 'rb') as f: +def test_ico(filename, metadata, tdata): + with open(tdata.path(filename), 'rb') as f: assert metadata == image_parser.parse_ico(f.read()) diff --git a/test/mitmproxy/contentviews/image/test_view.py b/test/mitmproxy/contentviews/image/test_view.py index 6da5b1d0..84dffcc7 100644 --- a/test/mitmproxy/contentviews/image/test_view.py +++ b/test/mitmproxy/contentviews/image/test_view.py @@ -1,9 +1,8 @@ from mitmproxy.contentviews import image -from mitmproxy.test import tutils from .. import full_eval -def test_view_image(): +def test_view_image(tdata): v = full_eval(image.ViewImage()) for img in [ "mitmproxy/data/image.png", @@ -11,7 +10,7 @@ def test_view_image(): "mitmproxy/data/all.jpeg", "mitmproxy/data/image.ico", ]: - with open(tutils.test_data.path(img), "rb") as f: + with open(tdata.path(img), "rb") as f: viewname, lines = v(f.read()) assert img.split(".")[-1].upper() in viewname diff --git a/test/mitmproxy/contentviews/test_css.py b/test/mitmproxy/contentviews/test_css.py index 814f6e83..af1f776b 100644 --- a/test/mitmproxy/contentviews/test_css.py +++ b/test/mitmproxy/contentviews/test_css.py @@ -1,11 +1,8 @@ import pytest from mitmproxy.contentviews import css -from mitmproxy.test import tutils from . import full_eval -data = tutils.test_data.push("mitmproxy/contentviews/test_css_data/") - @pytest.mark.parametrize("filename", [ "animation-keyframe.css", @@ -19,8 +16,8 @@ data = tutils.test_data.push("mitmproxy/contentviews/test_css_data/") "selectors.css", "simple.css", ]) -def test_beautify(filename): - path = data.path(filename) +def test_beautify(filename, tdata): + path = tdata.path("mitmproxy/contentviews/test_css_data/" + filename) with open(path) as f: input = f.read() with open("-formatted.".join(path.rsplit(".", 1))) as f: diff --git a/test/mitmproxy/contentviews/test_javascript.py b/test/mitmproxy/contentviews/test_javascript.py index 23dd106e..8a102797 100644 --- a/test/mitmproxy/contentviews/test_javascript.py +++ b/test/mitmproxy/contentviews/test_javascript.py @@ -1,11 +1,8 @@ import pytest from mitmproxy.contentviews import javascript -from mitmproxy.test import tutils from . import full_eval -data = tutils.test_data.push("mitmproxy/contentviews/test_js_data/") - def test_view_javascript(): v = full_eval(javascript.ViewJavaScript()) @@ -22,8 +19,8 @@ def test_view_javascript(): @pytest.mark.parametrize("filename", [ "simple.js", ]) -def test_format_xml(filename): - path = data.path(filename) +def test_format_xml(filename, tdata): + path = tdata.path("mitmproxy/contentviews/test_js_data/" + filename) with open(path) as f: input = f.read() with open("-formatted.".join(path.rsplit(".", 1))) as f: diff --git a/test/mitmproxy/contentviews/test_protobuf.py b/test/mitmproxy/contentviews/test_protobuf.py index d0c4f42a..791045e7 100644 --- a/test/mitmproxy/contentviews/test_protobuf.py +++ b/test/mitmproxy/contentviews/test_protobuf.py @@ -1,15 +1,14 @@ import pytest from mitmproxy.contentviews import protobuf -from mitmproxy.test import tutils from . import full_eval -data = tutils.test_data.push("mitmproxy/contentviews/test_protobuf_data/") +datadir = "mitmproxy/contentviews/test_protobuf_data/" -def test_view_protobuf_request(): +def test_view_protobuf_request(tdata): v = full_eval(protobuf.ViewProtobuf()) - p = data.path("protobuf01") + p = tdata.path(datadir + "protobuf01") with open(p, "rb") as f: raw = f.read() @@ -21,8 +20,8 @@ def test_view_protobuf_request(): @pytest.mark.parametrize("filename", ["protobuf02", "protobuf03"]) -def test_format_pbuf(filename): - path = data.path(filename) +def test_format_pbuf(filename, tdata): + path = tdata.path(datadir + filename) with open(path, "rb") as f: input = f.read() with open(path + "-decoded") as f: diff --git a/test/mitmproxy/contentviews/test_wbxml.py b/test/mitmproxy/contentviews/test_wbxml.py index 09c770e7..441a7749 100644 --- a/test/mitmproxy/contentviews/test_wbxml.py +++ b/test/mitmproxy/contentviews/test_wbxml.py @@ -1,17 +1,16 @@ from mitmproxy.contentviews import wbxml -from mitmproxy.test import tutils from . import full_eval -data = tutils.test_data.push("mitmproxy/contentviews/test_wbxml_data/") +datadir = "mitmproxy/contentviews/test_wbxml_data/" -def test_wbxml(): +def test_wbxml(tdata): v = full_eval(wbxml.ViewWBXML()) assert v(b'\x03\x01\x6A\x00') == ('WBXML', [[('text', '<?xml version="1.0" ?>')]]) assert v(b'foo') is None - path = data.path("data.wbxml") # File taken from https://github.com/davidpshaw/PyWBXMLDecoder/tree/master/wbxml_samples + path = tdata.path(datadir + "data.wbxml") # File taken from https://github.com/davidpshaw/PyWBXMLDecoder/tree/master/wbxml_samples with open(path, 'rb') as f: input = f.read() with open("-formatted.".join(path.rsplit(".", 1))) as f: diff --git a/test/mitmproxy/contentviews/test_xml_html.py b/test/mitmproxy/contentviews/test_xml_html.py index 8148fd4c..82f85c15 100644 --- a/test/mitmproxy/contentviews/test_xml_html.py +++ b/test/mitmproxy/contentviews/test_xml_html.py @@ -1,20 +1,19 @@ import pytest from mitmproxy.contentviews import xml_html -from mitmproxy.test import tutils from . import full_eval -data = tutils.test_data.push("mitmproxy/contentviews/test_xml_html_data/") +datadir = "mitmproxy/contentviews/test_xml_html_data/" -def test_simple(): +def test_simple(tdata): v = full_eval(xml_html.ViewXmlHtml()) assert v(b"foo") == ('XML', [[('text', 'foo')]]) assert v(b"<html></html>") == ('HTML', [[('text', '<html></html>')]]) assert v(b"<>") == ('XML', [[('text', '<>')]]) assert v(b"<p") == ('XML', [[('text', '<p')]]) - with open(data.path("simple.html")) as f: + with open(tdata.path(datadir + "simple.html")) as f: input = f.read() tokens = xml_html.tokenize(input) assert str(next(tokens)) == "Tag(<!DOCTYPE html>)" @@ -27,8 +26,8 @@ def test_simple(): "inline.html", "test.html" ]) -def test_format_xml(filename): - path = data.path(filename) +def test_format_xml(filename, tdata): + path = tdata.path(datadir + filename) with open(path) as f: input = f.read() with open("-formatted.".join(path.rsplit(".", 1))) as f: diff --git a/test/mitmproxy/data/addonscripts/addon.py b/test/mitmproxy/data/addonscripts/addon.py index 8c834d82..c6b540d4 100644 --- a/test/mitmproxy/data/addonscripts/addon.py +++ b/test/mitmproxy/data/addonscripts/addon.py @@ -1,3 +1,4 @@ +from mitmproxy import ctx event_log = [] @@ -7,6 +8,7 @@ class Addon: return event_log def load(self, opts): + ctx.log.info("addon running") event_log.append("addonload") def configure(self, updated): diff --git a/test/mitmproxy/data/addonscripts/error.py b/test/mitmproxy/data/addonscripts/error.py index 4a3c370f..2f0c1755 100644 --- a/test/mitmproxy/data/addonscripts/error.py +++ b/test/mitmproxy/data/addonscripts/error.py @@ -1,6 +1,9 @@ -def mkerr(): - raise ValueError("Error!") +from mitmproxy import ctx + + +def running(): + ctx.log.info("error running") def request(flow): - mkerr() + raise ValueError("Error!") diff --git a/test/mitmproxy/data/addonscripts/shutdown.py b/test/mitmproxy/data/addonscripts/shutdown.py new file mode 100644 index 00000000..f4a8f55d --- /dev/null +++ b/test/mitmproxy/data/addonscripts/shutdown.py @@ -0,0 +1,5 @@ +from mitmproxy import ctx + + +def running(): + ctx.master.shutdown()
\ No newline at end of file diff --git a/test/mitmproxy/data/addonscripts/stream_modify.py b/test/mitmproxy/data/addonscripts/stream_modify.py index 4fbf45c2..4ebcc3e9 100644 --- a/test/mitmproxy/data/addonscripts/stream_modify.py +++ b/test/mitmproxy/data/addonscripts/stream_modify.py @@ -1,7 +1,13 @@ +from mitmproxy import ctx + def modify(chunks): for chunk in chunks: yield chunk.replace(b"foo", b"bar") +def running(): + ctx.log.info("stream_modify running") + + def responseheaders(flow): flow.response.stream = modify diff --git a/test/mitmproxy/io/test_compat.py b/test/mitmproxy/io/test_compat.py index 288de4fc..4c31e363 100644 --- a/test/mitmproxy/io/test_compat.py +++ b/test/mitmproxy/io/test_compat.py @@ -2,27 +2,26 @@ import pytest from mitmproxy import io from mitmproxy import exceptions -from mitmproxy.test import tutils -def test_load(): - with open(tutils.test_data.path("mitmproxy/data/dumpfile-011"), "rb") as f: +def test_load(tdata): + with open(tdata.path("mitmproxy/data/dumpfile-011"), "rb") as f: flow_reader = io.FlowReader(f) flows = list(flow_reader.stream()) assert len(flows) == 1 assert flows[0].request.url == "https://example.com/" -def test_load_018(): - with open(tutils.test_data.path("mitmproxy/data/dumpfile-018"), "rb") as f: +def test_load_018(tdata): + with open(tdata.path("mitmproxy/data/dumpfile-018"), "rb") as f: flow_reader = io.FlowReader(f) flows = list(flow_reader.stream()) assert len(flows) == 1 assert flows[0].request.url == "https://www.example.com/" -def test_cannot_convert(): - with open(tutils.test_data.path("mitmproxy/data/dumpfile-010"), "rb") as f: +def test_cannot_convert(tdata): + with open(tdata.path("mitmproxy/data/dumpfile-010"), "rb") as f: flow_reader = io.FlowReader(f) with pytest.raises(exceptions.FlowReadException): list(flow_reader.stream()) diff --git a/test/mitmproxy/net/test_tcp.py b/test/mitmproxy/net/test_tcp.py index e862d0ad..db8dff05 100644 --- a/test/mitmproxy/net/test_tcp.py +++ b/test/mitmproxy/net/test_tcp.py @@ -12,12 +12,15 @@ from OpenSSL import SSL from mitmproxy import certs from mitmproxy.net import tcp from mitmproxy import exceptions -from mitmproxy.test import tutils +from mitmproxy.utils import data from ...conftest import skip_no_ipv6 from . import tservers +cdata = data.Data(__name__) + + class EchoHandler(tcp.BaseHandler): sni = None @@ -172,7 +175,7 @@ class TestServerSSL(tservers.ServerTestBase): handler = EchoHandler ssl = dict( cipher_list="AES256-SHA", - chain_file=tutils.test_data.path("mitmproxy/net/data/server.crt") + chain_file=cdata.path("data/server.crt") ) def test_echo(self): @@ -209,14 +212,14 @@ class TestSSLv3Only(tservers.ServerTestBase): class TestInvalidTrustFile(tservers.ServerTestBase): - def test_invalid_trust_file_should_fail(self): + def test_invalid_trust_file_should_fail(self, tdata): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.TlsException): c.convert_to_tls( sni="example.mitmproxy.org", verify=SSL.VERIFY_PEER, - ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/generate.py") + ca_pemfile=tdata.path("mitmproxy/net/data/verificationcerts/generate.py") ) @@ -224,8 +227,8 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): handler = EchoHandler ssl = dict( - cert=tutils.test_data.path("mitmproxy/net/data/verificationcerts/self-signed.crt"), - key=tutils.test_data.path("mitmproxy/net/data/verificationcerts/self-signed.key") + cert=cdata.path("data/verificationcerts/self-signed.crt"), + key=cdata.path("data/verificationcerts/self-signed.key") ) def test_mode_default_should_pass(self): @@ -255,14 +258,14 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): c.wfile.flush() assert c.rfile.readline() == testval - def test_mode_strict_should_fail(self): + def test_mode_strict_should_fail(self, tdata): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.InvalidCertificateException): c.convert_to_tls( sni="example.mitmproxy.org", verify=SSL.VERIFY_PEER, - ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") + ca_pemfile=tdata.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") ) assert c.ssl_verification_error @@ -276,37 +279,37 @@ class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): handler = EchoHandler ssl = dict( - cert=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-leaf.crt"), - key=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-leaf.key") + cert=cdata.path("data/verificationcerts/trusted-leaf.crt"), + key=cdata.path("data/verificationcerts/trusted-leaf.key") ) - def test_should_fail_without_sni(self): + def test_should_fail_without_sni(self, tdata): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.TlsException): c.convert_to_tls( verify=SSL.VERIFY_PEER, - ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") + ca_pemfile=tdata.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") ) - def test_mode_none_should_pass_without_sni(self): + def test_mode_none_should_pass_without_sni(self, tdata): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): c.convert_to_tls( verify=SSL.VERIFY_NONE, - ca_path=tutils.test_data.path("mitmproxy/net/data/verificationcerts/") + ca_path=tdata.path("mitmproxy/net/data/verificationcerts/") ) assert "'no-hostname' doesn't match" in str(c.ssl_verification_error) - def test_should_fail(self): + def test_should_fail(self, tdata): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.InvalidCertificateException): c.convert_to_tls( sni="mitmproxy.org", verify=SSL.VERIFY_PEER, - ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") + ca_pemfile=tdata.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") ) assert c.ssl_verification_error @@ -315,17 +318,17 @@ class TestSSLUpstreamCertVerificationWValidCertChain(tservers.ServerTestBase): handler = EchoHandler ssl = dict( - cert=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-leaf.crt"), - key=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-leaf.key") + cert=cdata.path("data/verificationcerts/trusted-leaf.crt"), + key=cdata.path("data/verificationcerts/trusted-leaf.key") ) - def test_mode_strict_w_pemfile_should_pass(self): + def test_mode_strict_w_pemfile_should_pass(self, tdata): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): c.convert_to_tls( sni="example.mitmproxy.org", verify=SSL.VERIFY_PEER, - ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") + ca_pemfile=tdata.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") ) assert c.ssl_verification_error is None @@ -335,13 +338,13 @@ class TestSSLUpstreamCertVerificationWValidCertChain(tservers.ServerTestBase): c.wfile.flush() assert c.rfile.readline() == testval - def test_mode_strict_w_cadir_should_pass(self): + def test_mode_strict_w_cadir_should_pass(self, tdata): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): c.convert_to_tls( sni="example.mitmproxy.org", verify=SSL.VERIFY_PEER, - ca_path=tutils.test_data.path("mitmproxy/net/data/verificationcerts/") + ca_path=tdata.path("mitmproxy/net/data/verificationcerts/") ) assert c.ssl_verification_error is None @@ -369,18 +372,18 @@ class TestSSLClientCert(tservers.ServerTestBase): v3_only=False ) - def test_clientcert(self): + def test_clientcert(self, tdata): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): c.convert_to_tls( - cert=tutils.test_data.path("mitmproxy/net/data/clientcert/client.pem")) + cert=tdata.path("mitmproxy/net/data/clientcert/client.pem")) assert c.rfile.readline().strip() == b"1" - def test_clientcert_err(self): + def test_clientcert_err(self, tdata): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.TlsException): - c.convert_to_tls(cert=tutils.test_data.path("mitmproxy/net/data/clientcert/make")) + c.convert_to_tls(cert=tdata.path("mitmproxy/net/data/clientcert/make")) class TestSNI(tservers.ServerTestBase): @@ -597,7 +600,7 @@ class TestDHParams(tservers.ServerTestBase): handler = HangHandler ssl = dict( dhparams=certs.CertStore.load_dhparam( - tutils.test_data.path("mitmproxy/net/data/dhparam.pem"), + cdata.path("data/dhparam.pem"), ), cipher_list="DHE-RSA-AES256-SHA" ) diff --git a/test/mitmproxy/net/tservers.py b/test/mitmproxy/net/tservers.py index 22e195e3..fea4a73a 100644 --- a/test/mitmproxy/net/tservers.py +++ b/test/mitmproxy/net/tservers.py @@ -4,7 +4,9 @@ import io import OpenSSL from mitmproxy.net import tcp -from mitmproxy.test import tutils +from mitmproxy.utils import data + +cdata = data.Data(__name__) class _ServerThread(threading.Thread): @@ -47,10 +49,10 @@ class _TServer(tcp.TCPServer): if self.ssl is not None: cert = self.ssl.get( "cert", - tutils.test_data.path("mitmproxy/net/data/server.crt")) + cdata.path("data/server.crt")) raw_key = self.ssl.get( "key", - tutils.test_data.path("mitmproxy/net/data/server.key")) + cdata.path("data/server.key")) with open(raw_key) as f: raw_key = f.read() key = OpenSSL.crypto.load_privatekey(OpenSSL.crypto.FILETYPE_PEM, raw_key) diff --git a/test/mitmproxy/platform/test_pf.py b/test/mitmproxy/platform/test_pf.py index b048a697..9795a2db 100644 --- a/test/mitmproxy/platform/test_pf.py +++ b/test/mitmproxy/platform/test_pf.py @@ -1,16 +1,15 @@ import sys import pytest from mitmproxy.platform import pf -from mitmproxy.test import tutils class TestLookup: - def test_simple(self): + def test_simple(self, tdata): if sys.platform == "freebsd10": - p = tutils.test_data.path("mitmproxy/data/pf02") + p = tdata.path("mitmproxy/data/pf02") else: - p = tutils.test_data.path("mitmproxy/data/pf01") + p = tdata.path("mitmproxy/data/pf01") with open(p, "rb") as f: d = f.read() diff --git a/test/mitmproxy/proxy/test_config.py b/test/mitmproxy/proxy/test_config.py index 60a0deb5..a2fd8f37 100644 --- a/test/mitmproxy/proxy/test_config.py +++ b/test/mitmproxy/proxy/test_config.py @@ -3,7 +3,6 @@ import pytest from mitmproxy import options from mitmproxy import exceptions from mitmproxy.proxy.config import ProxyConfig -from mitmproxy.test import tutils class TestProxyConfig: @@ -13,8 +12,8 @@ class TestProxyConfig: with pytest.raises(exceptions.OptionsError, match="parent directory does not exist"): ProxyConfig(opts) - def test_invalid_certificate(self): + def test_invalid_certificate(self, tdata): opts = options.Options() - opts.certs = [tutils.test_data.path("mitmproxy/data/dumpfile-011")] + opts.certs = [tdata.path("mitmproxy/data/dumpfile-011")] with pytest.raises(exceptions.OptionsError, match="Invalid certificate format"): ProxyConfig(opts) diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index bf24e28b..936414ab 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -16,13 +16,16 @@ from mitmproxy.net import socks from mitmproxy.net import tcp from mitmproxy.net.http import http1 from mitmproxy.proxy.config import HostMatcher -from mitmproxy.test import tutils +from mitmproxy.utils import data from pathod import pathoc from pathod import pathod from .. import tservers from ...conftest import skip_appveyor +cdata = data.Data(__name__) + + class CommonMixin: def test_large(self): @@ -256,11 +259,15 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin): resp = p.request("get:'http://foo':h':foo'='bar'") assert resp.status_code == 400 - def test_stream_modify(self): + @pytest.mark.asyncio + async def test_stream_modify(self, tdata): s = script.Script( - tutils.test_data.path("mitmproxy/data/addonscripts/stream_modify.py") + tdata.path("mitmproxy/data/addonscripts/stream_modify.py"), + False, ) self.set_addons(s) + await self.master.await_log("stream_modify running") + d = self.pathod('200:b"foo"') assert d.content == b"bar" @@ -284,19 +291,19 @@ class TestHTTPS(tservers.HTTPProxyTest, CommonMixin, TcpMixin): ssl = True ssloptions = pathod.SSLOptions(request_client_cert=True) - def test_clientcert_file(self): + def test_clientcert_file(self, tdata): try: self.options.client_certs = os.path.join( - tutils.test_data.path("mitmproxy/data/clientcert"), "client.pem") + tdata.path("mitmproxy/data/clientcert"), "client.pem") f = self.pathod("304") assert f.status_code == 304 assert self.server.last_log()["request"]["clientcert"]["keyinfo"] finally: self.options.client_certs = None - def test_clientcert_dir(self): + def test_clientcert_dir(self, tdata): try: - self.options.client_certs = tutils.test_data.path("mitmproxy/data/clientcert") + self.options.client_certs = tdata.path("mitmproxy/data/clientcert") f = self.pathod("304") assert f.status_code == 304 assert self.server.last_log()["request"]["clientcert"]["keyinfo"] @@ -335,7 +342,7 @@ class TestHTTPSUpstreamServerVerificationWTrustedCert(tservers.HTTPProxyTest): ssloptions = pathod.SSLOptions( cn=b"example.mitmproxy.org", certs=[ - ("example.mitmproxy.org", tutils.test_data.path("mitmproxy/data/servercert/trusted-leaf.pem")) + ("example.mitmproxy.org", cdata.path("../data/servercert/trusted-leaf.pem")) ] ) @@ -344,21 +351,21 @@ class TestHTTPSUpstreamServerVerificationWTrustedCert(tservers.HTTPProxyTest): with p.connect(): return p.request("get:/p/242") - def test_verification_w_cadir(self): + def test_verification_w_cadir(self, tdata): self.options.update( ssl_insecure=False, - ssl_verify_upstream_trusted_cadir=tutils.test_data.path( + ssl_verify_upstream_trusted_cadir=tdata.path( "mitmproxy/data/servercert/" ), ssl_verify_upstream_trusted_ca=None, ) assert self._request().status_code == 242 - def test_verification_w_pemfile(self): + def test_verification_w_pemfile(self, tdata): self.options.update( ssl_insecure=False, ssl_verify_upstream_trusted_cadir=None, - ssl_verify_upstream_trusted_ca=tutils.test_data.path( + ssl_verify_upstream_trusted_ca=tdata.path( "mitmproxy/data/servercert/trusted-root.pem" ), ) @@ -374,7 +381,7 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest): ssloptions = pathod.SSLOptions( cn=b"example.mitmproxy.org", certs=[ - ("example.mitmproxy.org", tutils.test_data.path("mitmproxy/data/servercert/self-signed.pem")) + ("example.mitmproxy.org", cdata.path("../data/servercert/self-signed.pem")) ]) def _request(self): @@ -385,8 +392,8 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest): @classmethod def get_options(cls): opts = super().get_options() - opts.ssl_verify_upstream_trusted_ca = tutils.test_data.path( - "mitmproxy/data/servercert/trusted-root.pem" + opts.ssl_verify_upstream_trusted_ca = cdata.path( + "../data/servercert/trusted-root.pem" ) return opts @@ -413,7 +420,7 @@ class TestHTTPSNoCommonName(tservers.HTTPProxyTest): ssl = True ssloptions = pathod.SSLOptions( certs=[ - ("*", tutils.test_data.path("mitmproxy/data/no_common_name.pem")) + ("*", cdata.path("../data/no_common_name.pem")) ] ) @@ -562,9 +569,10 @@ class TestHttps2Http(tservers.ReverseProxyTest): class TestTransparent(tservers.TransparentProxyTest, CommonMixin, TcpMixin): ssl = False - def test_tcp_stream_modify(self): + def test_tcp_stream_modify(self, tdata): s = script.Script( - tutils.test_data.path("mitmproxy/data/addonscripts/tcp_stream_modify.py") + tdata.path("mitmproxy/data/addonscripts/tcp_stream_modify.py"), + False, ) self.set_addons(s) self._tcpproxy_on() @@ -822,7 +830,7 @@ class TestServerConnect(tservers.HTTPProxyTest): self.set_addons(AFakeResponse()) assert self.pathod("200").status_code == 200 asyncio.sleep(0.1) - assert not self.proxy.tmaster._has_log("serverconnect") + assert not self.proxy.tmaster.has_log("serverconnect") class AKillRequest: @@ -1064,7 +1072,7 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest): class AddUpstreamCertsToClientChainMixin: ssl = True - servercert = tutils.test_data.path("mitmproxy/data/servercert/trusted-root.pem") + servercert = cdata.path("../data/servercert/trusted-root.pem") ssloptions = pathod.SSLOptions( cn=b"example.mitmproxy.org", certs=[ diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index 876093f7..3ec58760 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -1,7 +1,6 @@ import pytest from mitmproxy.test import tflow -from mitmproxy.test import tutils from mitmproxy.test import taddons from mitmproxy import controller @@ -17,10 +16,10 @@ class Thing: class TestConcurrent(tservers.MasterTest): - def test_concurrent(self): + def test_concurrent(self, tdata): with taddons.context() as tctx: sc = tctx.script( - tutils.test_data.path( + tdata.path( "mitmproxy/data/addonscripts/concurrent_decorator.py" ) ) @@ -34,19 +33,19 @@ class TestConcurrent(tservers.MasterTest): raise ValueError("Script never acked") @pytest.mark.asyncio - async def test_concurrent_err(self): + async def test_concurrent_err(self, tdata): with taddons.context() as tctx: tctx.script( - tutils.test_data.path( + tdata.path( "mitmproxy/data/addonscripts/concurrent_decorator_err.py" ) ) assert await tctx.master.await_log("decorator not supported") - def test_concurrent_class(self): + def test_concurrent_class(self, tdata): with taddons.context() as tctx: sc = tctx.script( - tutils.test_data.path( + tdata.path( "mitmproxy/data/addonscripts/concurrent_decorator_class.py" ) ) diff --git a/test/mitmproxy/test_certs.py b/test/mitmproxy/test_certs.py index dcc185c0..12d3dc96 100644 --- a/test/mitmproxy/test_certs.py +++ b/test/mitmproxy/test_certs.py @@ -1,6 +1,5 @@ import os from mitmproxy import certs -from mitmproxy.test import tutils # class TestDNTree: # def test_simple(self): @@ -138,14 +137,14 @@ class TestDummyCert: class TestCert: - def test_simple(self): - with open(tutils.test_data.path("mitmproxy/net/data/text_cert"), "rb") as f: + def test_simple(self, tdata): + with open(tdata.path("mitmproxy/net/data/text_cert"), "rb") as f: d = f.read() c1 = certs.Cert.from_pem(d) assert c1.cn == b"google.com" assert len(c1.altnames) == 436 - with open(tutils.test_data.path("mitmproxy/net/data/text_cert_2"), "rb") as f: + with open(tdata.path("mitmproxy/net/data/text_cert_2"), "rb") as f: d = f.read() c2 = certs.Cert.from_pem(d) assert c2.cn == b"www.inode.co.nz" @@ -162,21 +161,21 @@ class TestCert: assert c1 != c2 - def test_err_broken_sans(self): - with open(tutils.test_data.path("mitmproxy/net/data/text_cert_weird1"), "rb") as f: + def test_err_broken_sans(self, tdata): + with open(tdata.path("mitmproxy/net/data/text_cert_weird1"), "rb") as f: d = f.read() c = certs.Cert.from_pem(d) # This breaks unless we ignore a decoding error. assert c.altnames is not None - def test_der(self): - with open(tutils.test_data.path("mitmproxy/net/data/dercert"), "rb") as f: + def test_der(self, tdata): + with open(tdata.path("mitmproxy/net/data/dercert"), "rb") as f: d = f.read() s = certs.Cert.from_der(d) assert s.cn - def test_state(self): - with open(tutils.test_data.path("mitmproxy/net/data/text_cert"), "rb") as f: + def test_state(self, tdata): + with open(tdata.path("mitmproxy/net/data/text_cert"), "rb") as f: d = f.read() c = certs.Cert.from_pem(d) diff --git a/test/mitmproxy/test_connections.py b/test/mitmproxy/test_connections.py index acd98eb8..7c371c1e 100644 --- a/test/mitmproxy/test_connections.py +++ b/test/mitmproxy/test_connections.py @@ -10,7 +10,6 @@ from mitmproxy import exceptions from mitmproxy.net import tcp from mitmproxy.net.http import http1 from mitmproxy.test import tflow -from mitmproxy.test import tutils from .net import tservers from pathod import test @@ -185,7 +184,7 @@ class TestClientConnectionTLS: None, "example.com" ]) - def test_tls_with_sni(self, sni): + def test_tls_with_sni(self, sni, tdata): address = ('127.0.0.1', 0) sock = socket.socket(socket.AF_INET, socket.SOCK_STREAM) sock.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) @@ -206,8 +205,8 @@ class TestClientConnectionTLS: connection, client_address = sock.accept() c = connections.ClientConnection(connection, client_address, None) - cert = tutils.test_data.path("mitmproxy/net/data/server.crt") - with open(tutils.test_data.path("mitmproxy/net/data/server.key")) as f: + cert = tdata.path("mitmproxy/net/data/server.crt") + with open(tdata.path("mitmproxy/net/data/server.key")) as f: raw_key = f.read() key = OpenSSL.crypto.load_privatekey( OpenSSL.crypto.FILETYPE_PEM, @@ -230,10 +229,12 @@ class TestServerConnectionTLS(tservers.ServerTestBase): @pytest.mark.parametrize("client_certs", [ None, - tutils.test_data.path("mitmproxy/data/clientcert"), - tutils.test_data.path("mitmproxy/data/clientcert/client.pem"), + "mitmproxy/data/clientcert", + "mitmproxy/data/clientcert/client.pem", ]) - def test_tls(self, client_certs): + def test_tls(self, client_certs, tdata): + if client_certs: + client_certs = tdata.path(client_certs) c = connections.ServerConnection(("127.0.0.1", self.port)) c.connect() c.establish_tls(client_certs=client_certs) diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 75d4cdf0..00086c4b 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -8,7 +8,6 @@ from mitmproxy import options from mitmproxy.proxy import ProxyConfig from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler from mitmproxy.proxy import config -from mitmproxy.test import tutils from ..conftest import skip_windows @@ -42,10 +41,10 @@ class TestProcessProxyOptions: def test_simple(self): assert self.p() - def test_certs(self): + def test_certs(self, tdata): self.assert_noerr( "--cert", - tutils.test_data.path("mitmproxy/data/testkey.pem")) + tdata.path("mitmproxy/data/testkey.pem")) with pytest.raises(Exception, match="does not exist"): self.p("--cert", "nonexistent") diff --git a/test/mitmproxy/test_taddons.py b/test/mitmproxy/test_taddons.py index 39dd65c2..5266e038 100644 --- a/test/mitmproxy/test_taddons.py +++ b/test/mitmproxy/test_taddons.py @@ -3,17 +3,16 @@ import io import pytest from mitmproxy.test import taddons -from mitmproxy.test import tutils from mitmproxy import ctx @pytest.mark.asyncio async def test_recordingmaster(): with taddons.context() as tctx: - assert not tctx.master._has_log("nonexistent") + assert not tctx.master.has_log("nonexistent") assert not tctx.master.has_event("nonexistent") ctx.log.error("foo") - assert not tctx.master._has_log("foo", level="debug") + assert not tctx.master.has_log("foo", level="debug") assert await tctx.master.await_log("foo", level="error") @@ -27,10 +26,10 @@ async def test_dumplog(): assert s.getvalue() -def test_load_script(): +def test_load_script(tdata): with taddons.context() as tctx: s = tctx.script( - tutils.test_data.path( + tdata.path( "mitmproxy/data/addonscripts/recorder/recorder.py" ) ) diff --git a/test/mitmproxy/test_types.py b/test/mitmproxy/test_types.py index 72492fa9..b4a643ad 100644 --- a/test/mitmproxy/test_types.py +++ b/test/mitmproxy/test_types.py @@ -3,7 +3,6 @@ import os import typing import contextlib -from mitmproxy.test import tutils import mitmproxy.exceptions import mitmproxy.types from mitmproxy.test import taddons @@ -64,7 +63,7 @@ def test_int(): b.parse(tctx.master.commands, int, "foo") -def test_path(): +def test_path(tdata): with taddons.context() as tctx: b = mitmproxy.types._PathType() assert b.parse(tctx.master.commands, mitmproxy.types.Path, "/foo") == "/foo" @@ -80,7 +79,7 @@ def test_path(): ret.append(s) return ret - cd = os.path.normpath(tutils.test_data.path("mitmproxy/completion")) + cd = os.path.normpath(tdata.path("mitmproxy/completion")) assert normPathOpts(cd, cd) == ['/aaa', '/aab', '/aac', '/bbb/'] assert normPathOpts(cd, os.path.join(cd, "a")) == ['/aaa', '/aab', '/aac'] with chdir(cd): diff --git a/test/mitmproxy/tools/test_main.py b/test/mitmproxy/tools/test_main.py index 57544276..f75f07ef 100644 --- a/test/mitmproxy/tools/test_main.py +++ b/test/mitmproxy/tools/test_main.py @@ -1,19 +1,23 @@ -import pytest +import asyncio from mitmproxy.tools import main -from mitmproxy import ctx -@pytest.mark.asyncio -async def test_mitmweb(event_loop): +shutdown_script = "mitmproxy/data/addonscripts/shutdown.py" + + +def test_mitmweb(event_loop, tdata): + asyncio.set_event_loop(event_loop) main.mitmweb([ "--no-web-open-browser", + "-s", tdata.path(shutdown_script), "-q", "-p", "0", ]) - await ctx.master._shutdown() -@pytest.mark.asyncio -async def test_mitmdump(): - main.mitmdump(["-q", "-p", "0"]) - await ctx.master._shutdown() +def test_mitmdump(event_loop, tdata): + asyncio.set_event_loop(event_loop) + main.mitmdump([ + "-s", tdata.path(shutdown_script), + "-q", "-p", "0", + ]) diff --git a/test/mitmproxy/tools/web/test_static_viewer.py b/test/mitmproxy/tools/web/test_static_viewer.py index dfc45bc2..c044dee8 100644 --- a/test/mitmproxy/tools/web/test_static_viewer.py +++ b/test/mitmproxy/tools/web/test_static_viewer.py @@ -1,5 +1,6 @@ import json from unittest import mock +import pytest from mitmproxy.test import taddons from mitmproxy.test import tflow @@ -57,7 +58,8 @@ def test_save_flows_content(ctx, tmpdir): assert p.join('response/content/Auto.json').check(file=1) -def test_static_viewer(tmpdir): +@pytest.mark.asyncio +async def test_static_viewer(tmpdir): s = static_viewer.StaticViewer() rf = readfile.ReadFile() sa = save.Save() diff --git a/test/pathod/test_pathoc.py b/test/pathod/test_pathoc.py index 297b54d4..85c46fff 100644 --- a/test/pathod/test_pathoc.py +++ b/test/pathod/test_pathoc.py @@ -61,10 +61,10 @@ class TestDaemonSSL(PathocTestDaemon): def test_showssl(self): assert "certificate chain" in self.tval(["get:/p/200"], showssl=True) - def test_clientcert(self): + def test_clientcert(self, tdata): self.tval( ["get:/p/200"], - clientcert=tutils.test_data.path("pathod/data/clientcert/client.pem"), + clientcert=tdata.path("pathod/data/clientcert/client.pem"), ) log = self.d.log() assert log[0]["request"]["clientcert"]["keyinfo"] diff --git a/test/pathod/test_pathoc_cmdline.py b/test/pathod/test_pathoc_cmdline.py index 7bc76ace..fecebe3d 100644 --- a/test/pathod/test_pathoc_cmdline.py +++ b/test/pathod/test_pathoc_cmdline.py @@ -4,11 +4,9 @@ from unittest import mock from pathod import pathoc_cmdline as cmdline -from mitmproxy.test import tutils - @mock.patch("argparse.ArgumentParser.error") -def test_pathoc(perror): +def test_pathoc(perror, tdata): assert cmdline.args_pathoc(["pathoc", "foo.com", "get:/"]) s = io.StringIO() with pytest.raises(SystemExit): @@ -53,7 +51,7 @@ def test_pathoc(perror): [ "pathoc", "foo.com:8888", - tutils.test_data.path("pathod/data/request") + tdata.path("pathod/data/request") ] ) assert len(list(a.requests)) == 1 diff --git a/test/pathod/test_pathod.py b/test/pathod/test_pathod.py index d6522cb6..246bff3b 100644 --- a/test/pathod/test_pathod.py +++ b/test/pathod/test_pathod.py @@ -5,11 +5,14 @@ import pytest from pathod import pathod from mitmproxy.net import tcp from mitmproxy import exceptions -from mitmproxy.test import tutils +from mitmproxy.utils import data from . import tservers +cdata = data.Data(__name__) + + class TestPathod: def test_logging(self): @@ -57,7 +60,7 @@ class TestNotAfterConnect(tservers.DaemonTests): class TestCustomCert(tservers.DaemonTests): ssl = True ssloptions = dict( - certs=[("*", tutils.test_data.path("pathod/data/testkey.pem"))], + certs=[("*", cdata.path("data/testkey.pem"))], ) def test_connect(self): diff --git a/test/pathod/test_pathod_cmdline.py b/test/pathod/test_pathod_cmdline.py index 34baf491..37427179 100644 --- a/test/pathod/test_pathod_cmdline.py +++ b/test/pathod/test_pathod_cmdline.py @@ -2,8 +2,6 @@ from unittest import mock from pathod import pathod_cmdline as cmdline -from mitmproxy.test import tutils - def test_parse_anchor_spec(): assert cmdline.parse_anchor_spec("foo=200") == ("foo", "200") @@ -11,14 +9,14 @@ def test_parse_anchor_spec(): @mock.patch("argparse.ArgumentParser.error") -def test_pathod(perror): +def test_pathod(perror, tdata): assert cmdline.args_pathod(["pathod"]) a = cmdline.args_pathod( [ "pathod", "--cert", - tutils.test_data.path("pathod/data/testkey.pem") + tdata.path("pathod/data/testkey.pem") ] ) assert a.ssl_certs @@ -46,7 +44,7 @@ def test_pathod(perror): [ "pathod", "-a", - "foo=" + tutils.test_data.path("pathod/data/response") + "foo=" + tdata.path("pathod/data/response") ] ) assert a.anchors diff --git a/test/pathod/tservers.py b/test/pathod/tservers.py index a7c92964..8fad133a 100644 --- a/test/pathod/tservers.py +++ b/test/pathod/tservers.py @@ -8,7 +8,7 @@ import urllib from mitmproxy.net import tcp -from mitmproxy.test import tutils +from mitmproxy.utils import data from pathod import language from pathod import pathoc @@ -17,6 +17,9 @@ from pathod import test from pathod.pathod import CA_CERT_NAME +cdata = data.Data(__name__) + + def treader(bytes): """ Construct a tcp.Read object from bytes. @@ -41,7 +44,7 @@ class DaemonTests: opts["confdir"] = cls.confdir so = pathod.SSLOptions(**opts) cls.d = test.Daemon( - staticdir=tutils.test_data.path("pathod/data"), + staticdir=cdata.path("data"), anchors=[ (re.compile("/anchor/.*"), "202:da") ], |