Merge pull request #2152 from cortesi/readfile

Extract flow reading into addons

18 files changed, 194 insertions, 93 deletions

diff --git a/test/mitmproxy/addons/test_check_alpn.py b/test/mitmproxy/addons/test_check_alpn.py
index 2dc0c835..ffaf6cff 100644
--- a/test/mitmproxy/addons/test_check_alpn.py
+++ b/test/mitmproxy/addons/test_check_alpn.py
@@ -12,7 +12,7 @@ class TestCheckALPN:
         with taddons.context() as tctx:
             a = check_alpn.CheckALPN()
             tctx.configure(a)
-            assert not any(msg in m for l, m in tctx.master.event_log)
+            assert not any(msg in m for l, m in tctx.master.logs)
 
     def test_check_no_alpn(self, disable_alpn):
         msg = 'ALPN support missing'
@@ -20,4 +20,4 @@ class TestCheckALPN:
         with taddons.context() as tctx:
             a = check_alpn.CheckALPN()
             tctx.configure(a)
-            assert any(msg in m for l, m in tctx.master.event_log)
+            assert any(msg in m for l, m in tctx.master.logs)
diff --git a/test/mitmproxy/addons/test_check_ca.py b/test/mitmproxy/addons/test_check_ca.py
index fc64621c..859b6d8d 100644
--- a/test/mitmproxy/addons/test_check_ca.py
+++ b/test/mitmproxy/addons/test_check_ca.py
@@ -16,4 +16,4 @@ class TestCheckCA:
             tctx.master.server.config.certstore.default_ca.has_expired = mock.MagicMock(return_value=expired)
             a = check_ca.CheckCA()
             tctx.configure(a)
-            assert any(msg in m for l, m in tctx.master.event_log) is expired
+            assert any(msg in m for l, m in tctx.master.logs) is expired
diff --git a/test/mitmproxy/addons/test_clientplayback.py b/test/mitmproxy/addons/test_clientplayback.py
index c22b3589..f71662f0 100644
--- a/test/mitmproxy/addons/test_clientplayback.py
+++ b/test/mitmproxy/addons/test_clientplayback.py
@@ -23,7 +23,7 @@ class MockThread():
 class TestClientPlayback:
     def test_playback(self):
         cp = clientplayback.ClientPlayback()
-        with taddons.context():
+        with taddons.context() as tctx:
             assert cp.count() == 0
             f = tflow.tflow(resp=True)
             cp.load([f])
@@ -35,17 +35,14 @@ class TestClientPlayback:
                 assert rp.called
                 assert cp.current_thread
 
-            cp.keepserving = False
             cp.flows = None
             cp.current_thread = None
-            with mock.patch("mitmproxy.master.Master.shutdown") as sd:
-                cp.tick()
-                assert sd.called
+            cp.tick()
+            assert tctx.master.has_event("processing_complete")
 
             cp.current_thread = MockThread()
-            with mock.patch("mitmproxy.master.Master.shutdown") as sd:
-                cp.tick()
-                assert cp.current_thread is None
+            cp.tick()
+            assert cp.current_thread is None
 
     def test_configure(self, tmpdir):
         cp = clientplayback.ClientPlayback()
diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py
index 47374617..23299431 100644
--- a/test/mitmproxy/addons/test_dumper.py
+++ b/test/mitmproxy/addons/test_dumper.py
@@ -151,7 +151,7 @@ class TestContentView:
         with taddons.context(options=options.Options()) as ctx:
             ctx.configure(d, flow_detail=4, verbosity=3)
             d.response(tflow.tflow())
-            assert "Content viewer failed" in ctx.master.event_log[0][1]
+            assert "Content viewer failed" in ctx.master.logs[0][1]
 
 
 def test_tcp():
diff --git a/test/mitmproxy/addons/test_keepserving.py b/test/mitmproxy/addons/test_keepserving.py
new file mode 100644
index 00000000..70f7e1e6
--- /dev/null
+++ b/test/mitmproxy/addons/test_keepserving.py
@@ -0,0 +1,10 @@
+from mitmproxy.addons import keepserving
+from mitmproxy.test import taddons
+
+
+def test_keepserving():
+    ks = keepserving.KeepServing()
+
+    with taddons.context() as tctx:
+        ks.event_processing_complete()
+        assert tctx.master.should_exit.is_set()
diff --git a/test/mitmproxy/addons/test_readfile.py b/test/mitmproxy/addons/test_readfile.py
new file mode 100644
index 00000000..b30c147b
--- /dev/null
+++ b/test/mitmproxy/addons/test_readfile.py
@@ -0,0 +1,62 @@
+from mitmproxy.addons import readfile
+from mitmproxy.test import taddons
+from mitmproxy.test import tflow
+from mitmproxy import io
+from mitmproxy import exceptions
+from unittest import mock
+
+import pytest
+
+
+def write_data(path, corrupt=False):
+    with open(path, "wb") as tf:
+        w = io.FlowWriter(tf)
+        for i in range(3):
+            f = tflow.tflow(resp=True)
+            w.add(f)
+        for i in range(3):
+            f = tflow.tflow(err=True)
+            w.add(f)
+        f = tflow.ttcpflow()
+        w.add(f)
+        f = tflow.ttcpflow(err=True)
+        w.add(f)
+        if corrupt:
+            tf.write(b"flibble")
+
+
+@mock.patch('mitmproxy.master.Master.load_flow')
+def test_configure(mck, tmpdir):
+
+    rf = readfile.ReadFile()
+    with taddons.context() as tctx:
+        tf = str(tmpdir.join("tfile"))
+        write_data(tf)
+        tctx.configure(rf, rfile=str(tf))
+        assert not mck.called
+        rf.running()
+        assert mck.called
+
+        write_data(tf, corrupt=True)
+        tctx.configure(rf, rfile=str(tf))
+        with pytest.raises(exceptions.OptionsError):
+            rf.running()
+
+
+@mock.patch('mitmproxy.master.Master.load_flow')
+def test_corruption(mck, tmpdir):
+
+    rf = readfile.ReadFile()
+    with taddons.context() as tctx:
+        with pytest.raises(exceptions.FlowReadException):
+            rf.load_flows_file("nonexistent")
+        assert not mck.called
+        assert len(tctx.master.logs) == 1
+
+        tfc = str(tmpdir.join("tfile"))
+        write_data(tfc, corrupt=True)
+
+        with pytest.raises(exceptions.FlowReadException):
+            rf.load_flows_file(tfc)
+        assert mck.called
+        assert len(tctx.master.logs) == 2
diff --git a/test/mitmproxy/addons/test_readstdin.py b/test/mitmproxy/addons/test_readstdin.py
new file mode 100644
index 00000000..76b01f4f
--- /dev/null
+++ b/test/mitmproxy/addons/test_readstdin.py
@@ -0,0 +1,53 @@
+
+import io
+from mitmproxy.addons import readstdin
+from mitmproxy.test import taddons
+from mitmproxy.test import tflow
+import mitmproxy.io
+from unittest import mock
+
+
+def gen_data(corrupt=False):
+    tf = io.BytesIO()
+    w = mitmproxy.io.FlowWriter(tf)
+    for i in range(3):
+        f = tflow.tflow(resp=True)
+        w.add(f)
+    for i in range(3):
+        f = tflow.tflow(err=True)
+        w.add(f)
+    f = tflow.ttcpflow()
+    w.add(f)
+    f = tflow.ttcpflow(err=True)
+    w.add(f)
+    if corrupt:
+        tf.write(b"flibble")
+    tf.seek(0)
+    return tf
+
+
+class mStdin:
+    def __init__(self, d):
+        self.buffer = d
+
+    def isatty(self):
+        return False
+
+
+@mock.patch('mitmproxy.master.Master.load_flow')
+def test_read(m, tmpdir):
+    rf = readstdin.ReadStdin()
+    with taddons.context() as tctx:
+        assert not m.called
+        rf.running(stdin=mStdin(gen_data()))
+        assert m.called
+
+        rf.running(stdin=mStdin(None))
+        assert tctx.master.logs
+        tctx.master.clear()
+
+        m.reset_mock()
+        assert not m.called
+        rf.running(stdin=mStdin(gen_data(corrupt=True)))
+        assert m.called
+        assert tctx.master.logs
diff --git a/test/mitmproxy/addons/test_replace.py b/test/mitmproxy/addons/test_replace.py
index 7d590b35..9002afb5 100644
--- a/test/mitmproxy/addons/test_replace.py
+++ b/test/mitmproxy/addons/test_replace.py
@@ -97,6 +97,6 @@ class TestReplaceFile:
             tmpfile.remove()
             f = tflow.tflow()
             f.request.content = b"foo"
-            assert not tctx.master.event_log
+            assert not tctx.master.logs
             r.request(f)
-            assert tctx.master.event_log
+            assert tctx.master.logs
diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py
index 4c1b2e43..ad3c9a1a 100644
--- a/test/mitmproxy/addons/test_script.py
+++ b/test/mitmproxy/addons/test_script.py
@@ -22,14 +22,14 @@ def test_scriptenv():
     with taddons.context() as tctx:
         with script.scriptenv("path", []):
             raise SystemExit
-        assert tctx.master.event_log[0][0] == "error"
-        assert "exited" in tctx.master.event_log[0][1]
+        assert tctx.master.logs[0][0] == "error"
+        assert "exited" in tctx.master.logs[0][1]
 
         tctx.master.clear()
         with script.scriptenv("path", []):
             raise ValueError("fooo")
-        assert tctx.master.event_log[0][0] == "error"
-        assert "foo" in tctx.master.event_log[0][1]
+        assert tctx.master.logs[0][0] == "error"
+        assert "foo" in tctx.master.logs[0][1]
 
 
 class Called:
@@ -135,7 +135,7 @@ class TestScript:
                 f.write(".")
                 sc.tick()
                 time.sleep(0.1)
-                if tctx.master.event_log:
+                if tctx.master.logs:
                     return
             raise AssertionError("Change event not detected.")
 
@@ -147,11 +147,11 @@ class TestScript:
             sc.start(tctx.options)
             f = tflow.tflow(resp=True)
             sc.request(f)
-            assert tctx.master.event_log[0][0] == "error"
-            assert len(tctx.master.event_log[0][1].splitlines()) == 6
-            assert re.search(r'addonscripts[\\/]error.py", line \d+, in request', tctx.master.event_log[0][1])
-            assert re.search(r'addonscripts[\\/]error.py", line \d+, in mkerr', tctx.master.event_log[0][1])
-            assert tctx.master.event_log[0][1].endswith("ValueError: Error!\n")
+            assert tctx.master.logs[0][0] == "error"
+            assert len(tctx.master.logs[0][1].splitlines()) == 6
+            assert re.search(r'addonscripts[\\/]error.py", line \d+, in request', tctx.master.logs[0][1])
+            assert re.search(r'addonscripts[\\/]error.py", line \d+, in mkerr', tctx.master.logs[0][1])
+            assert tctx.master.logs[0][1].endswith("ValueError: Error!\n")
 
     def test_addon(self):
         with taddons.context() as tctx:
@@ -256,7 +256,7 @@ class TestScriptLoader:
                     "%s %s" % (rec, "c"),
                 ]
             )
-            debug = [(i[0], i[1]) for i in tctx.master.event_log if i[0] == "debug"]
+            debug = [(i[0], i[1]) for i in tctx.master.logs if i[0] == "debug"]
             assert debug == [
                 ('debug', 'a start'),
                 ('debug', 'a configure'),
@@ -270,7 +270,7 @@ class TestScriptLoader:
                 ('debug', 'c configure'),
                 ('debug', 'c running'),
             ]
-            tctx.master.event_log = []
+            tctx.master.logs = []
             tctx.configure(
                 sc,
                 scripts = [
@@ -279,11 +279,11 @@ class TestScriptLoader:
                     "%s %s" % (rec, "b"),
                 ]
             )
-            debug = [(i[0], i[1]) for i in tctx.master.event_log if i[0] == "debug"]
+            debug = [(i[0], i[1]) for i in tctx.master.logs if i[0] == "debug"]
             # No events, only order has changed
             assert debug == []
 
-            tctx.master.event_log = []
+            tctx.master.logs = []
             tctx.configure(
                 sc,
                 scripts = [
@@ -291,7 +291,7 @@ class TestScriptLoader:
                     "%s %s" % (rec, "a"),
                 ]
             )
-            debug = [(i[0], i[1]) for i in tctx.master.event_log if i[0] == "debug"]
+            debug = [(i[0], i[1]) for i in tctx.master.logs if i[0] == "debug"]
             assert debug == [
                 ('debug', 'c done'),
                 ('debug', 'b done'),
diff --git a/test/mitmproxy/addons/test_serverplayback.py b/test/mitmproxy/addons/test_serverplayback.py
index 54e4d281..02642c35 100644
--- a/test/mitmproxy/addons/test_serverplayback.py
+++ b/test/mitmproxy/addons/test_serverplayback.py
@@ -34,7 +34,7 @@ def test_tick():
         s.final_flow = tflow.tflow()
         s.final_flow.live = False
         s.tick()
-        assert tctx.master.should_exit.is_set()
+        assert tctx.master.has_event("processing_complete")
 
 
 def test_server_playback():
@@ -315,7 +315,6 @@ def test_server_playback_full():
         tctx.configure(
             s,
             refresh_server_playback = True,
-            keepserving=False
         )
 
         f = tflow.tflow()
diff --git a/test/mitmproxy/addons/test_termstatus.py b/test/mitmproxy/addons/test_termstatus.py
index 01c14814..7becc857 100644
--- a/test/mitmproxy/addons/test_termstatus.py
+++ b/test/mitmproxy/addons/test_termstatus.py
@@ -6,7 +6,7 @@ def test_configure():
     ts = termstatus.TermStatus()
     with taddons.context() as ctx:
         ts.running()
-        assert not ctx.master.event_log
+        assert not ctx.master.logs
         ctx.configure(ts, server=True)
         ts.running()
-        assert ctx.master.event_log
+        assert ctx.master.logs
diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py
index a9b6f0c4..206482e2 100644
--- a/test/mitmproxy/script/test_concurrent.py
+++ b/test/mitmproxy/script/test_concurrent.py
@@ -43,7 +43,7 @@ class TestConcurrent(tservers.MasterTest):
                 )
             )
             sc.start(tctx.options)
-            assert "decorator not supported" in tctx.master.event_log[0][1]
+            assert "decorator not supported" in tctx.master.logs[0][1]
 
     def test_concurrent_class(self):
             with taddons.context() as tctx:
diff --git a/test/mitmproxy/test_addonmanager.py b/test/mitmproxy/test_addonmanager.py
index 3e5f71c6..e7be25b8 100644
--- a/test/mitmproxy/test_addonmanager.py
+++ b/test/mitmproxy/test_addonmanager.py
@@ -11,6 +11,7 @@ class TAddon:
     def __init__(self, name):
         self.name = name
         self.tick = True
+        self.custom_called = False
 
     def __repr__(self):
         return "Addon(%s)" % self.name
@@ -18,11 +19,17 @@ class TAddon:
     def done(self):
         pass
 
+    def event_custom(self):
+        self.custom_called = True
+
 
 def test_simple():
     o = options.Options()
     m = master.Master(o, proxy.DummyServer(o))
     a = addonmanager.AddonManager(m)
+    with pytest.raises(exceptions.AddonError):
+        a.invoke_addon(TAddon("one"), "done")
+
     a.add(TAddon("one"))
     assert a.get("one")
     assert not a.get("two")
@@ -30,6 +37,11 @@ def test_simple():
     assert not a.chain
 
     a.add(TAddon("one"))
-    a("done")
+    a.trigger("done")
     with pytest.raises(exceptions.AddonError):
-        a("tick")
+        a.trigger("tick")
+
+    ta = TAddon("one")
+    a.add(ta)
+    a.trigger("custom")
+    assert ta.custom_called
diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py
index f20e0c8c..56692364 100644
--- a/test/mitmproxy/test_examples.py
+++ b/test/mitmproxy/test_examples.py
@@ -145,7 +145,7 @@ class TestHARDump:
         path = str(tmpdir.join("somefile"))
 
         m, sc = tscript("complex/har_dump.py", shlex.quote(path))
-        m.addons.invoke(m, "response", self.flow())
+        m.addons.trigger("response", self.flow())
         m.addons.remove(sc)
 
         with open(path, "r") as inp:
@@ -156,7 +156,9 @@ class TestHARDump:
         path = str(tmpdir.join("somefile"))
 
         m, sc = tscript("complex/har_dump.py", shlex.quote(path))
-        m.addons.invoke(m, "response", self.flow(resp_content=b"foo" + b"\xFF" * 10))
+        m.addons.trigger(
+            "response", self.flow(resp_content=b"foo" + b"\xFF" * 10)
+        )
         m.addons.remove(sc)
 
         with open(path, "r") as inp:
@@ -194,7 +196,7 @@ class TestHARDump:
         path = str(tmpdir.join("somefile"))
 
         m, sc = tscript("complex/har_dump.py", shlex.quote(path))
-        m.addons.invoke(m, "response", f)
+        m.addons.trigger("response", f)
         m.addons.remove(sc)
 
         with open(path, "r") as inp:
diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py
index 7a37fbd1..4f87a6ae 100644
--- a/test/mitmproxy/test_flow.py
+++ b/test/mitmproxy/test_flow.py
@@ -3,12 +3,13 @@ import pytest
 
 from mitmproxy.test import tflow
 import mitmproxy.io
-from mitmproxy import flowfilter, options
+from mitmproxy import flowfilter
+from mitmproxy import options
+from mitmproxy.proxy import config
 from mitmproxy.contrib import tnetstring
 from mitmproxy.exceptions import FlowReadException
 from mitmproxy import flow
 from mitmproxy import http
-from mitmproxy.proxy import ProxyConfig
 from mitmproxy.proxy.server import DummyServer
 from mitmproxy import master
 from . import tservers
@@ -16,23 +17,6 @@ from . import tservers
 
 class TestSerialize:
 
-    def _treader(self):
-        sio = io.BytesIO()
-        w = mitmproxy.io.FlowWriter(sio)
-        for i in range(3):
-            f = tflow.tflow(resp=True)
-            w.add(f)
-        for i in range(3):
-            f = tflow.tflow(err=True)
-            w.add(f)
-        f = tflow.ttcpflow()
-        w.add(f)
-        f = tflow.ttcpflow(err=True)
-        w.add(f)
-
-        sio.seek(0)
-        return mitmproxy.io.FlowReader(sio)
-
     def test_roundtrip(self):
         sio = io.BytesIO()
         f = tflow.tflow()
@@ -51,26 +35,6 @@ class TestSerialize:
         assert f2.request == f.request
         assert f2.marked
 
-    def test_load_flows(self):
-        r = self._treader()
-        s = tservers.TestState()
-        fm = master.Master(None, DummyServer())
-        fm.addons.add(s)
-        fm.load_flows(r)
-        assert len(s.flows) == 6
-
-    def test_load_flows_reverse(self):
-        r = self._treader()
-        s = tservers.TestState()
-        opts = options.Options(
-            mode="reverse:https://use-this-domain"
-        )
-        conf = ProxyConfig(opts)
-        fm = master.Master(opts, DummyServer(conf))
-        fm.addons.add(s)
-        fm.load_flows(r)
-        assert s.flows[0].request.host == "use-this-domain"
-
     def test_filter(self):
         sio = io.BytesIO()
         flt = flowfilter.parse("~c 200")
@@ -122,6 +86,17 @@ class TestSerialize:
 
 
 class TestFlowMaster:
+    def test_load_flow_reverse(self):
+        s = tservers.TestState()
+        opts = options.Options(
+            mode="reverse:https://use-this-domain"
+        )
+        conf = config.ProxyConfig(opts)
+        fm = master.Master(opts, DummyServer(conf))
+        fm.addons.add(s)
+        f = tflow.tflow(resp=True)
+        fm.load_flow(f)
+        assert s.flows[0].request.host == "use-this-domain"
 
     def test_replay(self):
         fm = master.Master(None, DummyServer())
diff --git a/test/mitmproxy/tools/console/test_master.py b/test/mitmproxy/tools/console/test_master.py
index 6c716ad1..45908450 100644
--- a/test/mitmproxy/tools/console/test_master.py
+++ b/test/mitmproxy/tools/console/test_master.py
@@ -5,6 +5,7 @@ from mitmproxy import proxy
 from mitmproxy import options
 from mitmproxy.tools.console import common
 from ... import tservers
+import urwid
 
 
 def test_format_keyvals():
@@ -35,7 +36,10 @@ class TestMaster(tservers.MasterTest):
     def test_basic(self):
         m = self.mkmaster()
         for i in (1, 2, 3):
-            self.dummy_cycle(m, 1, b"")
+            try:
+                self.dummy_cycle(m, 1, b"")
+            except urwid.ExitMainLoop:
+                pass
             assert len(m.view) == i
 
     def test_run_script_once(self):
diff --git a/test/mitmproxy/tools/test_dump.py b/test/mitmproxy/tools/test_dump.py
index a15bf583..624bf08f 100644
--- a/test/mitmproxy/tools/test_dump.py
+++ b/test/mitmproxy/tools/test_dump.py
@@ -2,7 +2,6 @@ import pytest
 from unittest import mock
 
 from mitmproxy import proxy
-from mitmproxy import exceptions
 from mitmproxy import log
 from mitmproxy import controller
 from mitmproxy import options
@@ -17,18 +16,6 @@ class TestDumpMaster(tservers.MasterTest):
         m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=False, with_dumper=False)
         return m
 
-    def test_read(self, tmpdir):
-        p = str(tmpdir.join("read"))
-        self.flowfile(p)
-        self.dummy_cycle(
-            self.mkmaster(None, rfile=p),
-            1, b"",
-        )
-        with pytest.raises(exceptions.OptionsError):
-            self.mkmaster(None, rfile="/nonexistent")
-        with pytest.raises(exceptions.OptionsError):
-            self.mkmaster(None, rfile="test_dump.py")
-
     def test_has_error(self):
         m = self.mkmaster(None)
         ent = log.LogEntry("foo", "error")
diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py
index c47411ee..0f34e37e 100644
--- a/test/mitmproxy/tservers.py
+++ b/test/mitmproxy/tservers.py
@@ -80,7 +80,7 @@ class TestMaster(master.Master):
         self.addons.add(self.state)
         self.addons.add(*addons)
         self.addons.configure_all(self.options, self.options.keys())
-        self.addons.invoke_all_with_context("running")
+        self.addons.trigger("running")
 
     def clear_log(self):
         self.tlog = []