Add a light-weight custom event system, use it for keepserving

This patch implements the lightweight event system I propose in #2144, adds a
custom event "processing_complete" that is triggered after file read, client
replay and server replay, and introduces a KeepServing addon to handle this for
mitmdump.

13 files changed, 58 insertions, 46 deletions

diff --git a/test/mitmproxy/addons/test_check_alpn.py b/test/mitmproxy/addons/test_check_alpn.py
index 2dc0c835..ffaf6cff 100644
--- a/test/mitmproxy/addons/test_check_alpn.py
+++ b/test/mitmproxy/addons/test_check_alpn.py
@@ -12,7 +12,7 @@ class TestCheckALPN:
         with taddons.context() as tctx:
             a = check_alpn.CheckALPN()
             tctx.configure(a)
-            assert not any(msg in m for l, m in tctx.master.event_log)
+            assert not any(msg in m for l, m in tctx.master.logs)
 
     def test_check_no_alpn(self, disable_alpn):
         msg = 'ALPN support missing'
@@ -20,4 +20,4 @@ class TestCheckALPN:
         with taddons.context() as tctx:
             a = check_alpn.CheckALPN()
             tctx.configure(a)
-            assert any(msg in m for l, m in tctx.master.event_log)
+            assert any(msg in m for l, m in tctx.master.logs)
diff --git a/test/mitmproxy/addons/test_check_ca.py b/test/mitmproxy/addons/test_check_ca.py
index fc64621c..859b6d8d 100644
--- a/test/mitmproxy/addons/test_check_ca.py
+++ b/test/mitmproxy/addons/test_check_ca.py
@@ -16,4 +16,4 @@ class TestCheckCA:
             tctx.master.server.config.certstore.default_ca.has_expired = mock.MagicMock(return_value=expired)
             a = check_ca.CheckCA()
             tctx.configure(a)
-            assert any(msg in m for l, m in tctx.master.event_log) is expired
+            assert any(msg in m for l, m in tctx.master.logs) is expired
diff --git a/test/mitmproxy/addons/test_clientplayback.py b/test/mitmproxy/addons/test_clientplayback.py
index c22b3589..f71662f0 100644
--- a/test/mitmproxy/addons/test_clientplayback.py
+++ b/test/mitmproxy/addons/test_clientplayback.py
@@ -23,7 +23,7 @@ class MockThread():
 class TestClientPlayback:
     def test_playback(self):
         cp = clientplayback.ClientPlayback()
-        with taddons.context():
+        with taddons.context() as tctx:
             assert cp.count() == 0
             f = tflow.tflow(resp=True)
             cp.load([f])
@@ -35,17 +35,14 @@ class TestClientPlayback:
                 assert rp.called
                 assert cp.current_thread
 
-            cp.keepserving = False
             cp.flows = None
             cp.current_thread = None
-            with mock.patch("mitmproxy.master.Master.shutdown") as sd:
-                cp.tick()
-                assert sd.called
+            cp.tick()
+            assert tctx.master.has_event("processing_complete")
 
             cp.current_thread = MockThread()
-            with mock.patch("mitmproxy.master.Master.shutdown") as sd:
-                cp.tick()
-                assert cp.current_thread is None
+            cp.tick()
+            assert cp.current_thread is None
 
     def test_configure(self, tmpdir):
         cp = clientplayback.ClientPlayback()
diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py
index 47374617..23299431 100644
--- a/test/mitmproxy/addons/test_dumper.py
+++ b/test/mitmproxy/addons/test_dumper.py
@@ -151,7 +151,7 @@ class TestContentView:
         with taddons.context(options=options.Options()) as ctx:
             ctx.configure(d, flow_detail=4, verbosity=3)
             d.response(tflow.tflow())
-            assert "Content viewer failed" in ctx.master.event_log[0][1]
+            assert "Content viewer failed" in ctx.master.logs[0][1]
 
 
 def test_tcp():
diff --git a/test/mitmproxy/addons/test_keepserving.py b/test/mitmproxy/addons/test_keepserving.py
new file mode 100644
index 00000000..70f7e1e6
--- /dev/null
+++ b/test/mitmproxy/addons/test_keepserving.py
@@ -0,0 +1,10 @@
+from mitmproxy.addons import keepserving
+from mitmproxy.test import taddons
+
+
+def test_keepserving():
+    ks = keepserving.KeepServing()
+
+    with taddons.context() as tctx:
+        ks.event_processing_complete()
+        assert tctx.master.should_exit.is_set()
diff --git a/test/mitmproxy/addons/test_readfile.py b/test/mitmproxy/addons/test_readfile.py
index c0cf97ae..b30c147b 100644
--- a/test/mitmproxy/addons/test_readfile.py
+++ b/test/mitmproxy/addons/test_readfile.py
@@ -32,13 +32,13 @@ def test_configure(mck, tmpdir):
     with taddons.context() as tctx:
         tf = str(tmpdir.join("tfile"))
         write_data(tf)
-        tctx.configure(rf, rfile=str(tf), keepserving=False)
+        tctx.configure(rf, rfile=str(tf))
         assert not mck.called
         rf.running()
         assert mck.called
 
         write_data(tf, corrupt=True)
-        tctx.configure(rf, rfile=str(tf), keepserving=False)
+        tctx.configure(rf, rfile=str(tf))
         with pytest.raises(exceptions.OptionsError):
             rf.running()
 
@@ -51,7 +51,7 @@ def test_corruption(mck, tmpdir):
         with pytest.raises(exceptions.FlowReadException):
             rf.load_flows_file("nonexistent")
         assert not mck.called
-        assert len(tctx.master.event_log) == 1
+        assert len(tctx.master.logs) == 1
 
         tfc = str(tmpdir.join("tfile"))
         write_data(tfc, corrupt=True)
@@ -59,4 +59,4 @@ def test_corruption(mck, tmpdir):
         with pytest.raises(exceptions.FlowReadException):
             rf.load_flows_file(tfc)
         assert mck.called
-        assert len(tctx.master.event_log) == 2
+        assert len(tctx.master.logs) == 2
diff --git a/test/mitmproxy/addons/test_readstdin.py b/test/mitmproxy/addons/test_readstdin.py
index bbef81fc..76b01f4f 100644
--- a/test/mitmproxy/addons/test_readstdin.py
+++ b/test/mitmproxy/addons/test_readstdin.py
@@ -26,12 +26,6 @@ def gen_data(corrupt=False):
     return tf
 
 
-def test_configure(tmpdir):
-    rf = readstdin.ReadStdin()
-    with taddons.context() as tctx:
-        tctx.configure(rf, keepserving=False)
-
-
 class mStdin:
     def __init__(self, d):
         self.buffer = d
@@ -49,11 +43,11 @@ def test_read(m, tmpdir):
         assert m.called
 
         rf.running(stdin=mStdin(None))
-        assert tctx.master.event_log
+        assert tctx.master.logs
         tctx.master.clear()
 
         m.reset_mock()
         assert not m.called
         rf.running(stdin=mStdin(gen_data(corrupt=True)))
         assert m.called
-        assert tctx.master.event_log
+        assert tctx.master.logs
diff --git a/test/mitmproxy/addons/test_replace.py b/test/mitmproxy/addons/test_replace.py
index 7d590b35..9002afb5 100644
--- a/test/mitmproxy/addons/test_replace.py
+++ b/test/mitmproxy/addons/test_replace.py
@@ -97,6 +97,6 @@ class TestReplaceFile:
             tmpfile.remove()
             f = tflow.tflow()
             f.request.content = b"foo"
-            assert not tctx.master.event_log
+            assert not tctx.master.logs
             r.request(f)
-            assert tctx.master.event_log
+            assert tctx.master.logs
diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py
index 4c1b2e43..ad3c9a1a 100644
--- a/test/mitmproxy/addons/test_script.py
+++ b/test/mitmproxy/addons/test_script.py
@@ -22,14 +22,14 @@ def test_scriptenv():
     with taddons.context() as tctx:
         with script.scriptenv("path", []):
             raise SystemExit
-        assert tctx.master.event_log[0][0] == "error"
-        assert "exited" in tctx.master.event_log[0][1]
+        assert tctx.master.logs[0][0] == "error"
+        assert "exited" in tctx.master.logs[0][1]
 
         tctx.master.clear()
         with script.scriptenv("path", []):
             raise ValueError("fooo")
-        assert tctx.master.event_log[0][0] == "error"
-        assert "foo" in tctx.master.event_log[0][1]
+        assert tctx.master.logs[0][0] == "error"
+        assert "foo" in tctx.master.logs[0][1]
 
 
 class Called:
@@ -135,7 +135,7 @@ class TestScript:
                 f.write(".")
                 sc.tick()
                 time.sleep(0.1)
-                if tctx.master.event_log:
+                if tctx.master.logs:
                     return
             raise AssertionError("Change event not detected.")
 
@@ -147,11 +147,11 @@ class TestScript:
             sc.start(tctx.options)
             f = tflow.tflow(resp=True)
             sc.request(f)
-            assert tctx.master.event_log[0][0] == "error"
-            assert len(tctx.master.event_log[0][1].splitlines()) == 6
-            assert re.search(r'addonscripts[\\/]error.py", line \d+, in request', tctx.master.event_log[0][1])
-            assert re.search(r'addonscripts[\\/]error.py", line \d+, in mkerr', tctx.master.event_log[0][1])
-            assert tctx.master.event_log[0][1].endswith("ValueError: Error!\n")
+            assert tctx.master.logs[0][0] == "error"
+            assert len(tctx.master.logs[0][1].splitlines()) == 6
+            assert re.search(r'addonscripts[\\/]error.py", line \d+, in request', tctx.master.logs[0][1])
+            assert re.search(r'addonscripts[\\/]error.py", line \d+, in mkerr', tctx.master.logs[0][1])
+            assert tctx.master.logs[0][1].endswith("ValueError: Error!\n")
 
     def test_addon(self):
         with taddons.context() as tctx:
@@ -256,7 +256,7 @@ class TestScriptLoader:
                     "%s %s" % (rec, "c"),
                 ]
             )
-            debug = [(i[0], i[1]) for i in tctx.master.event_log if i[0] == "debug"]
+            debug = [(i[0], i[1]) for i in tctx.master.logs if i[0] == "debug"]
             assert debug == [
                 ('debug', 'a start'),
                 ('debug', 'a configure'),
@@ -270,7 +270,7 @@ class TestScriptLoader:
                 ('debug', 'c configure'),
                 ('debug', 'c running'),
             ]
-            tctx.master.event_log = []
+            tctx.master.logs = []
             tctx.configure(
                 sc,
                 scripts = [
@@ -279,11 +279,11 @@ class TestScriptLoader:
                     "%s %s" % (rec, "b"),
                 ]
             )
-            debug = [(i[0], i[1]) for i in tctx.master.event_log if i[0] == "debug"]
+            debug = [(i[0], i[1]) for i in tctx.master.logs if i[0] == "debug"]
             # No events, only order has changed
             assert debug == []
 
-            tctx.master.event_log = []
+            tctx.master.logs = []
             tctx.configure(
                 sc,
                 scripts = [
@@ -291,7 +291,7 @@ class TestScriptLoader:
                     "%s %s" % (rec, "a"),
                 ]
             )
-            debug = [(i[0], i[1]) for i in tctx.master.event_log if i[0] == "debug"]
+            debug = [(i[0], i[1]) for i in tctx.master.logs if i[0] == "debug"]
             assert debug == [
                 ('debug', 'c done'),
                 ('debug', 'b done'),
diff --git a/test/mitmproxy/addons/test_serverplayback.py b/test/mitmproxy/addons/test_serverplayback.py
index 54e4d281..02642c35 100644
--- a/test/mitmproxy/addons/test_serverplayback.py
+++ b/test/mitmproxy/addons/test_serverplayback.py
@@ -34,7 +34,7 @@ def test_tick():
         s.final_flow = tflow.tflow()
         s.final_flow.live = False
         s.tick()
-        assert tctx.master.should_exit.is_set()
+        assert tctx.master.has_event("processing_complete")
 
 
 def test_server_playback():
@@ -315,7 +315,6 @@ def test_server_playback_full():
         tctx.configure(
             s,
             refresh_server_playback = True,
-            keepserving=False
         )
 
         f = tflow.tflow()
diff --git a/test/mitmproxy/addons/test_termstatus.py b/test/mitmproxy/addons/test_termstatus.py
index 01c14814..7becc857 100644
--- a/test/mitmproxy/addons/test_termstatus.py
+++ b/test/mitmproxy/addons/test_termstatus.py
@@ -6,7 +6,7 @@ def test_configure():
     ts = termstatus.TermStatus()
     with taddons.context() as ctx:
         ts.running()
-        assert not ctx.master.event_log
+        assert not ctx.master.logs
         ctx.configure(ts, server=True)
         ts.running()
-        assert ctx.master.event_log
+        assert ctx.master.logs
diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py
index a9b6f0c4..206482e2 100644
--- a/test/mitmproxy/script/test_concurrent.py
+++ b/test/mitmproxy/script/test_concurrent.py
@@ -43,7 +43,7 @@ class TestConcurrent(tservers.MasterTest):
                 )
             )
             sc.start(tctx.options)
-            assert "decorator not supported" in tctx.master.event_log[0][1]
+            assert "decorator not supported" in tctx.master.logs[0][1]
 
     def test_concurrent_class(self):
             with taddons.context() as tctx:
diff --git a/test/mitmproxy/test_addonmanager.py b/test/mitmproxy/test_addonmanager.py
index ef34371f..e7be25b8 100644
--- a/test/mitmproxy/test_addonmanager.py
+++ b/test/mitmproxy/test_addonmanager.py
@@ -11,6 +11,7 @@ class TAddon:
     def __init__(self, name):
         self.name = name
         self.tick = True
+        self.custom_called = False
 
     def __repr__(self):
         return "Addon(%s)" % self.name
@@ -18,11 +19,17 @@ class TAddon:
     def done(self):
         pass
 
+    def event_custom(self):
+        self.custom_called = True
+
 
 def test_simple():
     o = options.Options()
     m = master.Master(o, proxy.DummyServer(o))
     a = addonmanager.AddonManager(m)
+    with pytest.raises(exceptions.AddonError):
+        a.invoke_addon(TAddon("one"), "done")
+
     a.add(TAddon("one"))
     assert a.get("one")
     assert not a.get("two")
@@ -33,3 +40,8 @@ def test_simple():
     a.trigger("done")
     with pytest.raises(exceptions.AddonError):
         a.trigger("tick")
+
+    ta = TAddon("one")
+    a.add(ta)
+    a.trigger("custom")
+    assert ta.custom_called