Extract console dump functionality into an addon

This removes all the code that deals with printing flows to screen from dump.py
into a self-contained addon.

- This fixes a bug - by moving dumping into an
addon, we now dump flows AFTER addon transformation, so we can see the changes
made.
- We get dumping "for free" in other places by simply adding the addon. It's
now easy to add dumping to console to mitmweb for debugging and development.
The same goes for external projects that derive from master.
- We also get major benefits in clarity for a frankly hairy part of our
project. Mitmdump is much clearer, and all the hairyness is now isolated for
further refactoring.

3 files changed, 94 insertions, 82 deletions

diff --git a/test/mitmproxy/builtins/test_dumper.py b/test/mitmproxy/builtins/test_dumper.py
new file mode 100644
index 00000000..57e3d036
--- /dev/null
+++ b/test/mitmproxy/builtins/test_dumper.py
@@ -0,0 +1,86 @@
+from .. import tutils, mastertest
+from six.moves import cStringIO as StringIO
+
+from mitmproxy.builtins import dumper
+from mitmproxy.flow import state
+from mitmproxy import exceptions
+from mitmproxy import dump
+from mitmproxy import models
+import netlib.tutils
+import mock
+
+
+class TestDumper(mastertest.MasterTest):
+    def test_simple(self):
+        d = dumper.Dumper()
+        sio = StringIO()
+
+        d.configure(dump.Options(tfile = sio, flow_detail = 0))
+        d.response(tutils.tflow())
+        assert not sio.getvalue()
+
+        d.configure(dump.Options(tfile = sio, flow_detail = 4))
+        d.response(tutils.tflow())
+        assert sio.getvalue()
+
+        sio = StringIO()
+        d.configure(dump.Options(tfile = sio, flow_detail = 4))
+        d.response(tutils.tflow(resp=True))
+        assert "<<" in sio.getvalue()
+
+        sio = StringIO()
+        d.configure(dump.Options(tfile = sio, flow_detail = 4))
+        d.response(tutils.tflow(err=True))
+        assert "<<" in sio.getvalue()
+
+        sio = StringIO()
+        d.configure(dump.Options(tfile = sio, flow_detail = 4))
+        flow = tutils.tflow()
+        flow.request = netlib.tutils.treq()
+        flow.request.stickycookie = True
+        flow.client_conn = mock.MagicMock()
+        flow.client_conn.address.host = "foo"
+        flow.response = netlib.tutils.tresp(content=None)
+        flow.response.is_replay = True
+        flow.response.status_code = 300
+        d.response(flow)
+        assert sio.getvalue()
+
+        sio = StringIO()
+        d.configure(dump.Options(tfile = sio, flow_detail = 4))
+        flow = tutils.tflow(resp=netlib.tutils.tresp(content=b"{"))
+        flow.response.headers["content-type"] = "application/json"
+        flow.response.status_code = 400
+        d.response(flow)
+        assert sio.getvalue()
+
+        sio = StringIO()
+        d.configure(dump.Options(tfile = sio))
+        flow = tutils.tflow()
+        flow.request.content = None
+        flow.response = models.HTTPResponse.wrap(netlib.tutils.tresp())
+        flow.response.content = None
+        d.response(flow)
+        assert "content missing" in sio.getvalue()
+
+
+class TestContentView(mastertest.MasterTest):
+    @mock.patch("mitmproxy.contentviews.get_content_view")
+    def test_contentview(self, get_content_view):
+        se = exceptions.ContentViewException(""), ("x", iter([]))
+        get_content_view.side_effect = se
+
+        s = state.State()
+        sio = StringIO()
+        m = mastertest.RecordingMaster(
+            dump.Options(
+                flow_detail=4,
+                verbosity=3,
+                tfile=sio,
+            ),
+            None, s
+        )
+        d = dumper.Dumper()
+        m.addons.add(d)
+        self.invoke(m, "response", tutils.tflow())
+        assert "Content viewer failed" in m.event_log[0][1]
diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py
index d1fe8cb4..dcc0dc48 100644
--- a/test/mitmproxy/mastertest.py
+++ b/test/mitmproxy/mastertest.py
@@ -8,11 +8,12 @@ from mitmproxy import flow, proxy, models, controller
 
 
 class MasterTest:
-    def invoke(self, master, handler, message):
+    def invoke(self, master, handler, *message):
         with master.handlecontext():
             func = getattr(master, handler)
-            func(message)
-        message.reply = controller.DummyReply()
+            func(*message)
+        if message:
+            message[0].reply = controller.DummyReply()
 
     def cycle(self, master, content):
         f = tutils.tflow(req=netlib.tutils.treq(content=content))
diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py
index c94630a9..90f33264 100644
--- a/test/mitmproxy/test_dump.py
+++ b/test/mitmproxy/test_dump.py
@@ -1,67 +1,11 @@
 import os
 from six.moves import cStringIO as StringIO
-from mitmproxy.exceptions import ContentViewException
 
-import netlib.tutils
-
-from mitmproxy import dump, flow, models, exceptions
+from mitmproxy import dump, flow, exceptions
 from . import tutils, mastertest
 import mock
 
 
-def test_strfuncs():
-    o = dump.Options(
-        tfile = StringIO(),
-        flow_detail = 0,
-    )
-    m = dump.DumpMaster(None, o)
-
-    m.o.flow_detail = 0
-    m.echo_flow(tutils.tflow())
-    assert not o.tfile.getvalue()
-
-    m.o.flow_detail = 4
-    m.echo_flow(tutils.tflow())
-    assert o.tfile.getvalue()
-
-    o.tfile = StringIO()
-    m.echo_flow(tutils.tflow(resp=True))
-    assert "<<" in o.tfile.getvalue()
-
-    o.tfile = StringIO()
-    m.echo_flow(tutils.tflow(err=True))
-    assert "<<" in o.tfile.getvalue()
-
-    flow = tutils.tflow()
-    flow.request = netlib.tutils.treq()
-    flow.request.stickycookie = True
-    flow.client_conn = mock.MagicMock()
-    flow.client_conn.address.host = "foo"
-    flow.response = netlib.tutils.tresp(content=None)
-    flow.response.is_replay = True
-    flow.response.status_code = 300
-    m.echo_flow(flow)
-
-    flow = tutils.tflow(resp=netlib.tutils.tresp(content=b"{"))
-    flow.response.headers["content-type"] = "application/json"
-    flow.response.status_code = 400
-    m.echo_flow(flow)
-
-
-@mock.patch("mitmproxy.contentviews.get_content_view")
-def test_contentview(get_content_view):
-    get_content_view.side_effect = ContentViewException(""), ("x", iter([]))
-
-    o = dump.Options(
-        flow_detail=4,
-        verbosity=3,
-        tfile=StringIO(),
-    )
-    m = dump.DumpMaster(None, o)
-    m.echo_flow(tutils.tflow())
-    assert "Content viewer failed" in m.options.tfile.getvalue()
-
-
 class TestDumpMaster(mastertest.MasterTest):
     def dummy_cycle(self, master, n, content):
         mastertest.MasterTest.dummy_cycle(self, master, n, content)
@@ -72,11 +16,7 @@ class TestDumpMaster(mastertest.MasterTest):
             options["verbosity"] = 0
         if "flow_detail" not in options:
             options["flow_detail"] = 0
-        o = dump.Options(
-            filtstr=filt,
-            tfile=StringIO(),
-            **options
-        )
+        o = dump.Options(filtstr=filt, tfile=StringIO(), **options)
         return dump.DumpMaster(None, o)
 
     def test_basic(self):
@@ -104,24 +44,10 @@ class TestDumpMaster(mastertest.MasterTest):
         )
         m = dump.DumpMaster(None, o)
         f = tutils.tflow(err=True)
-        m.request(f)
+        m.error(f)
         assert m.error(f)
         assert "error" in o.tfile.getvalue()
 
-    def test_missing_content(self):
-        o = dump.Options(
-            flow_detail=3,
-            tfile=StringIO(),
-        )
-        m = dump.DumpMaster(None, o)
-        f = tutils.tflow()
-        f.request.content = None
-        m.request(f)
-        f.response = models.HTTPResponse.wrap(netlib.tutils.tresp())
-        f.response.content = None
-        m.response(f)
-        assert "content missing" in o.tfile.getvalue()
-
     def test_replay(self):
         o = dump.Options(server_replay=["nonexistent"], kill=True)
         tutils.raises(dump.DumpError, dump.DumpMaster, None, o)
@@ -155,9 +81,8 @@ class TestDumpMaster(mastertest.MasterTest):
             self.flowfile(p)
             assert "GET" in self.dummy_cycle(
                 self.mkmaster(None, flow_detail=1, rfile=p),
-                0, b"",
+                1, b"",
             )
-
             tutils.raises(
                 dump.DumpError,
                 self.mkmaster, None, verbosity=1, rfile="/nonexistent"