Extract console dump functionality into an addon

This removes all the code that deals with printing flows to screen from dump.py
into a self-contained addon.

- This fixes a bug - by moving dumping into an
addon, we now dump flows AFTER addon transformation, so we can see the changes
made.
- We get dumping "for free" in other places by simply adding the addon. It's
now easy to add dumping to console to mitmweb for debugging and development.
The same goes for external projects that derive from master.
- We also get major benefits in clarity for a frankly hairy part of our
project. Mitmdump is much clearer, and all the hairyness is now isolated for
further refactoring.

5 files changed, 324 insertions, 300 deletions

diff --git a/mitmproxy/builtins/dumper.py b/mitmproxy/builtins/dumper.py
new file mode 100644
index 00000000..73de5591
--- /dev/null
+++ b/mitmproxy/builtins/dumper.py
@@ -0,0 +1,223 @@
+from __future__ import absolute_import, print_function, division
+
+import itertools
+import traceback
+
+import click
+
+from mitmproxy import contentviews
+from mitmproxy import ctx
+from mitmproxy import exceptions
+from mitmproxy import filt
+from netlib import human
+from netlib import strutils
+
+
+def indent(n, text):
+    l = str(text).strip().splitlines()
+    pad = " " * n
+    return "\n".join(pad + i for i in l)
+
+
+class Dumper():
+    def __init__(self):
+        self.filter = None
+        self.flow_detail = None
+        self.outfp = None
+        self.showhost = None
+
+    def echo(self, text, ident=None, **style):
+        if ident:
+            text = indent(ident, text)
+        click.secho(text, file=self.outfp, **style)
+
+    def _echo_message(self, message):
+        if self.flow_detail >= 2 and hasattr(message, "headers"):
+            headers = "\r\n".join(
+                "{}: {}".format(
+                    click.style(strutils.bytes_to_escaped_str(k), fg="blue", bold=True),
+                    click.style(strutils.bytes_to_escaped_str(v), fg="blue"))
+                for k, v in message.headers.fields
+            )
+            self.echo(headers, ident=4)
+        if self.flow_detail >= 3:
+            try:
+                content = message.content
+            except ValueError:
+                content = message.get_content(strict=False)
+
+            if content is None:
+                self.echo("(content missing)", ident=4)
+            elif content:
+                self.echo("")
+
+                try:
+                    type, lines = contentviews.get_content_view(
+                        contentviews.get("Auto"),
+                        content,
+                        headers=getattr(message, "headers", None)
+                    )
+                except exceptions.ContentViewException:
+                    s = "Content viewer failed: \n" + traceback.format_exc()
+                    ctx.log.debug(s)
+                    type, lines = contentviews.get_content_view(
+                        contentviews.get("Raw"),
+                        content,
+                        headers=getattr(message, "headers", None)
+                    )
+
+                styles = dict(
+                    highlight=dict(bold=True),
+                    offset=dict(fg="blue"),
+                    header=dict(fg="green", bold=True),
+                    text=dict(fg="green")
+                )
+
+                def colorful(line):
+                    yield u"    "  # we can already indent here
+                    for (style, text) in line:
+                        yield click.style(text, **styles.get(style, {}))
+
+                if self.flow_detail == 3:
+                    lines_to_echo = itertools.islice(lines, 70)
+                else:
+                    lines_to_echo = lines
+
+                lines_to_echo = list(lines_to_echo)
+
+                content = u"\r\n".join(
+                    u"".join(colorful(line)) for line in lines_to_echo
+                )
+
+                self.echo(content)
+                if next(lines, None):
+                    self.echo("(cut off)", ident=4, dim=True)
+
+        if self.flow_detail >= 2:
+            self.echo("")
+
+    def _echo_request_line(self, flow):
+        if flow.request.stickycookie:
+            stickycookie = click.style(
+                "[stickycookie] ", fg="yellow", bold=True
+            )
+        else:
+            stickycookie = ""
+
+        if flow.client_conn:
+            client = click.style(strutils.escape_control_characters(flow.client_conn.address.host), bold=True)
+        else:
+            client = click.style("[replay]", fg="yellow", bold=True)
+
+        method = flow.request.method
+        method_color = dict(
+            GET="green",
+            DELETE="red"
+        ).get(method.upper(), "magenta")
+        method = click.style(strutils.escape_control_characters(method), fg=method_color, bold=True)
+        if self.showhost:
+            url = flow.request.pretty_url
+        else:
+            url = flow.request.url
+        url = click.style(strutils.escape_control_characters(url), bold=True)
+
+        httpversion = ""
+        if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"):
+            httpversion = " " + flow.request.http_version  # We hide "normal" HTTP 1.
+
+        line = "{stickycookie}{client} {method} {url}{httpversion}".format(
+            stickycookie=stickycookie,
+            client=client,
+            method=method,
+            url=url,
+            httpversion=httpversion
+        )
+        self.echo(line)
+
+    def _echo_response_line(self, flow):
+        if flow.response.is_replay:
+            replay = click.style("[replay] ", fg="yellow", bold=True)
+        else:
+            replay = ""
+
+        code = flow.response.status_code
+        code_color = None
+        if 200 <= code < 300:
+            code_color = "green"
+        elif 300 <= code < 400:
+            code_color = "magenta"
+        elif 400 <= code < 600:
+            code_color = "red"
+        code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418))
+        reason = click.style(strutils.escape_control_characters(flow.response.reason), fg=code_color, bold=True)
+
+        if flow.response.raw_content is None:
+            size = "(content missing)"
+        else:
+            size = human.pretty_size(len(flow.response.raw_content))
+        size = click.style(size, bold=True)
+
+        arrows = click.style("<<", bold=True)
+
+        line = "{replay} {arrows} {code} {reason} {size}".format(
+            replay=replay,
+            arrows=arrows,
+            code=code,
+            reason=reason,
+            size=size
+        )
+        self.echo(line)
+
+    def echo_flow(self, f):
+        if self.flow_detail == 0:
+            return
+
+        if f.request:
+            self._echo_request_line(f)
+            self._echo_message(f.request)
+
+        if f.response:
+            self._echo_response_line(f)
+            self._echo_message(f.response)
+
+        if f.error:
+            self.echo(" << {}".format(f.error.msg), bold=True, fg="red")
+
+        if self.outfp:
+            self.outfp.flush()
+
+    def _process_flow(self, f):
+        if self.filt and not f.match(self.filt):
+            return
+        self.echo_flow(f)
+
+    def configure(self, options):
+        if options.filtstr:
+            self.filt = filt.parse(options.filtstr)
+            if not self.filt:
+                raise exceptions.OptionsError(
+                    "Invalid filter expression: %s" % options.filtstr
+                )
+        else:
+            self.filt = None
+        self.flow_detail = options.flow_detail
+        self.outfp = options.tfile
+        self.showhost = options.showhost
+
+    def response(self, f):
+        self._process_flow(f)
+
+    def error(self, f):
+        self._process_flow(f)
+
+    def tcp_message(self, f):
+        if self.options.flow_detail == 0:
+            return
+        message = f.messages[-1]
+        direction = "->" if message.from_client else "<-"
+        self.echo("{client} {direction} tcp {direction} {server}".format(
+            client=repr(f.client_conn.address),
+            server=repr(f.server_conn.address),
+            direction=direction,
+        ))
+        self._echo_message(message)
diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py
index e7cebf99..cf4a6b93 100644
--- a/mitmproxy/dump.py
+++ b/mitmproxy/dump.py
@@ -1,24 +1,19 @@
 from __future__ import absolute_import, print_function, division
 
-import itertools
 import sys
-import traceback
-
-import click
 
 from typing import Optional  # noqa
 import typing  # noqa
 
-from mitmproxy import contentviews
+import click
+
 from mitmproxy import controller
 from mitmproxy import exceptions
-from mitmproxy import filt
 from mitmproxy import flow
 from mitmproxy import builtins
 from mitmproxy import utils
-from netlib import human
+from mitmproxy.builtins import dumper
 from netlib import tcp
-from netlib import strutils
 
 
 class DumpError(Exception):
@@ -28,9 +23,9 @@ class DumpError(Exception):
 class Options(flow.options.Options):
     def __init__(
             self,
+            keepserving=False,  # type: bool
             filtstr=None,  # type: Optional[str]
             flow_detail=1,  # type: int
-            keepserving=False,  # type: bool
             tfile=None,  # type: Optional[typing.io.TextIO]
             **kwargs
     ):
@@ -47,10 +42,9 @@ class DumpMaster(flow.FlowMaster):
         flow.FlowMaster.__init__(self, options, server, flow.State())
         self.has_errored = False
         self.addons.add(*builtins.default_addons())
+        self.addons.add(dumper.Dumper())
         # This line is just for type hinting
         self.options = self.options  # type: Options
-        self.o = options
-        self.showhost = options.showhost
         self.replay_ignore_params = options.replay_ignore_params
         self.replay_ignore_content = options.replay_ignore_content
         self.replay_ignore_host = options.replay_ignore_host
@@ -64,11 +58,6 @@ class DumpMaster(flow.FlowMaster):
                   "HTTP/2 is disabled. Use --no-http2 to silence this warning.",
                   file=sys.stderr)
 
-        if options.filtstr:
-            self.filt = filt.parse(options.filtstr)
-        else:
-            self.filt = None
-
         if options.setheaders:
             for i in options.setheaders:
                 self.setheaders.add(*i)
@@ -115,185 +104,14 @@ class DumpMaster(flow.FlowMaster):
         if level == "error":
             self.has_errored = True
         if self.options.verbosity >= utils.log_tier(level):
-            self.echo(
+            click.secho(
                 e,
+                file=self.options.tfile,
                 fg="red" if level == "error" else None,
                 dim=(level == "debug"),
                 err=(level == "error")
             )
 
-    @staticmethod
-    def indent(n, text):
-        l = str(text).strip().splitlines()
-        pad = " " * n
-        return "\n".join(pad + i for i in l)
-
-    def echo(self, text, indent=None, **style):
-        if indent:
-            text = self.indent(indent, text)
-        click.secho(text, file=self.options.tfile, **style)
-
-    def _echo_message(self, message):
-        if self.options.flow_detail >= 2 and hasattr(message, "headers"):
-            headers = "\r\n".join(
-                "{}: {}".format(
-                    click.style(strutils.bytes_to_escaped_str(k), fg="blue", bold=True),
-                    click.style(strutils.bytes_to_escaped_str(v), fg="blue"))
-                for k, v in message.headers.fields
-            )
-            self.echo(headers, indent=4)
-        if self.options.flow_detail >= 3:
-            try:
-                content = message.content
-            except ValueError:
-                content = message.get_content(strict=False)
-
-            if content is None:
-                self.echo("(content missing)", indent=4)
-            elif content:
-                self.echo("")
-
-                try:
-                    type, lines = contentviews.get_content_view(
-                        contentviews.get("Auto"),
-                        content,
-                        headers=getattr(message, "headers", None)
-                    )
-                except exceptions.ContentViewException:
-                    s = "Content viewer failed: \n" + traceback.format_exc()
-                    self.add_log(s, "debug")
-                    type, lines = contentviews.get_content_view(
-                        contentviews.get("Raw"),
-                        content,
-                        headers=getattr(message, "headers", None)
-                    )
-
-                styles = dict(
-                    highlight=dict(bold=True),
-                    offset=dict(fg="blue"),
-                    header=dict(fg="green", bold=True),
-                    text=dict(fg="green")
-                )
-
-                def colorful(line):
-                    yield u"    "  # we can already indent here
-                    for (style, text) in line:
-                        yield click.style(text, **styles.get(style, {}))
-
-                if self.options.flow_detail == 3:
-                    lines_to_echo = itertools.islice(lines, 70)
-                else:
-                    lines_to_echo = lines
-
-                lines_to_echo = list(lines_to_echo)
-
-                content = u"\r\n".join(
-                    u"".join(colorful(line)) for line in lines_to_echo
-                )
-
-                self.echo(content)
-                if next(lines, None):
-                    self.echo("(cut off)", indent=4, dim=True)
-
-        if self.options.flow_detail >= 2:
-            self.echo("")
-
-    def _echo_request_line(self, flow):
-        if flow.request.stickycookie:
-            stickycookie = click.style(
-                "[stickycookie] ", fg="yellow", bold=True
-            )
-        else:
-            stickycookie = ""
-
-        if flow.client_conn:
-            client = click.style(strutils.escape_control_characters(flow.client_conn.address.host), bold=True)
-        else:
-            client = click.style("[replay]", fg="yellow", bold=True)
-
-        method = flow.request.method
-        method_color = dict(
-            GET="green",
-            DELETE="red"
-        ).get(method.upper(), "magenta")
-        method = click.style(strutils.escape_control_characters(method), fg=method_color, bold=True)
-        if self.showhost:
-            url = flow.request.pretty_url
-        else:
-            url = flow.request.url
-        url = click.style(strutils.escape_control_characters(url), bold=True)
-
-        httpversion = ""
-        if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"):
-            httpversion = " " + flow.request.http_version  # We hide "normal" HTTP 1.
-
-        line = "{stickycookie}{client} {method} {url}{httpversion}".format(
-            stickycookie=stickycookie,
-            client=client,
-            method=method,
-            url=url,
-            httpversion=httpversion
-        )
-        self.echo(line)
-
-    def _echo_response_line(self, flow):
-        if flow.response.is_replay:
-            replay = click.style("[replay] ", fg="yellow", bold=True)
-        else:
-            replay = ""
-
-        code = flow.response.status_code
-        code_color = None
-        if 200 <= code < 300:
-            code_color = "green"
-        elif 300 <= code < 400:
-            code_color = "magenta"
-        elif 400 <= code < 600:
-            code_color = "red"
-        code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418))
-        reason = click.style(strutils.escape_control_characters(flow.response.reason), fg=code_color, bold=True)
-
-        if flow.response.raw_content is None:
-            size = "(content missing)"
-        else:
-            size = human.pretty_size(len(flow.response.raw_content))
-        size = click.style(size, bold=True)
-
-        arrows = click.style("<<", bold=True)
-
-        line = "{replay} {arrows} {code} {reason} {size}".format(
-            replay=replay,
-            arrows=arrows,
-            code=code,
-            reason=reason,
-            size=size
-        )
-        self.echo(line)
-
-    def echo_flow(self, f):
-        if self.options.flow_detail == 0:
-            return
-
-        if f.request:
-            self._echo_request_line(f)
-            self._echo_message(f.request)
-
-        if f.response:
-            self._echo_response_line(f)
-            self._echo_message(f.response)
-
-        if f.error:
-            self.echo(" << {}".format(f.error.msg), bold=True, fg="red")
-
-        if self.options.tfile:
-            self.options.tfile.flush()
-
-    def _process_flow(self, f):
-        if self.filt and not f.match(self.filt):
-            return
-
-        self.echo_flow(f)
-
     @controller.handler
     def request(self, f):
         f = flow.FlowMaster.request(self, f)
@@ -301,35 +119,6 @@ class DumpMaster(flow.FlowMaster):
             self.state.delete_flow(f)
         return f
 
-    @controller.handler
-    def response(self, f):
-        f = flow.FlowMaster.response(self, f)
-        if f:
-            self._process_flow(f)
-        return f
-
-    @controller.handler
-    def error(self, f):
-        flow.FlowMaster.error(self, f)
-        if f:
-            self._process_flow(f)
-        return f
-
-    @controller.handler
-    def tcp_message(self, f):
-        super(DumpMaster, self).tcp_message(f)
-
-        if self.options.flow_detail == 0:
-            return
-        message = f.messages[-1]
-        direction = "->" if message.from_client else "<-"
-        self.echo("{client} {direction} tcp {direction} {server}".format(
-            client=repr(f.client_conn.address),
-            server=repr(f.server_conn.address),
-            direction=direction,
-        ))
-        self._echo_message(message)
-
     def run(self):  # pragma: no cover
         if self.options.rfile and not self.options.keepserving:
             return
diff --git a/test/mitmproxy/builtins/test_dumper.py b/test/mitmproxy/builtins/test_dumper.py
new file mode 100644
index 00000000..57e3d036
--- /dev/null
+++ b/test/mitmproxy/builtins/test_dumper.py
@@ -0,0 +1,86 @@
+from .. import tutils, mastertest
+from six.moves import cStringIO as StringIO
+
+from mitmproxy.builtins import dumper
+from mitmproxy.flow import state
+from mitmproxy import exceptions
+from mitmproxy import dump
+from mitmproxy import models
+import netlib.tutils
+import mock
+
+
+class TestDumper(mastertest.MasterTest):
+    def test_simple(self):
+        d = dumper.Dumper()
+        sio = StringIO()
+
+        d.configure(dump.Options(tfile = sio, flow_detail = 0))
+        d.response(tutils.tflow())
+        assert not sio.getvalue()
+
+        d.configure(dump.Options(tfile = sio, flow_detail = 4))
+        d.response(tutils.tflow())
+        assert sio.getvalue()
+
+        sio = StringIO()
+        d.configure(dump.Options(tfile = sio, flow_detail = 4))
+        d.response(tutils.tflow(resp=True))
+        assert "<<" in sio.getvalue()
+
+        sio = StringIO()
+        d.configure(dump.Options(tfile = sio, flow_detail = 4))
+        d.response(tutils.tflow(err=True))
+        assert "<<" in sio.getvalue()
+
+        sio = StringIO()
+        d.configure(dump.Options(tfile = sio, flow_detail = 4))
+        flow = tutils.tflow()
+        flow.request = netlib.tutils.treq()
+        flow.request.stickycookie = True
+        flow.client_conn = mock.MagicMock()
+        flow.client_conn.address.host = "foo"
+        flow.response = netlib.tutils.tresp(content=None)
+        flow.response.is_replay = True
+        flow.response.status_code = 300
+        d.response(flow)
+        assert sio.getvalue()
+
+        sio = StringIO()
+        d.configure(dump.Options(tfile = sio, flow_detail = 4))
+        flow = tutils.tflow(resp=netlib.tutils.tresp(content=b"{"))
+        flow.response.headers["content-type"] = "application/json"
+        flow.response.status_code = 400
+        d.response(flow)
+        assert sio.getvalue()
+
+        sio = StringIO()
+        d.configure(dump.Options(tfile = sio))
+        flow = tutils.tflow()
+        flow.request.content = None
+        flow.response = models.HTTPResponse.wrap(netlib.tutils.tresp())
+        flow.response.content = None
+        d.response(flow)
+        assert "content missing" in sio.getvalue()
+
+
+class TestContentView(mastertest.MasterTest):
+    @mock.patch("mitmproxy.contentviews.get_content_view")
+    def test_contentview(self, get_content_view):
+        se = exceptions.ContentViewException(""), ("x", iter([]))
+        get_content_view.side_effect = se
+
+        s = state.State()
+        sio = StringIO()
+        m = mastertest.RecordingMaster(
+            dump.Options(
+                flow_detail=4,
+                verbosity=3,
+                tfile=sio,
+            ),
+            None, s
+        )
+        d = dumper.Dumper()
+        m.addons.add(d)
+        self.invoke(m, "response", tutils.tflow())
+        assert "Content viewer failed" in m.event_log[0][1]
diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py
index d1fe8cb4..dcc0dc48 100644
--- a/test/mitmproxy/mastertest.py
+++ b/test/mitmproxy/mastertest.py
@@ -8,11 +8,12 @@ from mitmproxy import flow, proxy, models, controller
 
 
 class MasterTest:
-    def invoke(self, master, handler, message):
+    def invoke(self, master, handler, *message):
         with master.handlecontext():
             func = getattr(master, handler)
-            func(message)
-        message.reply = controller.DummyReply()
+            func(*message)
+        if message:
+            message[0].reply = controller.DummyReply()
 
     def cycle(self, master, content):
         f = tutils.tflow(req=netlib.tutils.treq(content=content))
diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py
index c94630a9..90f33264 100644
--- a/test/mitmproxy/test_dump.py
+++ b/test/mitmproxy/test_dump.py
@@ -1,67 +1,11 @@
 import os
 from six.moves import cStringIO as StringIO
-from mitmproxy.exceptions import ContentViewException
 
-import netlib.tutils
-
-from mitmproxy import dump, flow, models, exceptions
+from mitmproxy import dump, flow, exceptions
 from . import tutils, mastertest
 import mock
 
 
-def test_strfuncs():
-    o = dump.Options(
-        tfile = StringIO(),
-        flow_detail = 0,
-    )
-    m = dump.DumpMaster(None, o)
-
-    m.o.flow_detail = 0
-    m.echo_flow(tutils.tflow())
-    assert not o.tfile.getvalue()
-
-    m.o.flow_detail = 4
-    m.echo_flow(tutils.tflow())
-    assert o.tfile.getvalue()
-
-    o.tfile = StringIO()
-    m.echo_flow(tutils.tflow(resp=True))
-    assert "<<" in o.tfile.getvalue()
-
-    o.tfile = StringIO()
-    m.echo_flow(tutils.tflow(err=True))
-    assert "<<" in o.tfile.getvalue()
-
-    flow = tutils.tflow()
-    flow.request = netlib.tutils.treq()
-    flow.request.stickycookie = True
-    flow.client_conn = mock.MagicMock()
-    flow.client_conn.address.host = "foo"
-    flow.response = netlib.tutils.tresp(content=None)
-    flow.response.is_replay = True
-    flow.response.status_code = 300
-    m.echo_flow(flow)
-
-    flow = tutils.tflow(resp=netlib.tutils.tresp(content=b"{"))
-    flow.response.headers["content-type"] = "application/json"
-    flow.response.status_code = 400
-    m.echo_flow(flow)
-
-
-@mock.patch("mitmproxy.contentviews.get_content_view")
-def test_contentview(get_content_view):
-    get_content_view.side_effect = ContentViewException(""), ("x", iter([]))
-
-    o = dump.Options(
-        flow_detail=4,
-        verbosity=3,
-        tfile=StringIO(),
-    )
-    m = dump.DumpMaster(None, o)
-    m.echo_flow(tutils.tflow())
-    assert "Content viewer failed" in m.options.tfile.getvalue()
-
-
 class TestDumpMaster(mastertest.MasterTest):
     def dummy_cycle(self, master, n, content):
         mastertest.MasterTest.dummy_cycle(self, master, n, content)
@@ -72,11 +16,7 @@ class TestDumpMaster(mastertest.MasterTest):
             options["verbosity"] = 0
         if "flow_detail" not in options:
             options["flow_detail"] = 0
-        o = dump.Options(
-            filtstr=filt,
-            tfile=StringIO(),
-            **options
-        )
+        o = dump.Options(filtstr=filt, tfile=StringIO(), **options)
         return dump.DumpMaster(None, o)
 
     def test_basic(self):
@@ -104,24 +44,10 @@ class TestDumpMaster(mastertest.MasterTest):
         )
         m = dump.DumpMaster(None, o)
         f = tutils.tflow(err=True)
-        m.request(f)
+        m.error(f)
         assert m.error(f)
         assert "error" in o.tfile.getvalue()
 
-    def test_missing_content(self):
-        o = dump.Options(
-            flow_detail=3,
-            tfile=StringIO(),
-        )
-        m = dump.DumpMaster(None, o)
-        f = tutils.tflow()
-        f.request.content = None
-        m.request(f)
-        f.response = models.HTTPResponse.wrap(netlib.tutils.tresp())
-        f.response.content = None
-        m.response(f)
-        assert "content missing" in o.tfile.getvalue()
-
     def test_replay(self):
         o = dump.Options(server_replay=["nonexistent"], kill=True)
         tutils.raises(dump.DumpError, dump.DumpMaster, None, o)
@@ -155,9 +81,8 @@ class TestDumpMaster(mastertest.MasterTest):
             self.flowfile(p)
             assert "GET" in self.dummy_cycle(
                 self.mkmaster(None, flow_detail=1, rfile=p),
-                0, b"",
+                1, b"",
             )
-
             tutils.raises(
                 dump.DumpError,
                 self.mkmaster, None, verbosity=1, rfile="/nonexistent"