Rebase on master

7 files changed, 343 insertions, 295 deletions

diff --git a/mitmproxy/main.py b/mitmproxy/main.py
index bf0b7e4d..584e9abd 100644
--- a/mitmproxy/main.py
+++ b/mitmproxy/main.py
@@ -120,7 +120,7 @@ def mitmweb(args=None):  # pragma: no cover
         options.verbose = 0
 
     proxy_config = config.process_proxy_options(parser, options)
-    web_options = web.Options(**cmdline.get_common_options(options))
+    web_options = web.master.Options(**cmdline.get_common_options(options))
     web_options.intercept = options.intercept
     web_options.wdebug = options.wdebug
     web_options.wiface = options.wiface
@@ -131,7 +131,7 @@ def mitmweb(args=None):  # pragma: no cover
 
     server = get_server(web_options.no_server, proxy_config)
 
-    m = web.WebMaster(server, web_options)
+    m = web.master.WebMaster(server, web_options)
     try:
         m.run()
     except (KeyboardInterrupt, _thread.error):
diff --git a/mitmproxy/web/__init__.py b/mitmproxy/web/__init__.py
index 0dbde204..0c324059 100644
--- a/mitmproxy/web/__init__.py
+++ b/mitmproxy/web/__init__.py
@@ -1,218 +1,3 @@
-from __future__ import absolute_import, print_function, division
+import master
 
-import collections
-import sys
-
-import tornado.httpserver
-import tornado.ioloop
-
-from mitmproxy import controller
-from mitmproxy import exceptions
-from mitmproxy import flow
-from mitmproxy.web import app
-from netlib.http import authentication
-
-
-class Stop(Exception):
-    pass
-
-
-class WebFlowView(flow.FlowView):
-
-    def __init__(self, store):
-        super(WebFlowView, self).__init__(store, None)
-
-    def _add(self, f):
-        super(WebFlowView, self)._add(f)
-        app.ClientConnection.broadcast(
-            type="UPDATE_FLOWS",
-            cmd="add",
-            data=app._strip_content(f.get_state())
-        )
-
-    def _update(self, f):
-        super(WebFlowView, self)._update(f)
-        app.ClientConnection.broadcast(
-            type="UPDATE_FLOWS",
-            cmd="update",
-            data=app._strip_content(f.get_state())
-        )
-
-    def _remove(self, f):
-        super(WebFlowView, self)._remove(f)
-        app.ClientConnection.broadcast(
-            type="UPDATE_FLOWS",
-            cmd="remove",
-            data=f.id
-        )
-
-    def _recalculate(self, flows):
-        super(WebFlowView, self)._recalculate(flows)
-        app.ClientConnection.broadcast(
-            type="UPDATE_FLOWS",
-            cmd="reset"
-        )
-
-
-class WebState(flow.State):
-
-    def __init__(self):
-        super(WebState, self).__init__()
-        self.view._close()
-        self.view = WebFlowView(self.flows)
-
-        self._last_event_id = 0
-        self.events = collections.deque(maxlen=1000)
-
-    def add_event(self, e, level):
-        self._last_event_id += 1
-        entry = {
-            "id": self._last_event_id,
-            "message": e,
-            "level": level
-        }
-        self.events.append(entry)
-        app.ClientConnection.broadcast(
-            type="UPDATE_EVENTLOG",
-            cmd="add",
-            data=entry
-        )
-
-    def clear(self):
-        super(WebState, self).clear()
-        self.events.clear()
-        app.ClientConnection.broadcast(
-            type="events",
-            cmd="reset",
-            data=[]
-        )
-
-
-class Options(object):
-    attributes = [
-        "app",
-        "app_domain",
-        "app_ip",
-        "anticache",
-        "anticomp",
-        "client_replay",
-        "eventlog",
-        "keepserving",
-        "kill",
-        "intercept",
-        "no_server",
-        "refresh_server_playback",
-        "rfile",
-        "scripts",
-        "showhost",
-        "replacements",
-        "rheaders",
-        "setheaders",
-        "server_replay",
-        "stickycookie",
-        "stickyauth",
-        "stream_large_bodies",
-        "verbosity",
-        "wfile",
-        "nopop",
-
-        "wdebug",
-        "wport",
-        "wiface",
-        "wauthenticator",
-        "wsingleuser",
-        "whtpasswd",
-    ]
-
-    def __init__(self, **kwargs):
-        for k, v in kwargs.items():
-            setattr(self, k, v)
-        for i in self.attributes:
-            if not hasattr(self, i):
-                setattr(self, i, None)
-
-    def process_web_options(self, parser):
-        if self.wsingleuser or self.whtpasswd:
-            if self.wsingleuser:
-                if len(self.wsingleuser.split(':')) != 2:
-                    return parser.error(
-                        "Invalid single-user specification. Please use the format username:password"
-                    )
-                username, password = self.wsingleuser.split(':')
-                self.wauthenticator = authentication.PassManSingleUser(username, password)
-            elif self.whtpasswd:
-                try:
-                    self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd)
-                except ValueError as v:
-                    return parser.error(v.message)
-        else:
-            self.wauthenticator = None
-
-
-class WebMaster(flow.FlowMaster):
-
-    def __init__(self, server, options):
-        self.options = options
-        super(WebMaster, self).__init__(server, WebState())
-        self.app = app.Application(self, self.options.wdebug, self.options.wauthenticator)
-        if options.rfile:
-            try:
-                self.load_flows_file(options.rfile)
-            except exceptions.FlowReadException as v:
-                self.add_event(
-                    "Could not read flow file: %s" % v,
-                    "error"
-                )
-
-        if options.outfile:
-            err = self.start_stream_to_path(
-                options.outfile[0],
-                options.outfile[1]
-            )
-            if err:
-                print("Stream file error: {}".format(err), file=sys.stderr)
-                sys.exit(1)
-
-        if self.options.app:
-            self.start_app(self.options.app_host, self.options.app_port)
-
-    def run(self):  # pragma: no cover
-
-        iol = tornado.ioloop.IOLoop.instance()
-
-        http_server = tornado.httpserver.HTTPServer(self.app)
-        http_server.listen(self.options.wport)
-
-        iol.add_callback(self.start)
-        tornado.ioloop.PeriodicCallback(lambda: self.tick(timeout=0), 5).start()
-        try:
-            print("Server listening at http://{}:{}".format(
-                self.options.wiface, self.options.wport), file=sys.stderr)
-            iol.start()
-        except (Stop, KeyboardInterrupt):
-            self.shutdown()
-
-    def _process_flow(self, f):
-        if self.state.intercept and self.state.intercept(
-                f) and not f.request.is_replay:
-            f.intercept(self)
-            f.reply.take()
-
-    @controller.handler
-    def request(self, f):
-        super(WebMaster, self).request(f)
-        self._process_flow(f)
-
-    @controller.handler
-    def response(self, f):
-        super(WebMaster, self).response(f)
-        self._process_flow(f)
-
-    @controller.handler
-    def error(self, f):
-        super(WebMaster, self).error(f)
-        self._process_flow(f)
-
-    def add_event(self, e, level="info"):
-        super(WebMaster, self).add_event(e, level)
-        self.state.add_event(e, level)
+__all__ = ["master"]
diff --git a/mitmproxy/web/master.py b/mitmproxy/web/master.py
new file mode 100644
index 00000000..1c6457eb
--- /dev/null
+++ b/mitmproxy/web/master.py
@@ -0,0 +1,220 @@
+from __future__ import absolute_import, print_function, division
+
+import sys
+import collections
+
+import tornado.httpserver
+import tornado.ioloop
+
+from mitmproxy import controller
+from mitmproxy import exceptions
+from mitmproxy import flow
+from mitmproxy.web import app
+from netlib.http import authentication
+
+
+class Stop(Exception):
+    pass
+
+
+class WebFlowView(flow.FlowView):
+
+    def __init__(self, store):
+        super(WebFlowView, self).__init__(store, None)
+
+    def _add(self, f):
+        super(WebFlowView, self)._add(f)
+        app.ClientConnection.broadcast(
+            type="UPDATE_FLOWS",
+            cmd="add",
+            data=app._strip_content(f.get_state())
+        )
+
+    def _update(self, f):
+        super(WebFlowView, self)._update(f)
+        app.ClientConnection.broadcast(
+            type="UPDATE_FLOWS",
+            cmd="update",
+            data=app._strip_content(f.get_state())
+        )
+
+    def _remove(self, f):
+        super(WebFlowView, self)._remove(f)
+        app.ClientConnection.broadcast(
+            type="UPDATE_FLOWS",
+            cmd="remove",
+            data=f.id
+        )
+
+    def _recalculate(self, flows):
+        super(WebFlowView, self)._recalculate(flows)
+        app.ClientConnection.broadcast(
+            type="UPDATE_FLOWS",
+            cmd="reset"
+        )
+
+
+class WebState(flow.State):
+
+    def __init__(self):
+        super(WebState, self).__init__()
+        self.view._close()
+        self.view = WebFlowView(self.flows)
+
+        self._last_event_id = 0
+        self.events = collections.deque(maxlen=1000)
+
+    def add_event(self, e, level):
+        self._last_event_id += 1
+        entry = {
+            "id": self._last_event_id,
+            "message": e,
+            "level": level
+        }
+        self.events.append(entry)
+        app.ClientConnection.broadcast(
+            type="UPDATE_EVENTLOG",
+            cmd="add",
+            data=entry
+        )
+
+    def clear(self):
+        super(WebState, self).clear()
+        self.events.clear()
+        app.ClientConnection.broadcast(
+            type="events",
+            cmd="reset",
+            data=[]
+        )
+
+
+class Options(object):
+    attributes = [
+        "app",
+        "app_domain",
+        "app_ip",
+        "anticache",
+        "anticomp",
+        "client_replay",
+        "eventlog",
+        "keepserving",
+        "kill",
+        "intercept",
+        "no_server",
+        "outfile",
+        "refresh_server_playback",
+        "rfile",
+        "scripts",
+        "showhost",
+        "replacements",
+        "rheaders",
+        "setheaders",
+        "server_replay",
+        "stickycookie",
+        "stickyauth",
+        "stream_large_bodies",
+        "verbosity",
+        "wfile",
+        "nopop",
+
+        "wdebug",
+        "wport",
+        "wiface",
+        "wauthenticator",
+        "wsingleuser",
+        "whtpasswd",
+    ]
+
+    def __init__(self, **kwargs):
+        for k, v in kwargs.items():
+            setattr(self, k, v)
+        for i in self.attributes:
+            if not hasattr(self, i):
+                setattr(self, i, None)
+
+    def process_web_options(self, parser):
+        if self.wsingleuser or self.whtpasswd:
+            if self.wsingleuser:
+                if len(self.wsingleuser.split(':')) != 2:
+                    return parser.error(
+                        "Invalid single-user specification. Please use the format username:password"
+                    )
+                username, password = self.wsingleuser.split(':')
+                self.wauthenticator = authentication.PassManSingleUser(username, password)
+            elif self.whtpasswd:
+                try:
+                    self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd)
+                except ValueError as v:
+                    return parser.error(v.message)
+        else:
+            self.wauthenticator = None
+
+
+class WebMaster(flow.FlowMaster):
+
+    def __init__(self, server, options):
+        self.options = options
+        super(WebMaster, self).__init__(server, WebState())
+        self.app = app.Application(self, self.options.wdebug, self.options.wauthenticator)
+        if options.rfile:
+            try:
+                self.load_flows_file(options.rfile)
+            except exceptions.FlowReadException as v:
+                self.add_event(
+                    "Could not read flow file: %s" % v,
+                    "error"
+                )
+
+        if options.outfile:
+            err = self.start_stream_to_path(
+                options.outfile[0],
+                options.outfile[1]
+            )
+            if err:
+                print("Stream file error: {}".format(err), file=sys.stderr)
+                sys.exit(1)
+
+        if self.options.app:
+            self.start_app(self.options.app_host, self.options.app_port)
+
+    def run(self):  # pragma: no cover
+
+        iol = tornado.ioloop.IOLoop.instance()
+
+        http_server = tornado.httpserver.HTTPServer(self.app)
+        http_server.listen(self.options.wport)
+
+        iol.add_callback(self.start)
+        tornado.ioloop.PeriodicCallback(lambda: self.tick(timeout=0), 5).start()
+        try:
+            print("Server listening at http://{}:{}".format(
+                self.options.wiface, self.options.wport), file=sys.stderr)
+            iol.start()
+        except (Stop, KeyboardInterrupt):
+            self.shutdown()
+
+    def _process_flow(self, f):
+        if self.state.intercept and self.state.intercept(
+                f) and not f.request.is_replay:
+            f.intercept(self)
+            f.reply.take()
+        return f
+
+    @controller.handler
+    def request(self, f):
+        super(WebMaster, self).request(f)
+        return self._process_flow(f)
+
+    @controller.handler
+    def response(self, f):
+        super(WebMaster, self).response(f)
+        return self._process_flow(f)
+
+    @controller.handler
+    def error(self, f):
+        super(WebMaster, self).error(f)
+        return self._process_flow(f)
+
+    def add_event(self, e, level="info"):
+        super(WebMaster, self).add_event(e, level)
+        return self.state.add_event(e, level)
diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py
new file mode 100644
index 00000000..9bb8826d
--- /dev/null
+++ b/test/mitmproxy/mastertest.py
@@ -0,0 +1,33 @@
+import tutils
+import netlib.tutils
+import mock
+
+from mitmproxy import flow, proxy, models
+
+
+class MasterTest:
+    def cycle(self, master, content):
+        f = tutils.tflow(req=netlib.tutils.treq(content=content))
+        l = proxy.Log("connect")
+        l.reply = mock.MagicMock()
+        master.log(l)
+        master.clientconnect(f.client_conn)
+        master.serverconnect(f.server_conn)
+        master.request(f)
+        if not f.error:
+            f.response = models.HTTPResponse.wrap(netlib.tutils.tresp(content=content))
+            f = master.response(f)
+        master.clientdisconnect(f.client_conn)
+        return f
+
+    def dummy_cycle(self, master, n, content):
+        for i in range(n):
+            self.cycle(master, content)
+        master.shutdown()
+
+    def flowfile(self, path):
+        f = open(path, "wb")
+        fw = flow.FlowWriter(f)
+        t = tutils.tflow(resp=True)
+        fw.add(t)
+        f.close()
diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py
index 36b78168..234490f8 100644
--- a/test/mitmproxy/test_dump.py
+++ b/test/mitmproxy/test_dump.py
@@ -1,13 +1,11 @@
 import os
 from six.moves import cStringIO as StringIO
 from mitmproxy.exceptions import ContentViewException
-from mitmproxy.models import HTTPResponse
 
 import netlib.tutils
 
-from mitmproxy import dump, flow
-from mitmproxy.proxy import Log
-from . import tutils
+from mitmproxy import dump, flow, models
+from . import tutils, mastertest
 import mock
 
 
@@ -58,37 +56,28 @@ def test_contentview(get_content_view):
     assert "Content viewer failed" in m.outfile.getvalue()
 
 
-class TestDumpMaster:
+class TestDumpMaster(mastertest.MasterTest):
+    def dummy_cycle(self, master, n, content):
+        mastertest.MasterTest.dummy_cycle(self, master, n, content)
+        return master.outfile.getvalue()
 
-    def _cycle(self, m, content):
-        f = tutils.tflow(req=netlib.tutils.treq(content=content))
-        l = Log("connect")
-        l.reply = mock.MagicMock()
-        m.log(l)
-        m.clientconnect(f.client_conn)
-        m.serverconnect(f.server_conn)
-        m.request(f)
-        if not f.error:
-            f.response = HTTPResponse.wrap(netlib.tutils.tresp(content=content))
-            f = m.response(f)
-        m.clientdisconnect(f.client_conn)
-        return f
-
-    def _dummy_cycle(self, n, filt, content, **options):
+    def mkmaster(self, filt, **options):
         cs = StringIO()
         o = dump.Options(filtstr=filt, **options)
-        m = dump.DumpMaster(None, o, outfile=cs)
-        for i in range(n):
-            self._cycle(m, content)
-        m.shutdown()
-        return cs.getvalue()
-
-    def _flowfile(self, path):
-        f = open(path, "wb")
-        fw = flow.FlowWriter(f)
-        t = tutils.tflow(resp=True)
-        fw.add(t)
-        f.close()
+        return dump.DumpMaster(None, o, outfile=cs)
+
+    def test_basic(self):
+        for i in (1, 2, 3):
+            assert "GET" in self.dummy_cycle(self.mkmaster("~s", flow_detail=i), 1, "")
+            assert "GET" in self.dummy_cycle(
+                self.mkmaster("~s", flow_detail=i),
+                1,
+                "\x00\x00\x00"
+            )
+            assert "GET" in self.dummy_cycle(
+                self.mkmaster("~s", flow_detail=i),
+                1, "ascii"
+            )
 
     def test_error(self):
         cs = StringIO()
@@ -106,7 +95,7 @@ class TestDumpMaster:
         f = tutils.tflow()
         f.request.content = None
         m.request(f)
-        f.response = HTTPResponse.wrap(netlib.tutils.tresp())
+        f.response = models.HTTPResponse.wrap(netlib.tutils.tresp())
         f.response.content = None
         m.response(f)
         assert "content missing" in cs.getvalue()
@@ -119,17 +108,17 @@ class TestDumpMaster:
 
         with tutils.tmpdir() as t:
             p = os.path.join(t, "rep")
-            self._flowfile(p)
+            self.flowfile(p)
 
             o = dump.Options(server_replay=[p], kill=True)
             m = dump.DumpMaster(None, o, outfile=cs)
 
-            self._cycle(m, "content")
-            self._cycle(m, "content")
+            self.cycle(m, "content")
+            self.cycle(m, "content")
 
             o = dump.Options(server_replay=[p], kill=False)
             m = dump.DumpMaster(None, o, outfile=cs)
-            self._cycle(m, "nonexistent")
+            self.cycle(m, "nonexistent")
 
             o = dump.Options(client_replay=[p], kill=False)
             m = dump.DumpMaster(None, o, outfile=cs)
@@ -137,22 +126,19 @@ class TestDumpMaster:
     def test_read(self):
         with tutils.tmpdir() as t:
             p = os.path.join(t, "read")
-            self._flowfile(p)
-            assert "GET" in self._dummy_cycle(
-                0,
-                None,
-                "",
-                flow_detail=1,
-                rfile=p
+            self.flowfile(p)
+            assert "GET" in self.dummy_cycle(
+                self.mkmaster(None, flow_detail=1, rfile=p),
+                0, "",
             )
 
             tutils.raises(
-                dump.DumpError, self._dummy_cycle,
-                0, None, "", verbosity=1, rfile="/nonexistent"
+                dump.DumpError,
+                self.mkmaster, None, verbosity=1, rfile="/nonexistent"
             )
             tutils.raises(
-                dump.DumpError, self._dummy_cycle,
-                0, None, "", verbosity=1, rfile="test_dump.py"
+                dump.DumpError,
+                self.mkmaster, None, verbosity=1, rfile="test_dump.py"
             )
 
     def test_options(self):
@@ -160,7 +146,9 @@ class TestDumpMaster:
         assert o.verbosity == 2
 
     def test_filter(self):
-        assert "GET" not in self._dummy_cycle(1, "~u foo", "", verbosity=1)
+        assert "GET" not in self.dummy_cycle(
+            self.mkmaster("~u foo", verbosity=1), 1, ""
+        )
 
     def test_app(self):
         o = dump.Options(app=True)
@@ -172,53 +160,50 @@ class TestDumpMaster:
         cs = StringIO()
         o = dump.Options(replacements=[(".*", "content", "foo")])
         m = dump.DumpMaster(None, o, outfile=cs)
-        f = self._cycle(m, "content")
+        f = self.cycle(m, "content")
         assert f.request.content == "foo"
 
     def test_setheader(self):
         cs = StringIO()
         o = dump.Options(setheaders=[(".*", "one", "two")])
         m = dump.DumpMaster(None, o, outfile=cs)
-        f = self._cycle(m, "content")
+        f = self.cycle(m, "content")
         assert f.request.headers["one"] == "two"
 
-    def test_basic(self):
-        for i in (1, 2, 3):
-            assert "GET" in self._dummy_cycle(1, "~s", "", flow_detail=i)
-            assert "GET" in self._dummy_cycle(
-                1,
-                "~s",
-                "\x00\x00\x00",
-                flow_detail=i)
-            assert "GET" in self._dummy_cycle(1, "~s", "ascii", flow_detail=i)
-
     def test_write(self):
         with tutils.tmpdir() as d:
             p = os.path.join(d, "a")
-            self._dummy_cycle(1, None, "", outfile=(p, "wb"), verbosity=0)
+            self.dummy_cycle(
+                self.mkmaster(None, outfile=(p, "wb"), verbosity=0), 1, ""
+            )
             assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 1
 
     def test_write_append(self):
         with tutils.tmpdir() as d:
             p = os.path.join(d, "a.append")
-            self._dummy_cycle(1, None, "", outfile=(p, "wb"), verbosity=0)
-            self._dummy_cycle(1, None, "", outfile=(p, "ab"), verbosity=0)
+            self.dummy_cycle(
+                self.mkmaster(None, outfile=(p, "wb"), verbosity=0),
+                1, ""
+            )
+            self.dummy_cycle(
+                self.mkmaster(None, outfile=(p, "ab"), verbosity=0),
+                1, ""
+            )
             assert len(list(flow.FlowReader(open(p, "rb")).stream())) == 2
 
     def test_write_err(self):
         tutils.raises(
             dump.DumpError,
-            self._dummy_cycle,
-            1,
-            None,
-            "",
-            outfile = ("nonexistentdir/foo", "wb")
+            self.mkmaster, None, outfile = ("nonexistentdir/foo", "wb")
         )
 
     def test_script(self):
-        ret = self._dummy_cycle(
-            1, None, "",
-            scripts=[tutils.test_data.path("data/scripts/all.py")], verbosity=1
+        ret = self.dummy_cycle(
+            self.mkmaster(
+                None,
+                scripts=[tutils.test_data.path("data/scripts/all.py")], verbosity=1
+            ),
+            1, "",
         )
         assert "XCLIENTCONNECT" in ret
         assert "XSERVERCONNECT" in ret
@@ -227,15 +212,23 @@ class TestDumpMaster:
         assert "XCLIENTDISCONNECT" in ret
         tutils.raises(
             dump.DumpError,
-            self._dummy_cycle, 1, None, "", scripts=["nonexistent"]
+            self.mkmaster,
+            None, scripts=["nonexistent"]
         )
         tutils.raises(
             dump.DumpError,
-            self._dummy_cycle, 1, None, "", scripts=["starterr.py"]
+            self.mkmaster,
+            None, scripts=["starterr.py"]
         )
 
     def test_stickycookie(self):
-        self._dummy_cycle(1, None, "", stickycookie = ".*")
+        self.dummy_cycle(
+            self.mkmaster(None, stickycookie = ".*"),
+            1, ""
+        )
 
     def test_stickyauth(self):
-        self._dummy_cycle(1, None, "", stickyauth = ".*")
+        self.dummy_cycle(
+            self.mkmaster(None, stickyauth = ".*"),
+            1, ""
+        )
diff --git a/test/mitmproxy/test_web_app.py b/test/mitmproxy/test_web_app.py
new file mode 100644
index 00000000..e69de29b
--- /dev/null
+++ b/test/mitmproxy/test_web_app.py
diff --git a/test/mitmproxy/test_web_master.py b/test/mitmproxy/test_web_master.py
new file mode 100644
index 00000000..98f53c93
--- /dev/null
+++ b/test/mitmproxy/test_web_master.py
@@ -0,0 +1,17 @@
+from mitmproxy.web import master
+from . import mastertest
+
+
+class TestWebMaster(mastertest.MasterTest):
+    def mkmaster(self, filt, **options):
+        o = master.Options(
+            filtstr=filt,
+            **options
+        )
+        return master.WebMaster(None, o)
+
+    def test_basic(self):
+        m = self.mkmaster(None)
+        for i in (1, 2, 3):
+            self.dummy_cycle(m, 1, "")
+            assert len(m.state.flows) == i