Merge branch 'mitmweb-eventlog'

22 files changed, 505 insertions, 275 deletions

diff --git a/examples/complex/dup_and_replay.py b/examples/complex/dup_and_replay.py
index bf7c2a4e..2baa1ea6 100644
--- a/examples/complex/dup_and_replay.py
+++ b/examples/complex/dup_and_replay.py
@@ -2,6 +2,7 @@ from mitmproxy import ctx
 
 
 def request(flow):
-    f = ctx.master.state.duplicate_flow(flow)
+    f = flow.copy()
+    ctx.master.view.add(f)
     f.request.path = "/changed"
     ctx.master.replay_request(f, block=True)
diff --git a/mitmproxy/addons/eventstore.py b/mitmproxy/addons/eventstore.py
new file mode 100644
index 00000000..4e410c98
--- /dev/null
+++ b/mitmproxy/addons/eventstore.py
@@ -0,0 +1,19 @@
+from typing import List  # noqa
+
+import blinker
+from mitmproxy.log import LogEntry
+
+
+class EventStore:
+    def __init__(self):
+        self.data = []  # type: List[LogEntry]
+        self.sig_add = blinker.Signal()
+        self.sig_refresh = blinker.Signal()
+
+    def log(self, entry: LogEntry):
+        self.data.append(entry)
+        self.sig_add.send(self, entry=entry)
+
+    def clear(self):
+        self.data.clear()
+        self.sig_refresh.send(self)
diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py
index e151e081..b8b6093f 100644
--- a/mitmproxy/addons/view.py
+++ b/mitmproxy/addons/view.py
@@ -280,6 +280,13 @@ class View(collections.Sequence):
                     # The value was not in the view
                     pass
 
+    def get_by_id(self, flow_id: str) -> typing.Optional[mitmproxy.flow.Flow]:
+        """
+        Get flow with the given id from the store.
+        Returns None if the flow is not found.
+        """
+        return self._store.get(flow_id)
+
     # Event handlers
     def configure(self, opts, updated):
         if "filter" in updated:
diff --git a/mitmproxy/contentviews.py b/mitmproxy/contentviews.py
index f6a2262c..ef0c80e0 100644
--- a/mitmproxy/contentviews.py
+++ b/mitmproxy/contentviews.py
@@ -609,7 +609,7 @@ def get_message_content_view(viewname, message):
     """
     viewmode = get(viewname)
     if not viewmode:
-        get("auto")
+        viewmode = get("auto")
     try:
         content = message.content
     except ValueError:
diff --git a/mitmproxy/io.py b/mitmproxy/io.py
index ad2f00c4..780955a4 100644
--- a/mitmproxy/io.py
+++ b/mitmproxy/io.py
@@ -1,6 +1,8 @@
 import os
+from typing import Iterable
 
 from mitmproxy import exceptions
+from mitmproxy import flow
 from mitmproxy import flowfilter
 from mitmproxy import http
 from mitmproxy import tcp
@@ -29,7 +31,7 @@ class FlowReader:
     def __init__(self, fo):
         self.fo = fo
 
-    def stream(self):
+    def stream(self) -> Iterable[flow.Flow]:
         """
             Yields Flow objects from the dump.
         """
@@ -54,10 +56,10 @@ class FilteredFlowWriter:
         self.fo = fo
         self.flt = flt
 
-    def add(self, flow):
-        if self.flt and not flowfilter.match(self.flt, flow):
+    def add(self, f: flow.Flow):
+        if self.flt and not flowfilter.match(self.flt, f):
             return
-        d = flow.get_state()
+        d = f.get_state()
         tnetstring.dump(d, self.fo)
 
 
diff --git a/mitmproxy/master.py b/mitmproxy/master.py
index 7f114096..7581d816 100644
--- a/mitmproxy/master.py
+++ b/mitmproxy/master.py
@@ -156,7 +156,7 @@ class Master:
         for e, o in events.event_sequence(f):
             getattr(self, e)(o)
 
-    def load_flows(self, fr):
+    def load_flows(self, fr: io.FlowReader) -> int:
         """
             Load flows from a FlowReader object.
         """
@@ -166,7 +166,7 @@ class Master:
             self.load_flow(i)
         return cnt
 
-    def load_flows_file(self, path):
+    def load_flows_file(self, path: str) -> int:
         path = os.path.expanduser(path)
         try:
             if path == "-":
@@ -180,7 +180,11 @@ class Master:
         except IOError as v:
             raise exceptions.FlowReadException(v.strerror)
 
-    def replay_request(self, f, block=False):
+    def replay_request(
+            self,
+            f: http.HTTPFlow,
+            block: bool=False
+    ) -> http_replay.RequestReplayThread:
         """
         Replay a HTTP request to receive a new response from the server.
 
diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py
index 395a9d52..d7c312e5 100644
--- a/mitmproxy/tools/console/flowlist.py
+++ b/mitmproxy/tools/console/flowlist.py
@@ -303,8 +303,8 @@ class FlowListWalker(urwid.ListWalker):
 
 class FlowListBox(urwid.ListBox):
 
-    def __init__(self, master):
-        self.master = master
+    def __init__(self, master: "mitmproxy.tools.console.master.ConsoleMaster"):
+        self.master = master  # type: "mitmproxy.tools.console.master.ConsoleMaster"
         super().__init__(FlowListWalker(master))
 
     def get_method_raw(self, k):
@@ -348,7 +348,7 @@ class FlowListBox(urwid.ListBox):
         if key == "A":
             for f in self.master.view:
                 if f.intercepted:
-                    f.resume()
+                    f.resume(self.master)
             signals.flowlist_change.send(self)
         elif key == "z":
             self.master.view.clear()
diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py
index e2b24fab..ecb070d8 100644
--- a/mitmproxy/tools/console/flowview.py
+++ b/mitmproxy/tools/console/flowview.py
@@ -510,8 +510,10 @@ class FlowView(tabs.Tabs):
             self.flow.resume(self.master)
             signals.flow_change.send(self, flow = self.flow)
         elif key == "A":
-            self.master.accept_all()
-            signals.flow_change.send(self, flow = self.flow)
+            for f in self.view:
+                if f.intercepted:
+                    f.resume(self.master)
+                    signals.flow_change.send(self, flow=f)
         elif key == "d":
             if self.flow.killable:
                 self.flow.kill(self.master)
diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py
index 184038ef..5d0e0ef4 100644
--- a/mitmproxy/tools/console/master.py
+++ b/mitmproxy/tools/console/master.py
@@ -71,7 +71,7 @@ class ConsoleMaster(master.Master):
 
     def __init__(self, options, server):
         super().__init__(options, server)
-        self.view = view.View()
+        self.view = view.View()  # type: view.View
         self.stream_path = None
         # This line is just for type hinting
         self.options = self.options  # type: Options
diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py
index 69dd3791..478690eb 100644
--- a/mitmproxy/tools/main.py
+++ b/mitmproxy/tools/main.py
@@ -135,7 +135,6 @@ def mitmweb(args=None):  # pragma: no cover
         web_options.wdebug = args.wdebug
         web_options.wiface = args.wiface
         web_options.wport = args.wport
-        web_options.process_web_options(parser)
 
         server = process_options(parser, web_options, args)
         m = web.master.WebMaster(web_options, server)
diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py
index 25a46169..f617bd08 100644
--- a/mitmproxy/tools/web/app.py
+++ b/mitmproxy/tools/web/app.py
@@ -1,5 +1,3 @@
-
-import base64
 import hashlib
 import json
 import logging
@@ -7,19 +5,21 @@ import os.path
 import re
 from io import BytesIO
 
+import mitmproxy.addons.view
+import mitmproxy.flow
+import tornado.escape
 import tornado.web
 import tornado.websocket
-import tornado.escape
 from mitmproxy import contentviews
+from mitmproxy import exceptions
 from mitmproxy import flowfilter
 from mitmproxy import http
 from mitmproxy import io
+from mitmproxy import log
 from mitmproxy import version
-import mitmproxy.addons.view
-import mitmproxy.flow
 
 
-def convert_flow_to_json_dict(flow: mitmproxy.flow.Flow) -> dict:
+def flow_to_json(flow: mitmproxy.flow.Flow) -> dict:
     """
     Remove flow message content and cert to save transmission space.
 
@@ -46,8 +46,10 @@ def convert_flow_to_json_dict(flow: mitmproxy.flow.Flow) -> dict:
                 "path": flow.request.path,
                 "http_version": flow.request.http_version,
                 "headers": tuple(flow.request.headers.items(True)),
-                "contentLength": len(flow.request.raw_content) if flow.request.raw_content is not None else None,
-                "contentHash": hashlib.sha256(flow.request.raw_content).hexdigest() if flow.request.raw_content is not None else None,
+                "contentLength": len(
+                    flow.request.raw_content) if flow.request.raw_content is not None else None,
+                "contentHash": hashlib.sha256(
+                    flow.request.raw_content).hexdigest() if flow.request.raw_content is not None else None,
                 "timestamp_start": flow.request.timestamp_start,
                 "timestamp_end": flow.request.timestamp_end,
                 "is_replay": flow.request.is_replay,
@@ -58,8 +60,10 @@ def convert_flow_to_json_dict(flow: mitmproxy.flow.Flow) -> dict:
                 "status_code": flow.response.status_code,
                 "reason": flow.response.reason,
                 "headers": tuple(flow.response.headers.items(True)),
-                "contentLength": len(flow.response.raw_content) if flow.response.raw_content is not None else None,
-                "contentHash": hashlib.sha256(flow.response.raw_content).hexdigest() if flow.response.raw_content is not None else None,
+                "contentLength": len(
+                    flow.response.raw_content) if flow.response.raw_content is not None else None,
+                "contentHash": hashlib.sha256(
+                    flow.response.raw_content).hexdigest() if flow.response.raw_content is not None else None,
                 "timestamp_start": flow.response.timestamp_start,
                 "timestamp_end": flow.response.timestamp_end,
                 "is_replay": flow.response.is_replay,
@@ -69,34 +73,19 @@ def convert_flow_to_json_dict(flow: mitmproxy.flow.Flow) -> dict:
     return f
 
 
-class APIError(tornado.web.HTTPError):
-    pass
-
-
-class BasicAuth:
-
-    def set_auth_headers(self):
-        self.set_status(401)
-        self.set_header('WWW-Authenticate', 'Basic realm=MITMWeb')
-        self._transforms = []
-        self.finish()
+def logentry_to_json(e: log.LogEntry) -> dict:
+    return {
+        "id": id(e),  # we just need some kind of id.
+        "message": e.msg,
+        "level": e.level
+    }
 
-    def prepare(self):
-        wauthenticator = self.application.settings['wauthenticator']
-        if wauthenticator:
-            auth_header = self.request.headers.get('Authorization')
-            if auth_header is None or not auth_header.startswith('Basic '):
-                self.set_auth_headers()
-            else:
-                auth_decoded = base64.decodebytes(auth_header[6:])
-                username, password = auth_decoded.split(':', 2)
-                if not wauthenticator.test(username, password):
-                    self.set_auth_headers()
-                    raise APIError(401, "Invalid username or password.")
 
+class APIError(tornado.web.HTTPError):
+    pass
 
-class RequestHandler(BasicAuth, tornado.web.RequestHandler):
 
+class RequestHandler(tornado.web.RequestHandler):
     def write(self, chunk):
         # Writing arrays on the top level is ok nowadays.
         # http://flask.pocoo.org/docs/0.11/security/#json-security
@@ -120,9 +109,23 @@ class RequestHandler(BasicAuth, tornado.web.RequestHandler):
 
     @property
     def json(self):
-        if not self.request.headers.get("Content-Type").startswith("application/json"):
-            return None
-        return json.loads(self.request.body.decode())
+        if not self.request.headers.get("Content-Type", "").startswith("application/json"):
+            raise APIError(400, "Invalid Content-Type, expected application/json.")
+        try:
+            return json.loads(self.request.body.decode())
+        except Exception as e:
+            raise APIError(400, "Malformed JSON: {}".format(str(e)))
+
+    @property
+    def filecontents(self):
+        """
+        Accept either a multipart/form file upload or just take the plain request body.
+
+        """
+        if self.request.files:
+            return next(iter(self.request.files.values()))[0].body
+        else:
+            return self.request.body
 
     @property
     def view(self) -> mitmproxy.addons.view.View:
@@ -136,11 +139,11 @@ class RequestHandler(BasicAuth, tornado.web.RequestHandler):
     def flow(self) -> mitmproxy.flow.Flow:
         flow_id = str(self.path_kwargs["flow_id"])
         # FIXME: Add a facility to addon.view to safely access the store
-        flow = self.view._store.get(flow_id)
+        flow = self.view.get_by_id(flow_id)
         if flow:
             return flow
         else:
-            raise APIError(400, "Flow not found.")
+            raise APIError(404, "Flow not found.")
 
     def write_error(self, status_code: int, **kwargs):
         if "exc_info" in kwargs and isinstance(kwargs["exc_info"][1], APIError):
@@ -150,7 +153,6 @@ class RequestHandler(BasicAuth, tornado.web.RequestHandler):
 
 
 class IndexHandler(RequestHandler):
-
     def get(self):
         token = self.xsrf_token  # https://github.com/tornadoweb/tornado/issues/645
         assert token
@@ -158,14 +160,13 @@ class IndexHandler(RequestHandler):
 
 
 class FilterHelp(RequestHandler):
-
     def get(self):
         self.write(dict(
             commands=flowfilter.help
         ))
 
 
-class WebSocketEventBroadcaster(BasicAuth, tornado.websocket.WebSocketHandler):
+class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler):
     # raise an error if inherited class doesn't specify its own instance.
     connections = None  # type: set
 
@@ -182,7 +183,7 @@ class WebSocketEventBroadcaster(BasicAuth, tornado.websocket.WebSocketHandler):
         for conn in cls.connections:
             try:
                 conn.write_message(message)
-            except Exception:
+            except Exception:  # pragma: no cover
                 logging.error("Error sending message", exc_info=True)
 
 
@@ -191,9 +192,8 @@ class ClientConnection(WebSocketEventBroadcaster):
 
 
 class Flows(RequestHandler):
-
     def get(self):
-        self.write([convert_flow_to_json_dict(f) for f in self.view])
+        self.write([flow_to_json(f) for f in self.view])
 
 
 class DumpFlows(RequestHandler):
@@ -211,33 +211,29 @@ class DumpFlows(RequestHandler):
 
     def post(self):
         self.view.clear()
-
-        content = self.request.files.values()[0][0].body
-        bio = BytesIO(content)
-        self.master.load_flows(io.FlowReader(bio).stream())
+        bio = BytesIO(self.filecontents)
+        self.master.load_flows(io.FlowReader(bio))
         bio.close()
 
 
 class ClearAll(RequestHandler):
-
     def post(self):
         self.view.clear()
+        self.master.events.clear()
 
 
 class AcceptFlows(RequestHandler):
-
     def post(self):
-        self.master.accept_all(self.master)
+        for f in self.view:
+            f.resume(self.master)
 
 
 class AcceptFlow(RequestHandler):
-
     def post(self, flow_id):
         self.flow.resume(self.master)
 
 
 class FlowHandler(RequestHandler):
-
     def delete(self, flow_id):
         if self.flow.killable:
             self.flow.kill(self.master)
@@ -246,75 +242,78 @@ class FlowHandler(RequestHandler):
     def put(self, flow_id):
         flow = self.flow
         flow.backup()
-        for a, b in self.json.items():
-            if a == "request" and hasattr(flow, "request"):
-                request = flow.request
-                for k, v in b.items():
-                    if k in ["method", "scheme", "host", "path", "http_version"]:
-                        setattr(request, k, str(v))
-                    elif k == "port":
-                        request.port = int(v)
-                    elif k == "headers":
-                        request.headers.clear()
-                        for header in v:
-                            request.headers.add(*header)
-                    elif k == "content":
-                        request.text = v
-                    else:
-                        print("Warning: Unknown update {}.{}: {}".format(a, k, v))
-
-            elif a == "response" and hasattr(flow, "response"):
-                response = flow.response
-                for k, v in b.items():
-                    if k == "msg":
-                        response.msg = str(v)
-                    elif k == "code":
-                        response.status_code = int(v)
-                    elif k == "http_version":
-                        response.http_version = str(v)
-                    elif k == "headers":
-                        response.headers.clear()
-                        for header in v:
-                            response.headers.add(*header)
-                    elif k == "content":
-                        response.text = v
-                    else:
-                        print("Warning: Unknown update {}.{}: {}".format(a, k, v))
-            else:
-                print("Warning: Unknown update {}: {}".format(a, b))
+        try:
+            for a, b in self.json.items():
+                if a == "request" and hasattr(flow, "request"):
+                    request = flow.request
+                    for k, v in b.items():
+                        if k in ["method", "scheme", "host", "path", "http_version"]:
+                            setattr(request, k, str(v))
+                        elif k == "port":
+                            request.port = int(v)
+                        elif k == "headers":
+                            request.headers.clear()
+                            for header in v:
+                                request.headers.add(*header)
+                        elif k == "content":
+                            request.text = v
+                        else:
+                            raise APIError(400, "Unknown update request.{}: {}".format(k, v))
+
+                elif a == "response" and hasattr(flow, "response"):
+                    response = flow.response
+                    for k, v in b.items():
+                        if k in ["msg", "http_version"]:
+                            setattr(response, k, str(v))
+                        elif k == "code":
+                            response.status_code = int(v)
+                        elif k == "headers":
+                            response.headers.clear()
+                            for header in v:
+                                response.headers.add(*header)
+                        elif k == "content":
+                            response.text = v
+                        else:
+                            raise APIError(400, "Unknown update response.{}: {}".format(k, v))
+                else:
+                    raise APIError(400, "Unknown update {}: {}".format(a, b))
+        except APIError:
+            flow.revert()
+            raise
         self.view.update(flow)
 
 
 class DuplicateFlow(RequestHandler):
-
     def post(self, flow_id):
-        self.master.view.duplicate_flow(self.flow)
+        f = self.flow.copy()
+        self.view.add(f)
+        self.write(f.id)
 
 
 class RevertFlow(RequestHandler):
-
     def post(self, flow_id):
-        self.flow.revert()
+        if self.flow.modified():
+            self.flow.revert()
+            self.view.update(self.flow)
 
 
 class ReplayFlow(RequestHandler):
-
     def post(self, flow_id):
         self.flow.backup()
         self.flow.response = None
         self.view.update(self.flow)
 
-        r = self.master.replay_request(self.flow)
-        if r:
-            raise APIError(400, r)
+        try:
+            self.master.replay_request(self.flow)
+        except exceptions.ReplayException as e:
+            raise APIError(400, str(e))
 
 
 class FlowContent(RequestHandler):
-
     def post(self, flow_id, message):
         self.flow.backup()
         message = getattr(self.flow, message)
-        message.content = self.request.files.values()[0][0].body
+        message.content = self.filecontents
         self.view.update(self.flow)
 
     def get(self, flow_id, message):
@@ -347,15 +346,14 @@ class FlowContent(RequestHandler):
 
 
 class FlowContentView(RequestHandler):
-
     def get(self, flow_id, message, content_view):
         message = getattr(self.flow, message)
 
         description, lines, error = contentviews.get_message_content_view(
             content_view.replace('_', ' '), message
         )
-#        if error:
-#           add event log
+        #        if error:
+        #           add event log
 
         self.write(dict(
             lines=list(lines),
@@ -364,13 +362,11 @@ class FlowContentView(RequestHandler):
 
 
 class Events(RequestHandler):
-
     def get(self):
-        self.write([])  # FIXME
+        self.write([logentry_to_json(e) for e in self.master.events.data])
 
 
 class Settings(RequestHandler):
-
     def get(self):
         self.write(dict(
             version=version.VERSION,
@@ -389,51 +385,20 @@ class Settings(RequestHandler):
         ))
 
     def put(self):
-        update = {}
-        for k, v in self.json.items():
-            if k == "intercept":
-                self.master.options.intercept = v
-                update[k] = v
-            elif k == "showhost":
-                self.master.options.showhost = v
-                update[k] = v
-            elif k == "no_upstream_cert":
-                self.master.options.no_upstream_cert = v
-                update[k] = v
-            elif k == "rawtcp":
-                self.master.options.rawtcp = v
-                update[k] = v
-            elif k == "http2":
-                self.master.options.http2 = v
-                update[k] = v
-            elif k == "anticache":
-                self.master.options.anticache = v
-                update[k] = v
-            elif k == "anticomp":
-                self.master.options.anticomp = v
-                update[k] = v
-            elif k == "stickycookie":
-                self.master.options.stickycookie = v
-                update[k] = v
-            elif k == "stickyauth":
-                self.master.options.stickyauth = v
-                update[k] = v
-            elif k == "stream":
-                self.master.options.stream_large_bodies = v
-                update[k] = v
-            else:
-                print("Warning: Unknown setting {}: {}".format(k, v))
-
-        ClientConnection.broadcast(
-            resource="settings",
-            cmd="update",
-            data=update
-        )
+        update = self.json
+        option_whitelist = {
+            "intercept", "showhost", "no_upstream_cert",
+            "rawtcp", "http2", "anticache", "anticomp",
+            "stickycookie", "stickyauth", "stream_large_bodies"
+        }
+        for k in update:
+            if k not in option_whitelist:
+                raise APIError(400, "Unknown setting {}".format(k))
+        self.master.options.update(**update)
 
 
 class Application(tornado.web.Application):
-
-    def __init__(self, master, debug, wauthenticator):
+    def __init__(self, master, debug):
         self.master = master
         handlers = [
             (r"/", IndexHandler),
@@ -449,7 +414,9 @@ class Application(tornado.web.Application):
             (r"/flows/(?P<flow_id>[0-9a-f\-]+)/replay", ReplayFlow),
             (r"/flows/(?P<flow_id>[0-9a-f\-]+)/revert", RevertFlow),
             (r"/flows/(?P<flow_id>[0-9a-f\-]+)/(?P<message>request|response)/content", FlowContent),
-            (r"/flows/(?P<flow_id>[0-9a-f\-]+)/(?P<message>request|response)/content/(?P<content_view>[0-9a-zA-Z\-\_]+)", FlowContentView),
+            (
+                r"/flows/(?P<flow_id>[0-9a-f\-]+)/(?P<message>request|response)/content/(?P<content_view>[0-9a-zA-Z\-\_]+)",
+                FlowContentView),
             (r"/settings", Settings),
             (r"/clear", ClearAll),
         ]
@@ -460,6 +427,5 @@ class Application(tornado.web.Application):
             cookie_secret=os.urandom(256),
             debug=debug,
             autoreload=False,
-            wauthenticator=wauthenticator,
         )
         super().__init__(handlers, **settings)
diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py
index d2203f10..5457fb40 100644
--- a/mitmproxy/tools/web/master.py
+++ b/mitmproxy/tools/web/master.py
@@ -1,49 +1,20 @@
 import sys
 import webbrowser
+from typing import Optional
 
 import tornado.httpserver
 import tornado.ioloop
-
-from typing import Optional
-
 from mitmproxy import addons
 from mitmproxy import exceptions
-from mitmproxy.addons import view
-from mitmproxy.addons import intercept
-from mitmproxy import options
+from mitmproxy import log
 from mitmproxy import master
+from mitmproxy import options
+from mitmproxy.addons import eventstore
+from mitmproxy.addons import intercept
+from mitmproxy.addons import view
 from mitmproxy.tools.web import app
 
 
-class Stop(Exception):
-    pass
-
-
-class _WebState():
-    def add_log(self, e, level):
-        # server-side log ids are odd
-        self._last_event_id += 2
-        entry = {
-            "id": self._last_event_id,
-            "message": e,
-            "level": level
-        }
-        self.events.append(entry)
-        app.ClientConnection.broadcast(
-            resource="events",
-            cmd="add",
-            data=entry
-        )
-
-    def clear(self):
-        super().clear()
-        self.events.clear()
-        app.ClientConnection.broadcast(
-            resource="events",
-            cmd="reset"
-        )
-
-
 class Options(options.Options):
     def __init__(
             self,
@@ -52,54 +23,34 @@ class Options(options.Options):
             wdebug: bool = False,
             wport: int = 8081,
             wiface: str = "127.0.0.1",
-            # wauthenticator: Optional[authentication.PassMan] = None,
-            wsingleuser: Optional[str] = None,
-            whtpasswd: Optional[str] = None,
             **kwargs
     ) -> None:
+        self.intercept = intercept
         self.wdebug = wdebug
         self.wport = wport
         self.wiface = wiface
-        # self.wauthenticator = wauthenticator
-        # self.wsingleuser = wsingleuser
-        # self.whtpasswd = whtpasswd
-        self.intercept = intercept
         super().__init__(**kwargs)
 
-    # TODO: This doesn't belong here.
-    def process_web_options(self, parser):
-        # if self.wsingleuser or self.whtpasswd:
-        #     if self.wsingleuser:
-        #         if len(self.wsingleuser.split(':')) != 2:
-        #             return parser.error(
-        #                 "Invalid single-user specification. Please use the format username:password"
-        #             )
-        #         username, password = self.wsingleuser.split(':')
-        #         # self.wauthenticator = authentication.PassManSingleUser(username, password)
-        #     elif self.whtpasswd:
-        #         try:
-        #             self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd)
-        #         except ValueError as v:
-        #             return parser.error(v.message)
-        # else:
-        #     self.wauthenticator = None
-        pass
-
 
 class WebMaster(master.Master):
-
     def __init__(self, options, server):
         super().__init__(options, server)
         self.view = view.View()
-        self.view.sig_view_add.connect(self._sig_add)
-        self.view.sig_view_remove.connect(self._sig_remove)
-        self.view.sig_view_update.connect(self._sig_update)
-        self.view.sig_view_refresh.connect(self._sig_refresh)
+        self.view.sig_view_add.connect(self._sig_view_add)
+        self.view.sig_view_remove.connect(self._sig_view_remove)
+        self.view.sig_view_update.connect(self._sig_view_update)
+        self.view.sig_view_refresh.connect(self._sig_view_refresh)
+
+        self.events = eventstore.EventStore()
+        self.events.sig_add.connect(self._sig_events_add)
+        self.events.sig_refresh.connect(self._sig_events_refresh)
+
+        self.options.changed.connect(self._sig_options_update)
 
         self.addons.add(*addons.default_addons())
-        self.addons.add(self.view, intercept.Intercept())
+        self.addons.add(self.view, self.events, intercept.Intercept())
         self.app = app.Application(
-            self, self.options.wdebug, False
+            self, self.options.wdebug
         )
         # This line is just for type hinting
         self.options = self.options  # type: Options
@@ -112,33 +63,53 @@ class WebMaster(master.Master):
                     "error"
                 )
 
-    def _sig_add(self, view, flow):
+    def _sig_view_add(self, view, flow):
         app.ClientConnection.broadcast(
             resource="flows",
             cmd="add",
-            data=app.convert_flow_to_json_dict(flow)
+            data=app.flow_to_json(flow)
         )
 
-    def _sig_update(self, view, flow):
+    def _sig_view_update(self, view, flow):
         app.ClientConnection.broadcast(
             resource="flows",
             cmd="update",
-            data=app.convert_flow_to_json_dict(flow)
+            data=app.flow_to_json(flow)
         )
 
-    def _sig_remove(self, view, flow):
+    def _sig_view_remove(self, view, flow):
         app.ClientConnection.broadcast(
             resource="flows",
             cmd="remove",
             data=dict(id=flow.id)
         )
 
-    def _sig_refresh(self, view):
+    def _sig_view_refresh(self, view):
         app.ClientConnection.broadcast(
             resource="flows",
             cmd="reset"
         )
 
+    def _sig_events_add(self, event_store, entry: log.LogEntry):
+        app.ClientConnection.broadcast(
+            resource="events",
+            cmd="add",
+            data=app.logentry_to_json(entry)
+        )
+
+    def _sig_events_refresh(self, event_store):
+        app.ClientConnection.broadcast(
+            resource="events",
+            cmd="reset"
+        )
+
+    def _sig_options_update(self, options, updated):
+        app.ClientConnection.broadcast(
+            resource="settings",
+            cmd="update",
+            data={k: getattr(options, k) for k in updated}
+        )
+
     def run(self):  # pragma: no cover
 
         iol = tornado.ioloop.IOLoop.instance()
@@ -155,13 +126,9 @@ class WebMaster(master.Master):
                 print("No webbrowser found. Please open a browser and point it to {}".format(url))
 
             iol.start()
-        except (Stop, KeyboardInterrupt):
+        except (KeyboardInterrupt):
             self.shutdown()
 
-    # def add_log(self, e, level="info"):
-    #     super().add_log(e, level)
-    #     return self.state.add_log(e, level)
-
 
 def open_browser(url: str) -> bool:
     """
diff --git a/test/mitmproxy/addons/test_evenstore.py b/test/mitmproxy/addons/test_evenstore.py
new file mode 100644
index 00000000..78eb3287
--- /dev/null
+++ b/test/mitmproxy/addons/test_evenstore.py
@@ -0,0 +1,32 @@
+import mock
+from mitmproxy import log
+from mitmproxy.addons import eventstore
+
+
+def test_simple():
+    store = eventstore.EventStore()
+    assert not store.data
+
+    sig_add = mock.Mock(spec=lambda: 42)
+    sig_refresh = mock.Mock(spec=lambda: 42)
+    store.sig_add.connect(sig_add)
+    store.sig_refresh.connect(sig_refresh)
+
+    assert not sig_add.called
+    assert not sig_refresh.called
+
+    # test .log()
+    store.log(log.LogEntry("test", "info"))
+    assert store.data
+
+    assert sig_add.called
+    assert not sig_refresh.called
+
+    # test .clear()
+    sig_add.reset_mock()
+
+    store.clear()
+    assert not store.data
+
+    assert not sig_add.called
+    assert sig_refresh.called
diff --git a/test/mitmproxy/data/addonscripts/duplicate_flow.py b/test/mitmproxy/data/addonscripts/duplicate_flow.py
deleted file mode 100644
index 02fb8dce..00000000
--- a/test/mitmproxy/data/addonscripts/duplicate_flow.py
+++ /dev/null
@@ -1,6 +0,0 @@
-from mitmproxy import ctx
-
-
-def request(flow):
-    f = ctx.master.state.duplicate_flow(flow)
-    ctx.master.replay_request(f, block=True)
diff --git a/test/mitmproxy/test_web_app.py b/test/mitmproxy/test_web_app.py
index 8fc3378a..be195528 100644
--- a/test/mitmproxy/test_web_app.py
+++ b/test/mitmproxy/test_web_app.py
@@ -1,15 +1,47 @@
-import tornado.testing
+import json as _json
 
+import mock
+import tornado.testing
+from mitmproxy import exceptions
 from mitmproxy import proxy
+from mitmproxy.test import tflow
 from mitmproxy.tools.web import app
 from mitmproxy.tools.web import master as webmaster
+from tornado import httpclient
+from tornado import websocket
+
+
+def json(resp: httpclient.HTTPResponse):
+    return _json.loads(resp.body.decode())
 
 
 class TestApp(tornado.testing.AsyncHTTPTestCase):
     def get_app(self):
         o = webmaster.Options()
         m = webmaster.WebMaster(o, proxy.DummyServer())
-        return app.Application(m, None, None)
+        f = tflow.tflow(resp=True)
+        f.id = "42"
+        m.view.add(f)
+        m.view.add(tflow.tflow(err=True))
+        m.add_log("test log", "info")
+        self.master = m
+        self.view = m.view
+        self.events = m.events
+        webapp = app.Application(m, None)
+        webapp.settings["xsrf_cookies"] = False
+        return webapp
+
+    def fetch(self, *args, **kwargs) -> httpclient.HTTPResponse:
+        # tornado disallows POST without content by default.
+        return super().fetch(*args, **kwargs, allow_nonstandard_methods=True)
+
+    def put_json(self, url, data: dict) -> httpclient.HTTPResponse:
+        return self.fetch(
+            url,
+            method="PUT",
+            body=_json.dumps(data),
+            headers={"Content-Type": "application/json"},
+        )
 
     def test_index(self):
         assert self.fetch("/").code == 200
@@ -17,8 +49,217 @@ class TestApp(tornado.testing.AsyncHTTPTestCase):
     def test_filter_help(self):
         assert self.fetch("/filter-help").code == 200
 
+    def test_flows(self):
+        resp = self.fetch("/flows")
+        assert resp.code == 200
+        assert json(resp)[0]["request"]["contentHash"]
+        assert json(resp)[1]["error"]
+
+    def test_flows_dump(self):
+        resp = self.fetch("/flows/dump")
+        assert b"address" in resp.body
+
+        self.view.clear()
+        assert not len(self.view)
+
+        assert self.fetch("/flows/dump", method="POST", body=resp.body).code == 200
+        assert len(self.view)
+
+    def test_clear(self):
+        events = self.events.data.copy()
+        flows = list(self.view)
+
+        assert self.fetch("/clear", method="POST").code == 200
+
+        assert not len(self.view)
+        assert not len(self.events.data)
+
+        # restore
+        for f in flows:
+            self.view.add(f)
+        self.events.data = events
+
+    def test_accept(self):
+        for f in self.view:
+            f.reply.handle()
+            f.intercept(self.master)
+
+        assert self.fetch(
+            "/flows/42/accept", method="POST").code == 200
+        assert sum(f.intercepted for f in self.view) == 1
+        assert self.fetch("/flows/accept", method="POST").code == 200
+        assert all(not f.intercepted for f in self.view)
+
+    def test_flow_delete(self):
+        f = self.view.get_by_id("42")
+        assert f
+
+        f.reply.handle()
+        assert self.fetch("/flows/42", method="DELETE").code == 200
+
+        assert not self.view.get_by_id("42")
+        self.view.add(f)
+
+        assert self.fetch("/flows/1234", method="DELETE").code == 404
+
+    def test_flow_update(self):
+        f = self.view.get_by_id("42")
+        assert f.request.method == "GET"
+        f.backup()
+
+        upd = {
+            "request": {
+                "method": "PATCH",
+                "port": 123,
+                "headers": [("foo", "bar")],
+                "content": "req",
+            },
+            "response": {
+                "msg": "Not Found",
+                "code": 404,
+                "headers": [("bar", "baz")],
+                "content": "resp",
+            }
+        }
+        assert self.put_json("/flows/42", upd).code == 200
+        assert f.request.method == "PATCH"
+        assert f.request.port == 123
+        assert f.request.headers["foo"] == "bar"
+        assert f.request.text == "req"
+        assert f.response.msg == "Not Found"
+        assert f.response.status_code == 404
+        assert f.response.headers["bar"] == "baz"
+        assert f.response.text == "resp"
+
+        f.revert()
+
+        assert self.put_json("/flows/42", {"foo": 42}).code == 400
+        assert self.put_json("/flows/42", {"request": {"foo": 42}}).code == 400
+        assert self.put_json("/flows/42", {"response": {"foo": 42}}).code == 400
+        assert self.fetch("/flows/42", method="PUT", body="{}").code == 400
+        assert self.fetch(
+            "/flows/42",
+            method="PUT",
+            headers={"Content-Type": "application/json"},
+            body="!!"
+        ).code == 400
+
+    def test_flow_duplicate(self):
+        resp = self.fetch("/flows/42/duplicate", method="POST")
+        assert resp.code == 200
+        f = self.view.get_by_id(resp.body.decode())
+        assert f
+        assert f.id != "42"
+        self.view.remove(f)
+
+    def test_flow_revert(self):
+        f = self.view.get_by_id("42")
+        f.backup()
+        f.request.method = "PATCH"
+        self.fetch("/flows/42/revert", method="POST")
+        assert not f._backup
+
+    def test_flow_replay(self):
+        with mock.patch("mitmproxy.master.Master.replay_request") as replay_request:
+            assert self.fetch("/flows/42/replay", method="POST").code == 200
+            assert replay_request.called
+            replay_request.side_effect = exceptions.ReplayException(
+                "out of replays"
+            )
+            assert self.fetch("/flows/42/replay", method="POST").code == 400
+
+    def test_flow_content(self):
+        f = self.view.get_by_id("42")
+        f.backup()
+        f.response.headers["Content-Encoding"] = "ran\x00dom"
+        f.response.headers["Content-Disposition"] = 'inline; filename="filename.jpg"'
+
+        r = self.fetch("/flows/42/response/content")
+        assert r.body == b"message"
+        assert r.headers["Content-Encoding"] == "random"
+        assert r.headers["Content-Disposition"] == 'attachment; filename="filename.jpg"'
+
+        del f.response.headers["Content-Disposition"]
+        f.request.path = "/foo/bar.jpg"
+        assert self.fetch(
+            "/flows/42/response/content"
+        ).headers["Content-Disposition"] == 'attachment; filename=bar.jpg'
+
+        f.response.content = b""
+        assert self.fetch("/flows/42/response/content").code == 400
+
+        f.revert()
+
+    def test_update_flow_content(self):
+        assert self.fetch(
+            "/flows/42/request/content",
+            method="POST",
+            body="new"
+        ).code == 200
+        f = self.view.get_by_id("42")
+        assert f.request.content == b"new"
+        assert f.modified()
+        f.revert()
+
+    def test_update_flow_content_multipart(self):
+        body = (
+            b'--somefancyboundary\r\n'
+            b'Content-Disposition: form-data; name="a"; filename="a.txt"\r\n'
+            b'\r\n'
+            b'such multipart. very wow.\r\n'
+            b'--somefancyboundary--\r\n'
+        )
+        assert self.fetch(
+            "/flows/42/request/content",
+            method="POST",
+            headers={"Content-Type": 'multipart/form-data; boundary="somefancyboundary"'},
+            body=body
+        ).code == 200
+        f = self.view.get_by_id("42")
+        assert f.request.content == b"such multipart. very wow."
+        assert f.modified()
+        f.revert()
+
+    def test_flow_content_view(self):
+        assert json(self.fetch("/flows/42/request/content/raw")) == {
+            "lines": [
+                [["text", "content"]]
+            ],
+            "description": "Raw"
+        }
+
     def test_events(self):
-        assert self.fetch("/events").code == 200
+        resp = self.fetch("/events")
+        assert resp.code == 200
+        assert json(resp)[0]["level"] == "info"
 
-    def test_flows(self):
-        assert self.fetch("/flows").code == 200
+    def test_settings(self):
+        assert json(self.fetch("/settings"))["mode"] == "regular"
+
+    def test_settings_update(self):
+        assert self.put_json("/settings", {"anticache": True}).code == 200
+        assert self.put_json("/settings", {"wtf": True}).code == 400
+
+    def test_err(self):
+        with mock.patch("mitmproxy.tools.web.app.IndexHandler.get") as f:
+            f.side_effect = RuntimeError
+            assert self.fetch("/").code == 500
+
+    @tornado.testing.gen_test
+    def test_websocket(self):
+        ws_url = "ws://localhost:{}/updates".format(self.get_http_port())
+
+        ws_client = yield websocket.websocket_connect(ws_url)
+        self.master.options.anticomp = True
+
+        response = yield ws_client.read_message()
+        assert _json.loads(response) == {
+            "resource": "settings",
+            "cmd": "update",
+            "data": {"anticomp": True},
+        }
+        ws_client.close()
+
+        # trigger on_close by opening a second connection.
+        ws_client2 = yield websocket.websocket_connect(ws_url)
+        ws_client2.close()
diff --git a/web/src/js/components/ContentView/ContentViewOptions.jsx b/web/src/js/components/ContentView/ContentViewOptions.jsx
index fed3a088..6bc66db2 100644
--- a/web/src/js/components/ContentView/ContentViewOptions.jsx
+++ b/web/src/js/components/ContentView/ContentViewOptions.jsx
@@ -9,15 +9,14 @@ ContentViewOptions.propTypes = {
     message: React.PropTypes.object.isRequired,
 }
 
-function ContentViewOptions(props) {
-    const { flow, message, uploadContent, readonly, contentViewDescription } = props
+function ContentViewOptions({ flow, message, uploadContent, readonly, contentViewDescription }) {
     return (
         <div className="view-options">
             <ViewSelector message={message}/>
             &nbsp;
             <DownloadContentButton flow={flow} message={message}/>
             &nbsp;
-            <UploadContentButton uploadContent={uploadContent}/>
+            {!readonly && <UploadContentButton uploadContent={uploadContent}/> }
             &nbsp;
             <span>{contentViewDescription}</span>
         </div>
@@ -26,6 +25,7 @@ function ContentViewOptions(props) {
 
 export default connect(
     state => ({
-        contentViewDescription: state.ui.flow.viewDescription
+        contentViewDescription: state.ui.flow.viewDescription,
+        readonly: !state.ui.flow.modifiedFlow,
     })
 )(ContentViewOptions)
diff --git a/web/src/js/components/ContentView/ContentViews.jsx b/web/src/js/components/ContentView/ContentViews.jsx
index 32a07564..db239195 100644
--- a/web/src/js/components/ContentView/ContentViews.jsx
+++ b/web/src/js/components/ContentView/ContentViews.jsx
@@ -63,6 +63,7 @@ class ViewServer extends Component {
         let lines = this.props.showFullContent ? this.data.lines : this.data.lines.slice(0, maxLines)
         return (
             <div>
+                {ViewImage.matches(message) && <ViewImage {...this.props} />}
                 <pre>
                     {lines.map((line, i) =>
                         <div key={`line${i}`}>
@@ -77,9 +78,6 @@ class ViewServer extends Component {
                         </div>
                     )}
                 </pre>
-                {ViewImage.matches(message) &&
-                <ViewImage {...this.props} />
-                }
             </div>
         )
     }
diff --git a/web/src/js/components/ContentView/ViewSelector.jsx b/web/src/js/components/ContentView/ViewSelector.jsx
index ab433ea3..fcdc3ee3 100644
--- a/web/src/js/components/ContentView/ViewSelector.jsx
+++ b/web/src/js/components/ContentView/ViewSelector.jsx
@@ -14,7 +14,7 @@ ViewSelector.propTypes = {
 
 function ViewSelector ({contentViews, activeView, isEdit, setContentView}){
     let edit = ContentViews.Edit.displayName
-    let inner = <span> <b>View:</b> {activeView}<span className="caret"></span> </span>
+    let inner = <span> <b>View:</b> {activeView} <span className="caret"></span> </span>
 
     return (
         <Dropdown dropup className="pull-left" btnClass="btn btn-default btn-xs" text={inner}>
diff --git a/web/src/js/components/EventLog.jsx b/web/src/js/components/EventLog.jsx
index 636e3e9a..1a449511 100644
--- a/web/src/js/components/EventLog.jsx
+++ b/web/src/js/components/EventLog.jsx
@@ -70,7 +70,7 @@ class EventLog extends Component {
 export default connect(
     state => ({
         filters: state.eventLog.filters,
-        events: state.eventLog.view.data,
+        events: state.eventLog.view,
     }),
     {
         close: toggleVisibility,
diff --git a/web/src/js/components/Footer.jsx b/web/src/js/components/Footer.jsx
index 96e7b7db..1ae4ee73 100644
--- a/web/src/js/components/Footer.jsx
+++ b/web/src/js/components/Footer.jsx
@@ -7,7 +7,7 @@ Footer.propTypes = {
 }
 
 function Footer({ settings }) {
-    let {mode, intercept, showhost, no_upstream_cert, rawtcp, http2, anticache, anticomp, stickyauth, stickycookie, stream} = settings;
+    let {mode, intercept, showhost, no_upstream_cert, rawtcp, http2, anticache, anticomp, stickyauth, stickycookie, stream_large_bodies} = settings;
     return (
         <footer>
             {mode && mode != "regular" && (
@@ -40,8 +40,8 @@ function Footer({ settings }) {
             {stickycookie && (
                 <span className="label label-success">stickycookie: {stickycookie}</span>
             )}
-            {stream && (
-                <span className="label label-success">stream: {formatSize(stream)}</span>
+            {stream_large_bodies && (
+                <span className="label label-success">stream: {formatSize(stream_large_bodies)}</span>
             )}
         </footer>
     )
diff --git a/web/src/js/components/Header/OptionMenu.jsx b/web/src/js/components/Header/OptionMenu.jsx
index a11062f2..186a9c6a 100644
--- a/web/src/js/components/Header/OptionMenu.jsx
+++ b/web/src/js/components/Header/OptionMenu.jsx
@@ -49,11 +49,11 @@ function OptionMenu({ settings, updateSettings }) {
                     txt={settings.stickycookie}
                     onToggleChanged={txt => updateSettings({ stickycookie: !settings.stickycookie ? txt : null })}
                 />
-                <ToggleInputButton name="stream" placeholder="stream..."
-                    checked={!!settings.stream}
-                    txt={settings.stream}
+                <ToggleInputButton name="stream_large_bodies" placeholder="stream..."
+                    checked={!!settings.stream_large_bodies}
+                    txt={settings.stream_large_bodies}
                     inputType="number"
-                    onToggleChanged={txt => updateSettings({ stream: !settings.stream ? txt : null })}
+                    onToggleChanged={txt => updateSettings({ stream_large_bodies: !settings.stream_large_bodies ? txt : null })}
                 />
             </div>
             <div className="clearfix"/>
diff --git a/web/src/js/ducks/eventLog.js b/web/src/js/ducks/eventLog.js
index 776e4b08..73eaf2e8 100644
--- a/web/src/js/ducks/eventLog.js
+++ b/web/src/js/ducks/eventLog.js
@@ -49,14 +49,12 @@ export function toggleVisibility() {
     return { type: TOGGLE_VISIBILITY }
 }
 
-let logId = 1  // client-side log ids are odd
 export function add(message, level = 'web') {
     let data = {
-        id: logId,
+        id: Math.random().toString(),
         message,
         level,
     }
-    logId += 2
     return {
         type: ADD,
         cmd: "add",