Merge pull request #1356 from cortesi/script

Scripts to addon

17 files changed, 402 insertions, 348 deletions

diff --git a/test/mitmproxy/builtins/test_script.py b/test/mitmproxy/builtins/test_script.py
new file mode 100644
index 00000000..da60b64c
--- /dev/null
+++ b/test/mitmproxy/builtins/test_script.py
@@ -0,0 +1,187 @@
+import time
+
+from mitmproxy.builtins import script
+from mitmproxy import exceptions
+from mitmproxy.flow import master
+from mitmproxy.flow import state
+from mitmproxy.flow import options
+
+from .. import tutils, mastertest
+
+
+class TestParseCommand:
+    def test_empty_command(self):
+        with tutils.raises(exceptions.AddonError):
+            script.parse_command("")
+
+        with tutils.raises(exceptions.AddonError):
+            script.parse_command("  ")
+
+    def test_no_script_file(self):
+        with tutils.raises("not found"):
+            script.parse_command("notfound")
+
+        with tutils.tmpdir() as dir:
+            with tutils.raises("not a file"):
+                script.parse_command(dir)
+
+    def test_parse_args(self):
+        with tutils.chdir(tutils.test_data.dirname):
+            assert script.parse_command("data/scripts/a.py") == ("data/scripts/a.py", [])
+            assert script.parse_command("data/scripts/a.py foo bar") == ("data/scripts/a.py", ["foo", "bar"])
+            assert script.parse_command("data/scripts/a.py 'foo bar'") == ("data/scripts/a.py", ["foo bar"])
+
+    @tutils.skip_not_windows
+    def test_parse_windows(self):
+        with tutils.chdir(tutils.test_data.dirname):
+            assert script.parse_command("data\\scripts\\a.py") == ("data\\scripts\\a.py", [])
+            assert script.parse_command("data\\scripts\\a.py 'foo \\ bar'") == ("data\\scripts\\a.py", ['foo \\ bar'])
+
+
+def test_load_script():
+    ns = script.load_script(
+        tutils.test_data.path(
+            "data/addonscripts/recorder.py"
+        ), []
+    )
+    assert ns["configure"]
+
+
+class TestScript(mastertest.MasterTest):
+    def test_simple(self):
+        s = state.State()
+        m = master.FlowMaster(options.Options(), None, s)
+        sc = script.Script(
+            tutils.test_data.path(
+                "data/addonscripts/recorder.py"
+            )
+        )
+        m.addons.add(sc)
+        assert sc.ns["call_log"] == [
+            ("solo", "start", (), {}),
+            ("solo", "configure", (options.Options(),), {})
+        ]
+
+        sc.ns["call_log"] = []
+        f = tutils.tflow(resp=True)
+        self.invoke(m, "request", f)
+
+        recf = sc.ns["call_log"][0]
+        assert recf[1] == "request"
+
+    def test_reload(self):
+        s = state.State()
+        m = mastertest.RecordingMaster(options.Options(), None, s)
+        with tutils.tmpdir():
+            with open("foo.py", "w"):
+                pass
+            sc = script.Script("foo.py")
+            m.addons.add(sc)
+
+            for _ in range(100):
+                with open("foo.py", "a") as f:
+                    f.write(".")
+                m.addons.invoke_with_context(sc, "tick")
+                time.sleep(0.1)
+                if m.event_log:
+                    return
+            raise AssertionError("Change event not detected.")
+
+    def test_exception(self):
+        s = state.State()
+        m = mastertest.RecordingMaster(options.Options(), None, s)
+        sc = script.Script(
+            tutils.test_data.path("data/addonscripts/error.py")
+        )
+        m.addons.add(sc)
+        f = tutils.tflow(resp=True)
+        self.invoke(m, "request", f)
+        assert m.event_log[0][0] == "error"
+
+    def test_duplicate_flow(self):
+        s = state.State()
+        fm = master.FlowMaster(None, None, s)
+        fm.addons.add(
+            script.Script(
+                tutils.test_data.path("data/addonscripts/duplicate_flow.py")
+            )
+        )
+        f = tutils.tflow()
+        fm.request(f)
+        assert fm.state.flow_count() == 2
+        assert not fm.state.view[0].request.is_replay
+        assert fm.state.view[1].request.is_replay
+
+
+class TestScriptLoader(mastertest.MasterTest):
+    def test_simple(self):
+        s = state.State()
+        o = options.Options(scripts=[])
+        m = master.FlowMaster(o, None, s)
+        sc = script.ScriptLoader()
+        m.addons.add(sc)
+        assert len(m.addons) == 1
+        o.update(
+            scripts = [
+                tutils.test_data.path("data/addonscripts/recorder.py")
+            ]
+        )
+        assert len(m.addons) == 2
+        o.update(scripts = [])
+        assert len(m.addons) == 1
+
+    def test_dupes(self):
+        s = state.State()
+        o = options.Options(scripts=["one", "one"])
+        m = master.FlowMaster(o, None, s)
+        sc = script.ScriptLoader()
+        tutils.raises(exceptions.OptionsError, m.addons.add, sc)
+
+    def test_order(self):
+        rec = tutils.test_data.path("data/addonscripts/recorder.py")
+
+        s = state.State()
+        o = options.Options(
+            scripts = [
+                "%s %s" % (rec, "a"),
+                "%s %s" % (rec, "b"),
+                "%s %s" % (rec, "c"),
+            ]
+        )
+        m = mastertest.RecordingMaster(o, None, s)
+        sc = script.ScriptLoader()
+        m.addons.add(sc)
+
+        debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"]
+        assert debug == [
+            ('debug', 'a start'), ('debug', 'a configure'),
+            ('debug', 'b start'), ('debug', 'b configure'),
+            ('debug', 'c start'), ('debug', 'c configure')
+        ]
+        m.event_log[:] = []
+
+        o.scripts = [
+            "%s %s" % (rec, "c"),
+            "%s %s" % (rec, "a"),
+            "%s %s" % (rec, "b"),
+        ]
+        debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"]
+        assert debug == [
+            ('debug', 'c configure'),
+            ('debug', 'a configure'),
+            ('debug', 'b configure'),
+        ]
+        m.event_log[:] = []
+
+        o.scripts = [
+            "%s %s" % (rec, "x"),
+            "%s %s" % (rec, "a"),
+        ]
+        debug = [(i[0], i[1]) for i in m.event_log if i[0] == "debug"]
+        assert debug == [
+            ('debug', 'c done'),
+            ('debug', 'b done'),
+            ('debug', 'x start'),
+            ('debug', 'x configure'),
+            ('debug', 'a configure'),
+        ]
diff --git a/test/mitmproxy/data/scripts/concurrent_decorator.py b/test/mitmproxy/data/addonscripts/concurrent_decorator.py
index 162c00f4..a56c2af1 100644
--- a/test/mitmproxy/data/scripts/concurrent_decorator.py
+++ b/test/mitmproxy/data/addonscripts/concurrent_decorator.py
@@ -1,7 +1,6 @@
 import time
 from mitmproxy.script import concurrent
 
-
 @concurrent
 def request(flow):
     time.sleep(0.1)
diff --git a/test/mitmproxy/data/scripts/concurrent_decorator_err.py b/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py
index 756869c8..756869c8 100644
--- a/test/mitmproxy/data/scripts/concurrent_decorator_err.py
+++ b/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py
diff --git a/test/mitmproxy/data/addonscripts/duplicate_flow.py b/test/mitmproxy/data/addonscripts/duplicate_flow.py
new file mode 100644
index 00000000..b466423c
--- /dev/null
+++ b/test/mitmproxy/data/addonscripts/duplicate_flow.py
@@ -0,0 +1,6 @@
+from mitmproxy import ctx
+
+
+def request(flow):
+    f = ctx.master.duplicate_flow(flow)
+    ctx.master.replay_request(f, block=True)
diff --git a/test/mitmproxy/data/addonscripts/error.py b/test/mitmproxy/data/addonscripts/error.py
new file mode 100644
index 00000000..8ece9fce
--- /dev/null
+++ b/test/mitmproxy/data/addonscripts/error.py
@@ -0,0 +1,7 @@
+
+def mkerr():
+    raise ValueError("Error!")
+
+
+def request(flow):
+    mkerr()
diff --git a/test/mitmproxy/data/addonscripts/recorder.py b/test/mitmproxy/data/addonscripts/recorder.py
new file mode 100644
index 00000000..b6ac8d89
--- /dev/null
+++ b/test/mitmproxy/data/addonscripts/recorder.py
@@ -0,0 +1,25 @@
+from mitmproxy import controller
+from mitmproxy import ctx
+import sys
+
+call_log = []
+
+if len(sys.argv) > 1:
+    name = sys.argv[1]
+else:
+    name = "solo"
+
+# Keep a log of all possible event calls
+evts = list(controller.Events) + ["configure"]
+for i in evts:
+    def mkprox():
+        evt = i
+
+        def prox(*args, **kwargs):
+            lg = (name, evt, args, kwargs)
+            if evt != "log":
+                ctx.log.info(str(lg))
+            call_log.append(lg)
+            ctx.log.debug("%s %s" % (name, evt))
+        return prox
+    globals()[i] = mkprox()
diff --git a/test/mitmproxy/data/addonscripts/stream_modify.py b/test/mitmproxy/data/addonscripts/stream_modify.py
new file mode 100644
index 00000000..bc616342
--- /dev/null
+++ b/test/mitmproxy/data/addonscripts/stream_modify.py
@@ -0,0 +1,8 @@
+
+def modify(chunks):
+    for chunk in chunks:
+        yield chunk.replace(b"foo", b"bar")
+
+
+def responseheaders(flow):
+    flow.response.stream = modify
diff --git a/test/mitmproxy/data/addonscripts/tcp_stream_modify.py b/test/mitmproxy/data/addonscripts/tcp_stream_modify.py
new file mode 100644
index 00000000..af4ccf7e
--- /dev/null
+++ b/test/mitmproxy/data/addonscripts/tcp_stream_modify.py
@@ -0,0 +1,5 @@
+
+def tcp_message(flow):
+    message = flow.messages[-1]
+    if not message.from_client:
+        message.content = message.content.replace(b"foo", b"bar")
diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py
index 9754d3a9..240f6a73 100644
--- a/test/mitmproxy/mastertest.py
+++ b/test/mitmproxy/mastertest.py
@@ -3,6 +3,7 @@ import mock
 from . import tutils
 import netlib.tutils
 
+from mitmproxy.flow import master
 from mitmproxy import flow, proxy, models, controller
 
 
@@ -39,3 +40,12 @@ class MasterTest:
         t = tutils.tflow(resp=True)
         fw.add(t)
         f.close()
+
+
+class RecordingMaster(master.FlowMaster):
+    def __init__(self, *args, **kwargs):
+        master.FlowMaster.__init__(self, *args, **kwargs)
+        self.event_log = []
+
+    def add_event(self, e, level):
+        self.event_log.append((level, e))
diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py
index 57eeca19..080746e8 100644
--- a/test/mitmproxy/script/test_concurrent.py
+++ b/test/mitmproxy/script/test_concurrent.py
@@ -1,28 +1,46 @@
-from mitmproxy.script import Script
-from test.mitmproxy import tutils
+from test.mitmproxy import tutils, mastertest
 from mitmproxy import controller
+from mitmproxy.builtins import script
+from mitmproxy import options
+from mitmproxy.flow import master
+from mitmproxy.flow import state
 import time
 
 
 class Thing:
     def __init__(self):
         self.reply = controller.DummyReply()
+        self.live = True
 
 
-@tutils.skip_appveyor
-def test_concurrent():
-    with Script(tutils.test_data.path("data/scripts/concurrent_decorator.py")) as s:
-        f1, f2 = Thing(), Thing()
-        s.run("request", f1)
-        s.run("request", f2)
+class TestConcurrent(mastertest.MasterTest):
+    @tutils.skip_appveyor
+    def test_concurrent(self):
+        s = state.State()
+        m = master.FlowMaster(options.Options(), None, s)
+        sc = script.Script(
+            tutils.test_data.path(
+                "data/addonscripts/concurrent_decorator.py"
+            )
+        )
+        m.addons.add(sc)
+        f1, f2 = tutils.tflow(), tutils.tflow()
+        self.invoke(m, "request", f1)
+        self.invoke(m, "request", f2)
         start = time.time()
         while time.time() - start < 5:
             if f1.reply.acked and f2.reply.acked:
                 return
         raise ValueError("Script never acked")
 
-
-def test_concurrent_err():
-    s = Script(tutils.test_data.path("data/scripts/concurrent_decorator_err.py"))
-    with tutils.raises("Concurrent decorator not supported for 'start' method"):
-        s.load()
+    def test_concurrent_err(self):
+        s = state.State()
+        m = mastertest.RecordingMaster(options.Options(), None, s)
+        sc = script.Script(
+            tutils.test_data.path(
+                "data/addonscripts/concurrent_decorator_err.py"
+            )
+        )
+        with m.handlecontext():
+            sc.start()
+        assert "decorator not supported" in m.event_log[0][1]
diff --git a/test/mitmproxy/script/test_reloader.py b/test/mitmproxy/script/test_reloader.py
deleted file mode 100644
index e33903b9..00000000
--- a/test/mitmproxy/script/test_reloader.py
+++ /dev/null
@@ -1,34 +0,0 @@
-import mock
-from mitmproxy.script.reloader import watch, unwatch
-from test.mitmproxy import tutils
-from threading import Event
-
-
-def test_simple():
-    with tutils.tmpdir():
-        with open("foo.py", "w"):
-            pass
-
-        script = mock.Mock()
-        script.path = "foo.py"
-
-        e = Event()
-
-        def _onchange():
-            e.set()
-
-        watch(script, _onchange)
-        with tutils.raises("already observed"):
-            watch(script, _onchange)
-
-        # Some reloaders don't register a change directly after watching, because they first need to initialize.
-        # To test if watching works at all, we do repeated writes every 100ms.
-        for _ in range(100):
-            with open("foo.py", "a") as f:
-                f.write(".")
-            if e.wait(0.1):
-                break
-        else:
-            raise AssertionError("No change detected.")
-
-        unwatch(script)
diff --git a/test/mitmproxy/script/test_script.py b/test/mitmproxy/script/test_script.py
deleted file mode 100644
index 48fe65c9..00000000
--- a/test/mitmproxy/script/test_script.py
+++ /dev/null
@@ -1,83 +0,0 @@
-from mitmproxy.script import Script
-from mitmproxy.exceptions import ScriptException
-from test.mitmproxy import tutils
-
-
-class TestParseCommand:
-    def test_empty_command(self):
-        with tutils.raises(ScriptException):
-            Script.parse_command("")
-
-        with tutils.raises(ScriptException):
-            Script.parse_command("  ")
-
-    def test_no_script_file(self):
-        with tutils.raises("not found"):
-            Script.parse_command("notfound")
-
-        with tutils.tmpdir() as dir:
-            with tutils.raises("not a file"):
-                Script.parse_command(dir)
-
-    def test_parse_args(self):
-        with tutils.chdir(tutils.test_data.dirname):
-            assert Script.parse_command("data/scripts/a.py") == ("data/scripts/a.py", [])
-            assert Script.parse_command("data/scripts/a.py foo bar") == ("data/scripts/a.py", ["foo", "bar"])
-            assert Script.parse_command("data/scripts/a.py 'foo bar'") == ("data/scripts/a.py", ["foo bar"])
-
-    @tutils.skip_not_windows
-    def test_parse_windows(self):
-        with tutils.chdir(tutils.test_data.dirname):
-            assert Script.parse_command("data\\scripts\\a.py") == ("data\\scripts\\a.py", [])
-            assert Script.parse_command("data\\scripts\\a.py 'foo \\ bar'") == ("data\\scripts\\a.py", ['foo \\ bar'])
-
-
-def test_simple():
-    with tutils.chdir(tutils.test_data.path("data/scripts")):
-        s = Script("a.py --var 42")
-        assert s.path == "a.py"
-        assert s.ns is None
-
-        s.load()
-        assert s.ns["var"] == 42
-
-        s.run("here")
-        assert s.ns["var"] == 43
-
-        s.unload()
-        assert s.ns is None
-
-        with tutils.raises(ScriptException):
-            s.run("here")
-
-        with Script("a.py --var 42") as s:
-            s.run("here")
-
-
-def test_script_exception():
-    with tutils.chdir(tutils.test_data.path("data/scripts")):
-        s = Script("syntaxerr.py")
-        with tutils.raises(ScriptException):
-            s.load()
-
-        s = Script("starterr.py")
-        with tutils.raises(ScriptException):
-            s.load()
-
-        s = Script("a.py")
-        s.load()
-        with tutils.raises(ScriptException):
-            s.load()
-
-        s = Script("a.py")
-        with tutils.raises(ScriptException):
-            s.run("here")
-
-        with tutils.raises(ScriptException):
-            with Script("reqerr.py") as s:
-                s.run("request", None)
-
-        s = Script("unloaderr.py")
-        s.load()
-        with tutils.raises(ScriptException):
-            s.unload()
diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py
index 9686be84..201386e3 100644
--- a/test/mitmproxy/test_dump.py
+++ b/test/mitmproxy/test_dump.py
@@ -245,12 +245,12 @@ class TestDumpMaster(mastertest.MasterTest):
         assert "XRESPONSE" in ret
         assert "XCLIENTDISCONNECT" in ret
         tutils.raises(
-            dump.DumpError,
+            exceptions.AddonError,
             self.mkmaster,
             None, scripts=["nonexistent"]
         )
         tutils.raises(
-            dump.DumpError,
+            exceptions.AddonError,
             self.mkmaster,
             None, scripts=["starterr.py"]
         )
diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py
index bdadcd11..f8646336 100644
--- a/test/mitmproxy/test_examples.py
+++ b/test/mitmproxy/test_examples.py
@@ -1,151 +1,126 @@
-import glob
 import json
-import mock
-import os
-import sys
-from contextlib import contextmanager
 
-from mitmproxy import script
+import six
+import sys
+import os.path
+from mitmproxy.flow import master
+from mitmproxy.flow import state
+from mitmproxy import options
+from mitmproxy import contentviews
+from mitmproxy.builtins import script
 import netlib.utils
 from netlib import tutils as netutils
 from netlib.http import Headers
-from . import tutils
-
-example_dir = netlib.utils.Data(__name__).path("../../examples")
-
-
-@contextmanager
-def example(command):
-    command = os.path.join(example_dir, command)
-    with script.Script(command) as s:
-        yield s
-
-
-@mock.patch("mitmproxy.ctx.master")
-@mock.patch("mitmproxy.ctx.log")
-def test_load_scripts(log, master):
-    scripts = glob.glob("%s/*.py" % example_dir)
-
-    for f in scripts:
-        if "har_extractor" in f:
-            continue
-        if "flowwriter" in f:
-            f += " -"
-        if "iframe_injector" in f:
-            f += " foo"  # one argument required
-        if "filt" in f:
-            f += " ~a"
-        if "modify_response_body" in f:
-            f += " foo bar"  # two arguments required
-
-        s = script.Script(f)
-        try:
-            s.load()
-        except Exception as v:
-            if "ImportError" not in str(v):
-                raise
-        else:
-            s.unload()
-
-
-def test_add_header():
-    flow = tutils.tflow(resp=netutils.tresp())
-    with example("add_header.py") as ex:
-        ex.run("response", flow)
-        assert flow.response.headers["newheader"] == "foo"
-
-
-@mock.patch("mitmproxy.contentviews.remove")
-@mock.patch("mitmproxy.contentviews.add")
-def test_custom_contentviews(add, remove):
-    with example("custom_contentviews.py"):
-        assert add.called
-        pig = add.call_args[0][0]
-        _, fmt = pig(b"<html>test!</html>")
-        assert any(b'esttay!' in val[0][1] for val in fmt)
-        assert not pig(b"gobbledygook")
-    assert remove.called
-
-
-def test_iframe_injector():
-    with tutils.raises(script.ScriptException):
-        with example("iframe_injector.py"):
-            pass
-
-    flow = tutils.tflow(resp=netutils.tresp(content=b"<html>mitmproxy</html>"))
-    with example("iframe_injector.py http://example.org/evil_iframe") as ex:
-        ex.run("response", flow)
-        content = flow.response.content
-        assert b'iframe' in content and b'evil_iframe' in content
+from . import tutils, mastertest
 
+example_dir = netlib.utils.Data(__name__).push("../../examples")
 
-def test_modify_form():
-    form_header = Headers(content_type="application/x-www-form-urlencoded")
-    flow = tutils.tflow(req=netutils.treq(headers=form_header))
-    with example("modify_form.py") as ex:
-        ex.run("request", flow)
-        assert flow.request.urlencoded_form[b"mitmproxy"] == b"rocks"
 
-        flow.request.headers["content-type"] = ""
-        ex.run("request", flow)
-        assert list(flow.request.urlencoded_form.items()) == [(b"foo", b"bar")]
+class ScriptError(Exception):
+    pass
 
 
-def test_modify_querystring():
-    flow = tutils.tflow(req=netutils.treq(path=b"/search?q=term"))
-    with example("modify_querystring.py") as ex:
-        ex.run("request", flow)
-        assert flow.request.query["mitmproxy"] == "rocks"
+class RaiseMaster(master.FlowMaster):
+    def add_event(self, e, level):
+        if level in ("warn", "error"):
+            raise ScriptError(e)
 
-        flow.request.path = "/"
-        ex.run("request", flow)
-        assert flow.request.query["mitmproxy"] == "rocks"
 
+def tscript(cmd, args=""):
+    cmd = example_dir.path(cmd) + " " + args
+    m = RaiseMaster(options.Options(), None, state.State())
+    sc = script.Script(cmd)
+    m.addons.add(sc)
+    return m, sc
 
-def test_modify_response_body():
-    with tutils.raises(script.ScriptException):
-        with example("modify_response_body.py"):
-            assert True
 
-    flow = tutils.tflow(resp=netutils.tresp(content=b"I <3 mitmproxy"))
-    with example("modify_response_body.py mitmproxy rocks") as ex:
-        assert ex.ns["state"]["old"] == b"mitmproxy" and ex.ns["state"]["new"] == b"rocks"
-        ex.run("response", flow)
-        assert flow.response.content == b"I <3 rocks"
+class TestScripts(mastertest.MasterTest):
+    def test_add_header(self):
+        m, _ = tscript("add_header.py")
+        f = tutils.tflow(resp=netutils.tresp())
+        self.invoke(m, "response", f)
+        assert f.response.headers["newheader"] == "foo"
 
+    def test_custom_contentviews(self):
+        m, sc = tscript("custom_contentviews.py")
+        pig = contentviews.get("pig_latin_HTML")
+        _, fmt = pig(b"<html>test!</html>")
+        assert any(b'esttay!' in val[0][1] for val in fmt)
+        assert not pig(b"gobbledygook")
 
-def test_redirect_requests():
-    flow = tutils.tflow(req=netutils.treq(host=b"example.org"))
-    with example("redirect_requests.py") as ex:
-        ex.run("request", flow)
-        assert flow.request.host == "mitmproxy.org"
-
-
-@mock.patch("mitmproxy.ctx.log")
-def test_har_extractor(log):
-    if sys.version_info >= (3, 0):
-        with tutils.raises("does not work on Python 3"):
-            with example("har_extractor.py -"):
-                pass
-        return
-
-    with tutils.raises(script.ScriptException):
-        with example("har_extractor.py"):
-            pass
-
-    times = dict(
-        timestamp_start=746203272,
-        timestamp_end=746203272,
-    )
-
-    flow = tutils.tflow(
-        req=netutils.treq(**times),
-        resp=netutils.tresp(**times)
-    )
+    def test_iframe_injector(self):
+        with tutils.raises(ScriptError):
+            tscript("iframe_injector.py")
 
-    with example("har_extractor.py -") as ex:
-        ex.run("response", flow)
+        m, sc = tscript("iframe_injector.py", "http://example.org/evil_iframe")
+        flow = tutils.tflow(resp=netutils.tresp(content=b"<html>mitmproxy</html>"))
+        self.invoke(m, "response", flow)
+        content = flow.response.content
+        assert b'iframe' in content and b'evil_iframe' in content
 
-        with open(tutils.test_data.path("data/har_extractor.har")) as fp:
-            test_data = json.load(fp)
-            assert json.loads(ex.ns["context"].HARLog.json()) == test_data["test_response"]
+    def test_modify_form(self):
+        m, sc = tscript("modify_form.py")
+
+        form_header = Headers(content_type="application/x-www-form-urlencoded")
+        f = tutils.tflow(req=netutils.treq(headers=form_header))
+        self.invoke(m, "request", f)
+
+        assert f.request.urlencoded_form[b"mitmproxy"] == b"rocks"
+
+        f.request.headers["content-type"] = ""
+        self.invoke(m, "request", f)
+        assert list(f.request.urlencoded_form.items()) == [(b"foo", b"bar")]
+
+    def test_modify_querystring(self):
+        m, sc = tscript("modify_querystring.py")
+        f = tutils.tflow(req=netutils.treq(path="/search?q=term"))
+
+        self.invoke(m, "request", f)
+        assert f.request.query["mitmproxy"] == "rocks"
+
+        f.request.path = "/"
+        self.invoke(m, "request", f)
+        assert f.request.query["mitmproxy"] == "rocks"
+
+    def test_modify_response_body(self):
+        with tutils.raises(ScriptError):
+            tscript("modify_response_body.py")
+
+        m, sc = tscript("modify_response_body.py", "mitmproxy rocks")
+        f = tutils.tflow(resp=netutils.tresp(content=b"I <3 mitmproxy"))
+        self.invoke(m, "response", f)
+        assert f.response.content == b"I <3 rocks"
+
+    def test_redirect_requests(self):
+        m, sc = tscript("redirect_requests.py")
+        f = tutils.tflow(req=netutils.treq(host="example.org"))
+        self.invoke(m, "request", f)
+        assert f.request.host == "mitmproxy.org"
+
+    def test_har_extractor(self):
+        if sys.version_info >= (3, 0):
+            with tutils.raises("does not work on Python 3"):
+                tscript("har_extractor.py")
+            return
+
+        with tutils.raises(ScriptError):
+            tscript("har_extractor.py")
+
+        with tutils.tmpdir() as tdir:
+            times = dict(
+                timestamp_start=746203272,
+                timestamp_end=746203272,
+            )
+
+            path = os.path.join(tdir, "file")
+            m, sc = tscript("har_extractor.py", six.moves.shlex_quote(path))
+            f = tutils.tflow(
+                req=netutils.treq(**times),
+                resp=netutils.tresp(**times)
+            )
+            self.invoke(m, "response", f)
+            m.addons.remove(sc)
+
+            with open(path, "rb") as f:
+                test_data = json.load(f)
+            assert len(test_data["log"]["pages"]) == 1
diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py
index 1a07f74d..c58a9703 100644
--- a/test/mitmproxy/test_flow.py
+++ b/test/mitmproxy/test_flow.py
@@ -5,7 +5,7 @@ import netlib.utils
 from netlib.http import Headers
 from mitmproxy import filt, controller, flow
 from mitmproxy.contrib import tnetstring
-from mitmproxy.exceptions import FlowReadException, ScriptException
+from mitmproxy.exceptions import FlowReadException
 from mitmproxy.models import Error
 from mitmproxy.models import Flow
 from mitmproxy.models import HTTPFlow
@@ -674,21 +674,6 @@ class TestSerialize:
 
 class TestFlowMaster:
 
-    def test_load_script(self):
-        s = flow.State()
-        fm = flow.FlowMaster(None, None, s)
-
-        fm.load_script(tutils.test_data.path("data/scripts/a.py"))
-        fm.load_script(tutils.test_data.path("data/scripts/a.py"))
-        fm.unload_scripts()
-        with tutils.raises(ScriptException):
-            fm.load_script("nonexistent")
-        try:
-            fm.load_script(tutils.test_data.path("data/scripts/starterr.py"))
-        except ScriptException as e:
-            assert "ValueError" in str(e)
-        assert len(fm.scripts) == 0
-
     def test_getset_ignore(self):
         p = mock.Mock()
         p.config.check_ignore = HostMatcher()
@@ -708,51 +693,7 @@ class TestFlowMaster:
         assert "intercepting" in fm.replay_request(f)
 
         f.live = True
-        assert "live" in fm.replay_request(f, run_scripthooks=True)
-
-    def test_script_reqerr(self):
-        s = flow.State()
-        fm = flow.FlowMaster(None, None, s)
-        fm.load_script(tutils.test_data.path("data/scripts/reqerr.py"))
-        f = tutils.tflow()
-        fm.clientconnect(f.client_conn)
-        assert fm.request(f)
-
-    def test_script(self):
-        s = flow.State()
-        fm = flow.FlowMaster(None, None, s)
-        fm.load_script(tutils.test_data.path("data/scripts/all.py"))
-        f = tutils.tflow(resp=True)
-
-        f.client_conn.acked = False
-        fm.clientconnect(f.client_conn)
-        assert fm.scripts[0].ns["log"][-1] == "clientconnect"
-        f.server_conn.acked = False
-        fm.serverconnect(f.server_conn)
-        assert fm.scripts[0].ns["log"][-1] == "serverconnect"
-        f.reply.acked = False
-        fm.request(f)
-        assert fm.scripts[0].ns["log"][-1] == "request"
-        f.reply.acked = False
-        fm.response(f)
-        assert fm.scripts[0].ns["log"][-1] == "response"
-        # load second script
-        fm.load_script(tutils.test_data.path("data/scripts/all.py"))
-        assert len(fm.scripts) == 2
-        f.server_conn.reply.acked = False
-        fm.clientdisconnect(f.server_conn)
-        assert fm.scripts[0].ns["log"][-1] == "clientdisconnect"
-        assert fm.scripts[1].ns["log"][-1] == "clientdisconnect"
-
-        # unload first script
-        fm.unload_scripts()
-        assert len(fm.scripts) == 0
-        fm.load_script(tutils.test_data.path("data/scripts/all.py"))
-
-        f.error = tutils.terr()
-        f.reply.acked = False
-        fm.error(f)
-        assert fm.scripts[0].ns["log"][-1] == "error"
+        assert "live" in fm.replay_request(f)
 
     def test_duplicate_flow(self):
         s = flow.State()
@@ -789,7 +730,6 @@ class TestFlowMaster:
         f.error.reply = controller.DummyReply()
         fm.error(f)
 
-        fm.load_script(tutils.test_data.path("data/scripts/a.py"))
         fm.shutdown()
 
     def test_client_playback(self):
diff --git a/test/mitmproxy/test_script.py b/test/mitmproxy/test_script.py
deleted file mode 100644
index 1e8220f1..00000000
--- a/test/mitmproxy/test_script.py
+++ /dev/null
@@ -1,13 +0,0 @@
-from mitmproxy import flow
-from . import tutils
-
-
-def test_duplicate_flow():
-    s = flow.State()
-    fm = flow.FlowMaster(None, None, s)
-    fm.load_script(tutils.test_data.path("data/scripts/duplicate_flow.py"))
-    f = tutils.tflow()
-    fm.request(f)
-    assert fm.state.flow_count() == 2
-    assert not fm.state.view[0].request.is_replay
-    assert fm.state.view[1].request.is_replay
diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py
index 9dd8b79c..a5196dae 100644
--- a/test/mitmproxy/test_server.py
+++ b/test/mitmproxy/test_server.py
@@ -13,6 +13,7 @@ from netlib.http import authentication, http1
 from netlib.tutils import raises
 from pathod import pathoc, pathod
 
+from mitmproxy.builtins import script
 from mitmproxy import controller
 from mitmproxy.proxy.config import HostMatcher
 from mitmproxy.models import Error, HTTPResponse, HTTPFlow
@@ -287,10 +288,13 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin, AppMixin):
         self.master.set_stream_large_bodies(None)
 
     def test_stream_modify(self):
-        self.master.load_script(tutils.test_data.path("data/scripts/stream_modify.py"))
+        s = script.Script(
+            tutils.test_data.path("data/addonscripts/stream_modify.py")
+        )
+        self.master.addons.add(s)
         d = self.pathod('200:b"foo"')
         assert d.content == b"bar"
-        self.master.unload_scripts()
+        self.master.addons.remove(s)
 
 
 class TestHTTPAuth(tservers.HTTPProxyTest):
@@ -512,15 +516,15 @@ class TestTransparent(tservers.TransparentProxyTest, CommonMixin, TcpMixin):
     ssl = False
 
     def test_tcp_stream_modify(self):
-        self.master.load_script(tutils.test_data.path("data/scripts/tcp_stream_modify.py"))
-
+        s = script.Script(
+            tutils.test_data.path("data/addonscripts/tcp_stream_modify.py")
+        )
+        self.master.addons.add(s)
         self._tcpproxy_on()
         d = self.pathod('200:b"foo"')
         self._tcpproxy_off()
-
         assert d.content == b"bar"
-
-        self.master.unload_scripts()
+        self.master.addons.remove(s)
 
 
 class TestTransparentSSL(tservers.TransparentProxyTest, CommonMixin, TcpMixin):