Revamp replacement hooks

- Replacement specifiers can be either strings or tuples. This lets us cope
gracefully with command-line parsing (and posible quick interactive
specification) without having to special-case replacement hooks, or have
knowledge of hook specification leak outside the addon. We can also now use the
same command-line spec format in config files.
- Split replacement and replacement from file into separate addons and options.
Files are now read on each replacement, so you can edit replacement files in
place without restart.
- Modernise the test suite to use addon test helpers.

TODO: editing and displaying replace-from-file in console app

2 files changed, 74 insertions, 103 deletions

diff --git a/test/mitmproxy/addons/test_replace.py b/test/mitmproxy/addons/test_replace.py
index 34fa5017..ec38b77d 100644
--- a/test/mitmproxy/addons/test_replace.py
+++ b/test/mitmproxy/addons/test_replace.py
@@ -1,57 +1,59 @@
+import os.path
 from mitmproxy.test import tflow
 from mitmproxy.test import tutils
 
 from .. import tservers
 from mitmproxy.addons import replace
-from mitmproxy import master
-from mitmproxy import options
-from mitmproxy import proxy
+from mitmproxy.test import taddons
 
 
 class TestReplace:
+    def test_parse_hook(self):
+        x = replace.parse_hook("/foo/bar/voing")
+        assert x == ("foo", "bar", "voing")
+        x = replace.parse_hook("/foo/bar/vo/ing/")
+        assert x == ("foo", "bar", "vo/ing/")
+        x = replace.parse_hook("/bar/voing")
+        assert x == (".*", "bar", "voing")
+        tutils.raises("invalid replacement", replace.parse_hook, "/")
+
     def test_configure(self):
         r = replace.Replace()
-        updated = set(["replacements"])
-        r.configure(options.Options(
-            replacements=[("one", "two", "three")]
-        ), updated)
-        tutils.raises(
-            "invalid filter pattern",
-            r.configure,
-            options.Options(
+        with taddons.context() as tctx:
+            tctx.configure(r, replacements=[("one", "two", "three")])
+            tutils.raises(
+                "invalid filter pattern",
+                tctx.configure,
+                r,
                 replacements=[("~b", "two", "three")]
-            ),
-            updated
-        )
-        tutils.raises(
-            "invalid regular expression",
-            r.configure,
-            options.Options(
+            )
+            tutils.raises(
+                "invalid regular expression",
+                tctx.configure,
+                r,
                 replacements=[("foo", "+", "three")]
-            ),
-            updated
-        )
+            )
+            tctx.configure(r, replacements=["/a/b/c/"])
 
     def test_simple(self):
-        o = options.Options(
-            replacements = [
-                ("~q", "foo", "bar"),
-                ("~s", "foo", "bar"),
-            ]
-        )
-        m = master.Master(o, proxy.DummyServer())
-        sa = replace.Replace()
-        m.addons.add(sa)
-
-        f = tflow.tflow()
-        f.request.content = b"foo"
-        m.request(f)
-        assert f.request.content == b"bar"
+        r = replace.Replace()
+        with taddons.context() as tctx:
+            tctx.configure(
+                r,
+                replacements = [
+                    ("~q", "foo", "bar"),
+                    ("~s", "foo", "bar"),
+                ]
+            )
+            f = tflow.tflow()
+            f.request.content = b"foo"
+            r.request(f)
+            assert f.request.content == b"bar"
 
-        f = tflow.tflow(resp=True)
-        f.response.content = b"foo"
-        m.response(f)
-        assert f.response.content == b"bar"
+            f = tflow.tflow(resp=True)
+            f.response.content = b"foo"
+            r.response(f)
+            assert f.response.content == b"bar"
 
 
 class TestUpstreamProxy(tservers.HTTPUpstreamProxyTest):
@@ -72,3 +74,36 @@ class TestUpstreamProxy(tservers.HTTPUpstreamProxyTest):
             req = p.request("get:'%s/p/418:b\"foo\"'" % self.server.urlbase)
         assert req.content == b"ORLY"
         assert req.status_code == 418
+
+
+class TestReplaceFile:
+    def test_simple(self):
+        r = replace.ReplaceFile()
+        with tutils.tmpdir() as td:
+            rp = os.path.join(td, "replacement")
+            with open(rp, "w") as f:
+                f.write("bar")
+            with taddons.context() as tctx:
+                tctx.configure(
+                    r,
+                    replacement_files = [
+                        ("~q", "foo", rp),
+                        ("~s", "foo", rp),
+                        ("~b nonexistent", "nonexistent", "nonexistent"),
+                    ]
+                )
+                f = tflow.tflow()
+                f.request.content = b"foo"
+                r.request(f)
+                assert f.request.content == b"bar"
+
+                f = tflow.tflow(resp=True)
+                f.response.content = b"foo"
+                r.response(f)
+                assert f.response.content == b"bar"
+
+                f = tflow.tflow()
+                f.request.content = b"nonexistent"
+                assert not tctx.master.event_log
+                r.request(f)
+                assert tctx.master.event_log
diff --git a/test/mitmproxy/test_cmdline.py b/test/mitmproxy/test_cmdline.py
index d2e0c8a5..fe0373d1 100644
--- a/test/mitmproxy/test_cmdline.py
+++ b/test/mitmproxy/test_cmdline.py
@@ -3,38 +3,6 @@ from mitmproxy.tools import cmdline
 from mitmproxy.test import tutils
 
 
-def test_parse_replace_hook():
-    x = cmdline.parse_replace_hook("/foo/bar/voing")
-    assert x == ("foo", "bar", "voing")
-
-    x = cmdline.parse_replace_hook("/foo/bar/vo/ing/")
-    assert x == ("foo", "bar", "vo/ing/")
-
-    x = cmdline.parse_replace_hook("/bar/voing")
-    assert x == (".*", "bar", "voing")
-
-    tutils.raises(
-        cmdline.ParseException,
-        cmdline.parse_replace_hook,
-        "/foo"
-    )
-    tutils.raises(
-        "replacement regex",
-        cmdline.parse_replace_hook,
-        "patt/[/rep"
-    )
-    tutils.raises(
-        "filter pattern",
-        cmdline.parse_replace_hook,
-        "/~/foo/rep"
-    )
-    tutils.raises(
-        "empty clause",
-        cmdline.parse_replace_hook,
-        "//"
-    )
-
-
 def test_parse_setheaders():
     x = cmdline.parse_setheader("/foo/bar/voing")
     assert x == ("foo", "bar", "voing")
@@ -65,38 +33,6 @@ def test_common():
     )
     opts.setheader = []
 
-    opts.replace = ["/foo/bar/voing"]
-    v = cmdline.get_common_options(opts)
-    assert v["replacements"] == [("foo", "bar", "voing")]
-
-    opts.replace = ["//"]
-    tutils.raises(
-        "empty clause",
-        cmdline.get_common_options,
-        opts
-    )
-
-    opts.replace = []
-    opts.replace_file = [("/foo/bar/nonexistent")]
-    tutils.raises(
-        "could not read replace file",
-        cmdline.get_common_options,
-        opts
-    )
-
-    opts.replace_file = [("/~/bar/nonexistent")]
-    tutils.raises(
-        "filter pattern",
-        cmdline.get_common_options,
-        opts
-    )
-
-    p = tutils.test_data.path("mitmproxy/data/replace")
-    opts.replace_file = [("/foo/bar/%s" % p)]
-    v = cmdline.get_common_options(opts)["replacements"]
-    assert len(v) == 1
-    assert v[0][2].strip() == b"replacecontents"
-
 
 def test_mitmproxy():
     ap = cmdline.mitmproxy()