Merge pull request #2484 from mhils/options-into-own-addon

Enable tool-specific options

16 files changed, 64 insertions, 85 deletions

diff --git a/test/filename_matching.py b/test/filename_matching.py
index 51cedf03..e74848d4 100644
--- a/test/filename_matching.py
+++ b/test/filename_matching.py
@@ -22,7 +22,7 @@ def check_src_files_have_test():
 def check_test_files_have_src():
     unknown_test_files = []
 
-    excluded = ['test/mitmproxy/data/', 'test/mitmproxy/net/data/', '/tservers.py']
+    excluded = ['test/mitmproxy/data/', 'test/mitmproxy/net/data/', '/tservers.py', '/conftest.py']
     test_files = glob.glob('test/mitmproxy/**/*.py', recursive=True) + glob.glob('test/pathod/**/*.py', recursive=True)
     test_files = [f for f in test_files if os.path.basename(f) != '__init__.py']
     test_files = [f for f in test_files if not any(os.path.normpath(p) in f for p in excluded)]
diff --git a/test/mitmproxy/addons/test_termstatus.py b/test/mitmproxy/addons/test_termstatus.py
index 2debaff5..5f960a1c 100644
--- a/test/mitmproxy/addons/test_termstatus.py
+++ b/test/mitmproxy/addons/test_termstatus.py
@@ -1,3 +1,4 @@
+from mitmproxy import proxy
 from mitmproxy.addons import termstatus
 from mitmproxy.test import taddons
 
@@ -5,6 +6,7 @@ from mitmproxy.test import taddons
 def test_configure():
     ts = termstatus.TermStatus()
     with taddons.context() as ctx:
+        ctx.master.server = proxy.DummyServer()
         ctx.configure(ts, server=False)
         ts.running()
         assert not ctx.master.logs
diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py
index 583e6e27..5e6fa701 100644
--- a/test/mitmproxy/proxy/protocol/test_http2.py
+++ b/test/mitmproxy/proxy/protocol/test_http2.py
@@ -8,7 +8,6 @@ import pytest
 import h2
 
 from mitmproxy import options
-from mitmproxy.proxy.config import ProxyConfig
 
 import mitmproxy.net
 from ...net import tservers as net_tservers
@@ -89,10 +88,8 @@ class _Http2TestBase:
 
     @classmethod
     def setup_class(cls):
-        opts = cls.get_options()
-        cls.config = ProxyConfig(opts)
-
-        tmaster = tservers.TestMaster(opts, cls.config)
+        cls.options = cls.get_options()
+        tmaster = tservers.TestMaster(cls.options)
         cls.proxy = tservers.ProxyThread(tmaster)
         cls.proxy.start()
 
@@ -319,7 +316,7 @@ class TestRequestWithPriority(_Http2Test):
         (False, (None, None, None), (None, None, None)),
     ])
     def test_request_with_priority(self, http2_priority_enabled, priority, expected_priority):
-        self.config.options.http2_priority = http2_priority_enabled
+        self.options.http2_priority = http2_priority_enabled
 
         h2_conn = self.setup_connection()
 
@@ -397,7 +394,7 @@ class TestPriority(_Http2Test):
         (False, (True, 42424242, 42), []),
     ])
     def test_priority(self, prioritize_before, http2_priority_enabled, priority, expected_priority):
-        self.config.options.http2_priority = http2_priority_enabled
+        self.options.http2_priority = http2_priority_enabled
         self.__class__.priority_data = []
 
         h2_conn = self.setup_connection()
@@ -508,8 +505,10 @@ class TestBodySizeLimit(_Http2Test):
         return True
 
     def test_body_size_limit(self):
-        self.config.options.body_size_limit = "20"
-        self.config.options._processed["body_size_limit"] = 20
+        self.options.body_size_limit = "20"
+
+        # FIXME: This should not be required?
+        self.options._processed["body_size_limit"] = 20
 
         h2_conn = self.setup_connection()
 
diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py
index 58857f92..460d85f8 100644
--- a/test/mitmproxy/proxy/protocol/test_websocket.py
+++ b/test/mitmproxy/proxy/protocol/test_websocket.py
@@ -7,7 +7,6 @@ from mitmproxy import options
 from mitmproxy import exceptions
 from mitmproxy.http import HTTPFlow
 from mitmproxy.websocket import WebSocketFlow
-from mitmproxy.proxy.config import ProxyConfig
 
 from mitmproxy.net import tcp
 from mitmproxy.net import http
@@ -49,10 +48,8 @@ class _WebSocketTestBase:
 
     @classmethod
     def setup_class(cls):
-        opts = cls.get_options()
-        cls.config = ProxyConfig(opts)
-
-        tmaster = tservers.TestMaster(opts, cls.config)
+        cls.options = cls.get_options()
+        tmaster = tservers.TestMaster(cls.options)
         cls.proxy = tservers.ProxyThread(tmaster)
         cls.proxy.start()
 
diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py
index 4cae756a..562f822c 100644
--- a/test/mitmproxy/proxy/test_server.py
+++ b/test/mitmproxy/proxy/test_server.py
@@ -117,13 +117,12 @@ class TcpMixin:
 
     def _ignore_on(self):
         assert not hasattr(self, "_ignore_backup")
-        self._ignore_backup = self.config.check_ignore
-        self.config.check_ignore = HostMatcher(
-            [".+:%s" % self.server.port] + self.config.check_ignore.patterns)
+        self._ignore_backup = self.options.ignore_hosts
+        self.options.ignore_hosts = [".+:%s" % self.server.port] + self.options.ignore_hosts
 
     def _ignore_off(self):
         assert hasattr(self, "_ignore_backup")
-        self.config.check_ignore = self._ignore_backup
+        self.options.ignore_hosts = self._ignore_backup
         del self._ignore_backup
 
     def test_ignore(self):
@@ -163,13 +162,12 @@ class TcpMixin:
 
     def _tcpproxy_on(self):
         assert not hasattr(self, "_tcpproxy_backup")
-        self._tcpproxy_backup = self.config.check_tcp
-        self.config.check_tcp = HostMatcher(
-            [".+:%s" % self.server.port] + self.config.check_tcp.patterns)
+        self._tcpproxy_backup = self.options.tcp_hosts
+        self.options.tcp_hosts = [".+:%s" % self.server.port] + self.options.tcp_hosts
 
     def _tcpproxy_off(self):
         assert hasattr(self, "_tcpproxy_backup")
-        self.config.check_tcp = self._tcpproxy_backup
+        self.options.tcp_hosts = self._tcpproxy_backup
         del self._tcpproxy_backup
 
     def test_tcp(self):
@@ -194,7 +192,8 @@ class TcpMixin:
             i2_cert = certs.SSLCert(i2.sslinfo.certchain[0])
             n_cert = certs.SSLCert(n.sslinfo.certchain[0])
 
-            assert i_cert == i2_cert == n_cert
+            assert i_cert == i2_cert
+            assert i_cert != n_cert
 
         # Make sure that TCP messages are in the event log.
         # Re-enable and fix this when we start keeping TCPFlows in the state.
@@ -353,22 +352,22 @@ class TestHTTPS(tservers.HTTPProxyTest, CommonMixin, TcpMixin):
 
     def test_clientcert_file(self):
         try:
-            self.config.client_certs = os.path.join(
+            self.options.client_certs = os.path.join(
                 tutils.test_data.path("mitmproxy/data/clientcert"), "client.pem")
             f = self.pathod("304")
             assert f.status_code == 304
             assert self.server.last_log()["request"]["clientcert"]["keyinfo"]
         finally:
-            self.config.client_certs = None
+            self.options.client_certs = None
 
     def test_clientcert_dir(self):
         try:
-            self.config.client_certs = tutils.test_data.path("mitmproxy/data/clientcert")
+            self.options.client_certs = tutils.test_data.path("mitmproxy/data/clientcert")
             f = self.pathod("304")
             assert f.status_code == 304
             assert self.server.last_log()["request"]["clientcert"]["keyinfo"]
         finally:
-            self.config.client_certs = None
+            self.options.client_certs = None
 
     def test_error_post_connect(self):
         p = self.pathoc()
@@ -412,7 +411,7 @@ class TestHTTPSUpstreamServerVerificationWTrustedCert(tservers.HTTPProxyTest):
             return p.request("get:/p/242")
 
     def test_verification_w_cadir(self):
-        self.config.options.update(
+        self.options.update(
             ssl_insecure=False,
             ssl_verify_upstream_trusted_cadir=tutils.test_data.path(
                 "mitmproxy/data/servercert/"
@@ -422,7 +421,7 @@ class TestHTTPSUpstreamServerVerificationWTrustedCert(tservers.HTTPProxyTest):
         assert self._request().status_code == 242
 
     def test_verification_w_pemfile(self):
-        self.config.options.update(
+        self.options.update(
             ssl_insecure=False,
             ssl_verify_upstream_trusted_cadir=None,
             ssl_verify_upstream_trusted_ca=tutils.test_data.path(
@@ -458,7 +457,7 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest):
         return opts
 
     def test_no_verification_w_bad_cert(self):
-        self.config.options.ssl_insecure = True
+        self.options.ssl_insecure = True
         r = self._request()
         assert r.status_code == 242
 
@@ -466,7 +465,7 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest):
         # We only test for a single invalid cert here.
         # Actual testing of different root-causes (invalid hostname, expired, ...)
         # is done in mitmproxy.net.
-        self.config.options.ssl_insecure = False
+        self.options.ssl_insecure = False
         r = self._request()
         assert r.status_code == 502
         assert b"Certificate Verification Error" in r.raw_content
@@ -493,7 +492,7 @@ class TestReverse(tservers.ReverseProxyTest, CommonMixin, TcpMixin):
     reverse = True
 
     def test_host_header(self):
-        self.config.options.keep_host_header = True
+        self.options.keep_host_header = True
         p = self.pathoc()
         with p.connect():
             resp = p.request("get:/p/200:h'Host'='example.com'")
@@ -503,7 +502,7 @@ class TestReverse(tservers.ReverseProxyTest, CommonMixin, TcpMixin):
         assert req.host_header == "example.com"
 
     def test_overridden_host_header(self):
-        self.config.options.keep_host_header = False  # default value
+        self.options.keep_host_header = False  # default value
         p = self.pathoc()
         with p.connect():
             resp = p.request("get:/p/200:h'Host'='example.com'")
diff --git a/test/mitmproxy/test_addonmanager.py b/test/mitmproxy/test_addonmanager.py
index fc048571..accf48e0 100644
--- a/test/mitmproxy/test_addonmanager.py
+++ b/test/mitmproxy/test_addonmanager.py
@@ -6,7 +6,6 @@ from mitmproxy import exceptions
 from mitmproxy import options
 from mitmproxy import command
 from mitmproxy import master
-from mitmproxy import proxy
 from mitmproxy.test import taddons
 from mitmproxy.test import tflow
 
@@ -51,7 +50,7 @@ def test_command():
 
 def test_halt():
     o = options.Options()
-    m = master.Master(o, proxy.DummyServer(o))
+    m = master.Master(o)
     a = addonmanager.AddonManager(m)
     halt = THalt()
     end = TAddon("end")
@@ -68,7 +67,7 @@ def test_halt():
 
 def test_lifecycle():
     o = options.Options()
-    m = master.Master(o, proxy.DummyServer(o))
+    m = master.Master(o)
     a = addonmanager.AddonManager(m)
     a.add(TAddon("one"))
 
@@ -141,7 +140,7 @@ def test_simple():
 
 def test_load_option():
     o = options.Options()
-    m = master.Master(o, proxy.DummyServer(o))
+    m = master.Master(o)
     a = addonmanager.AddonManager(m)
     a.add(AOption())
     assert "custom_option" in m.options._options
@@ -149,7 +148,7 @@ def test_load_option():
 
 def test_nesting():
     o = options.Options()
-    m = master.Master(o, proxy.DummyServer(o))
+    m = master.Master(o)
     a = addonmanager.AddonManager(m)
 
     a.add(
diff --git a/test/mitmproxy/test_controller.py b/test/mitmproxy/test_controller.py
index 2e13d298..e840380a 100644
--- a/test/mitmproxy/test_controller.py
+++ b/test/mitmproxy/test_controller.py
@@ -30,7 +30,8 @@ class TestMaster:
             assert ctx.master.should_exit.is_set()
 
     def test_server_simple(self):
-        m = master.Master(None, proxy.DummyServer(None))
+        m = master.Master(None)
+        m.server = proxy.DummyServer()
         m.start()
         m.shutdown()
         m.start()
diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py
index 0b04c4c1..7f9d577b 100644
--- a/test/mitmproxy/test_flow.py
+++ b/test/mitmproxy/test_flow.py
@@ -6,13 +6,11 @@ from mitmproxy.test import tflow, tutils
 import mitmproxy.io
 from mitmproxy import flowfilter
 from mitmproxy import options
-from mitmproxy.proxy import config
 from mitmproxy.io import tnetstring
 from mitmproxy.exceptions import FlowReadException, ReplayException, ControlException
 from mitmproxy import flow
 from mitmproxy import http
 from mitmproxy.net import http as net_http
-from mitmproxy.proxy.server import DummyServer
 from mitmproxy import master
 from . import tservers
 
@@ -93,8 +91,7 @@ class TestFlowMaster:
         opts = options.Options(
             mode="reverse:https://use-this-domain"
         )
-        conf = config.ProxyConfig(opts)
-        fm = master.Master(opts, DummyServer(conf))
+        fm = master.Master(opts)
         fm.addons.add(s)
         f = tflow.tflow(resp=True)
         fm.load_flow(f)
@@ -102,8 +99,7 @@ class TestFlowMaster:
 
     def test_replay(self):
         opts = options.Options()
-        conf = config.ProxyConfig(opts)
-        fm = master.Master(opts, DummyServer(conf))
+        fm = master.Master(opts)
         f = tflow.tflow(resp=True)
         f.request.content = None
         with pytest.raises(ReplayException, match="missing"):
@@ -131,7 +127,7 @@ class TestFlowMaster:
 
     def test_all(self):
         s = tservers.TestState()
-        fm = master.Master(None, DummyServer())
+        fm = master.Master(None)
         fm.addons.add(s)
         f = tflow.tflow(req=None)
         fm.addons.handle_lifecycle("clientconnect", f.client_conn)
diff --git a/test/mitmproxy/tools/console/conftest.py b/test/mitmproxy/tools/console/conftest.py
deleted file mode 100644
index afd94c6a..00000000
--- a/test/mitmproxy/tools/console/conftest.py
+++ /dev/null
@@ -1,9 +0,0 @@
-from unittest import mock
-
-import pytest
-
-
-@pytest.fixture(scope="module", autouse=True)
-def definitely_atty():
-    with mock.patch("sys.stdout.isatty", lambda: True):
-        yield
diff --git a/test/mitmproxy/tools/console/test_master.py b/test/mitmproxy/tools/console/test_master.py
index ef357c76..3aa0dc54 100644
--- a/test/mitmproxy/tools/console/test_master.py
+++ b/test/mitmproxy/tools/console/test_master.py
@@ -1,7 +1,6 @@
 import urwid
 
 from mitmproxy import options
-from mitmproxy import proxy
 from mitmproxy.test import tflow
 from mitmproxy.test import tutils
 from mitmproxy.tools import console
@@ -30,7 +29,7 @@ class TestMaster(tservers.MasterTest):
         if "verbosity" not in opts:
             opts["verbosity"] = 'warn'
         o = options.Options(**opts)
-        m = console.master.ConsoleMaster(o, proxy.DummyServer())
+        m = console.master.ConsoleMaster(o)
         m.addons.trigger("configure", o.keys())
         return m
 
diff --git a/test/mitmproxy/tools/console/test_statusbar.py b/test/mitmproxy/tools/console/test_statusbar.py
index 55a3c4a0..2f7825c9 100644
--- a/test/mitmproxy/tools/console/test_statusbar.py
+++ b/test/mitmproxy/tools/console/test_statusbar.py
@@ -1,4 +1,4 @@
-from mitmproxy import options, proxy
+from mitmproxy import options
 from mitmproxy.tools.console import statusbar, master
 
 
@@ -26,7 +26,7 @@ def test_statusbar(monkeypatch):
         scripts=["nonexistent"],
         save_stream_file="foo",
     )
-    m = master.ConsoleMaster(o, proxy.DummyServer())
+    m = master.ConsoleMaster(o)
     monkeypatch.setattr(m.addons.get("clientplayback"), "count", lambda: 42)
     monkeypatch.setattr(m.addons.get("serverplayback"), "count", lambda: 42)
 
diff --git a/test/mitmproxy/tools/test_cmdline.py b/test/mitmproxy/tools/test_cmdline.py
index 65cfeb07..e247dc1d 100644
--- a/test/mitmproxy/tools/test_cmdline.py
+++ b/test/mitmproxy/tools/test_cmdline.py
@@ -1,7 +1,8 @@
 import argparse
-from mitmproxy.tools import cmdline
-from mitmproxy.tools import main
+
 from mitmproxy import options
+from mitmproxy.tools import cmdline, web, dump, console
+from mitmproxy.tools import main
 
 
 def test_common():
@@ -14,17 +15,20 @@ def test_common():
 
 def test_mitmproxy():
     opts = options.Options()
+    console.master.ConsoleMaster(opts)
     ap = cmdline.mitmproxy(opts)
     assert ap
 
 
 def test_mitmdump():
     opts = options.Options()
+    dump.DumpMaster(opts)
     ap = cmdline.mitmdump(opts)
     assert ap
 
 
 def test_mitmweb():
     opts = options.Options()
+    web.master.WebMaster(opts)
     ap = cmdline.mitmweb(opts)
     assert ap
diff --git a/test/mitmproxy/tools/test_dump.py b/test/mitmproxy/tools/test_dump.py
index 597333af..952c3f4f 100644
--- a/test/mitmproxy/tools/test_dump.py
+++ b/test/mitmproxy/tools/test_dump.py
@@ -1,7 +1,6 @@
 import pytest
 from unittest import mock
 
-from mitmproxy import proxy
 from mitmproxy import log
 from mitmproxy import controller
 from mitmproxy import options
@@ -13,7 +12,7 @@ from .. import tservers
 class TestDumpMaster(tservers.MasterTest):
     def mkmaster(self, flt, **opts):
         o = options.Options(view_filter=flt, verbosity='error', flow_detail=0, **opts)
-        m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=False, with_dumper=False)
+        m = dump.DumpMaster(o, with_termlog=False, with_dumper=False)
         return m
 
     def test_has_error(self):
@@ -27,12 +26,12 @@ class TestDumpMaster(tservers.MasterTest):
     def test_addons_termlog(self, termlog):
         with mock.patch('sys.stdout'):
             o = options.Options()
-            m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=termlog)
+            m = dump.DumpMaster(o, with_termlog=termlog)
             assert (m.addons.get('termlog') is not None) == termlog
 
     @pytest.mark.parametrize("dumper", [False, True])
     def test_addons_dumper(self, dumper):
         with mock.patch('sys.stdout'):
             o = options.Options()
-            m = dump.DumpMaster(o, proxy.DummyServer(), with_dumper=dumper)
+            m = dump.DumpMaster(o, with_dumper=dumper)
             assert (m.addons.get('dumper') is not None) == dumper
diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py
index 4d290284..091ef5e8 100644
--- a/test/mitmproxy/tools/web/test_app.py
+++ b/test/mitmproxy/tools/web/test_app.py
@@ -7,7 +7,6 @@ from tornado import httpclient
 from tornado import websocket
 
 from mitmproxy import exceptions
-from mitmproxy import proxy
 from mitmproxy import options
 from mitmproxy.test import tflow
 from mitmproxy.tools.web import app
@@ -21,7 +20,7 @@ def json(resp: httpclient.HTTPResponse):
 class TestApp(tornado.testing.AsyncHTTPTestCase):
     def get_app(self):
         o = options.Options(http2=False)
-        m = webmaster.WebMaster(o, proxy.DummyServer(), with_termlog=False)
+        m = webmaster.WebMaster(o, with_termlog=False)
         f = tflow.tflow(resp=True)
         f.id = "42"
         m.view.add([f])
@@ -323,5 +322,5 @@ class TestApp(tornado.testing.AsyncHTTPTestCase):
         web_root = os.path.join(here, os.pardir, os.pardir, os.pardir, os.pardir, 'web')
         tflow_path = os.path.join(web_root, 'src/js/__tests__/ducks/_tflow.js')
         content = """export default function(){{\n    return {tflow_json}\n}}""".format(tflow_json=tflow_json)
-        with open(tflow_path, 'w') as f:
+        with open(tflow_path, 'w', newline="\n") as f:
             f.write(content)
diff --git a/test/mitmproxy/tools/web/test_master.py b/test/mitmproxy/tools/web/test_master.py
index 27f99a18..2bceb5ca 100644
--- a/test/mitmproxy/tools/web/test_master.py
+++ b/test/mitmproxy/tools/web/test_master.py
@@ -1,7 +1,5 @@
 from mitmproxy.tools.web import master
-from mitmproxy import proxy
 from mitmproxy import options
-from mitmproxy.proxy.config import ProxyConfig
 
 from ... import tservers
 
@@ -9,8 +7,7 @@ from ... import tservers
 class TestWebMaster(tservers.MasterTest):
     def mkmaster(self, **opts):
         o = options.Options(**opts)
-        c = ProxyConfig(o)
-        return master.WebMaster(o, proxy.DummyServer(c))
+        return master.WebMaster(o)
 
     def test_basic(self):
         m = self.mkmaster()
diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py
index 9faaf20e..dd5bb327 100644
--- a/test/mitmproxy/tservers.py
+++ b/test/mitmproxy/tservers.py
@@ -69,9 +69,10 @@ class TestState:
 
 class TestMaster(taddons.RecordingMaster):
 
-    def __init__(self, opts, config):
-        s = ProxyServer(config)
-        super().__init__(opts, s)
+    def __init__(self, opts):
+        super().__init__(opts)
+        config = ProxyConfig(opts)
+        self.server = ProxyServer(config)
 
     def clear_addons(self, addons):
         self.addons.clear()
@@ -129,9 +130,8 @@ class ProxyTestBase:
             ssl=cls.ssl,
             ssloptions=cls.ssloptions)
 
-        opts = cls.get_options()
-        cls.config = ProxyConfig(opts)
-        tmaster = cls.masterclass(opts, cls.config)
+        cls.options = cls.get_options()
+        tmaster = cls.masterclass(cls.options)
         cls.proxy = ProxyThread(tmaster)
         cls.proxy.start()
 
@@ -338,19 +338,16 @@ class ChainProxyTest(ProxyTestBase):
 
     @classmethod
     def setup_class(cls):
+        # We need to initialize the chain first so that the normal server gets a correct config.
         cls.chain = []
-        super().setup_class()
         for _ in range(cls.n):
             opts = cls.get_options()
-            config = ProxyConfig(opts)
-            tmaster = cls.masterclass(opts, config)
+            tmaster = cls.masterclass(opts)
             proxy = ProxyThread(tmaster)
             proxy.start()
             cls.chain.insert(0, proxy)
 
-        # Patch the orginal proxy to upstream mode
-        opts = cls.get_options()
-        cls.config = cls.proxy.tmaster.config = cls.proxy.tmaster.server.config = ProxyConfig(opts)
+        super().setup_class()
 
     @classmethod
     def teardown_class(cls):