disentangle ProxyServer and Master classes.

The proxy server should ultimately be an addon itself and not be passed
to the Master constructor. This commit already removes the server in the majority
of instances, and also replaces a large number of ProxyConfig usages with the Options class..

14 files changed, 57 insertions, 73 deletions

diff --git a/test/filename_matching.py b/test/filename_matching.py
index 51cedf03..e74848d4 100644
--- a/test/filename_matching.py
+++ b/test/filename_matching.py
@@ -22,7 +22,7 @@ def check_src_files_have_test():
 def check_test_files_have_src():
     unknown_test_files = []
 
-    excluded = ['test/mitmproxy/data/', 'test/mitmproxy/net/data/', '/tservers.py']
+    excluded = ['test/mitmproxy/data/', 'test/mitmproxy/net/data/', '/tservers.py', '/conftest.py']
     test_files = glob.glob('test/mitmproxy/**/*.py', recursive=True) + glob.glob('test/pathod/**/*.py', recursive=True)
     test_files = [f for f in test_files if os.path.basename(f) != '__init__.py']
     test_files = [f for f in test_files if not any(os.path.normpath(p) in f for p in excluded)]
diff --git a/test/mitmproxy/addons/test_termstatus.py b/test/mitmproxy/addons/test_termstatus.py
index 2debaff5..5f960a1c 100644
--- a/test/mitmproxy/addons/test_termstatus.py
+++ b/test/mitmproxy/addons/test_termstatus.py
@@ -1,3 +1,4 @@
+from mitmproxy import proxy
 from mitmproxy.addons import termstatus
 from mitmproxy.test import taddons
 
@@ -5,6 +6,7 @@ from mitmproxy.test import taddons
 def test_configure():
     ts = termstatus.TermStatus()
     with taddons.context() as ctx:
+        ctx.master.server = proxy.DummyServer()
         ctx.configure(ts, server=False)
         ts.running()
         assert not ctx.master.logs
diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py
index 583e6e27..5e6fa701 100644
--- a/test/mitmproxy/proxy/protocol/test_http2.py
+++ b/test/mitmproxy/proxy/protocol/test_http2.py
@@ -8,7 +8,6 @@ import pytest
 import h2
 
 from mitmproxy import options
-from mitmproxy.proxy.config import ProxyConfig
 
 import mitmproxy.net
 from ...net import tservers as net_tservers
@@ -89,10 +88,8 @@ class _Http2TestBase:
 
     @classmethod
     def setup_class(cls):
-        opts = cls.get_options()
-        cls.config = ProxyConfig(opts)
-
-        tmaster = tservers.TestMaster(opts, cls.config)
+        cls.options = cls.get_options()
+        tmaster = tservers.TestMaster(cls.options)
         cls.proxy = tservers.ProxyThread(tmaster)
         cls.proxy.start()
 
@@ -319,7 +316,7 @@ class TestRequestWithPriority(_Http2Test):
         (False, (None, None, None), (None, None, None)),
     ])
     def test_request_with_priority(self, http2_priority_enabled, priority, expected_priority):
-        self.config.options.http2_priority = http2_priority_enabled
+        self.options.http2_priority = http2_priority_enabled
 
         h2_conn = self.setup_connection()
 
@@ -397,7 +394,7 @@ class TestPriority(_Http2Test):
         (False, (True, 42424242, 42), []),
     ])
     def test_priority(self, prioritize_before, http2_priority_enabled, priority, expected_priority):
-        self.config.options.http2_priority = http2_priority_enabled
+        self.options.http2_priority = http2_priority_enabled
         self.__class__.priority_data = []
 
         h2_conn = self.setup_connection()
@@ -508,8 +505,10 @@ class TestBodySizeLimit(_Http2Test):
         return True
 
     def test_body_size_limit(self):
-        self.config.options.body_size_limit = "20"
-        self.config.options._processed["body_size_limit"] = 20
+        self.options.body_size_limit = "20"
+
+        # FIXME: This should not be required?
+        self.options._processed["body_size_limit"] = 20
 
         h2_conn = self.setup_connection()
 
diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py
index 58857f92..460d85f8 100644
--- a/test/mitmproxy/proxy/protocol/test_websocket.py
+++ b/test/mitmproxy/proxy/protocol/test_websocket.py
@@ -7,7 +7,6 @@ from mitmproxy import options
 from mitmproxy import exceptions
 from mitmproxy.http import HTTPFlow
 from mitmproxy.websocket import WebSocketFlow
-from mitmproxy.proxy.config import ProxyConfig
 
 from mitmproxy.net import tcp
 from mitmproxy.net import http
@@ -49,10 +48,8 @@ class _WebSocketTestBase:
 
     @classmethod
     def setup_class(cls):
-        opts = cls.get_options()
-        cls.config = ProxyConfig(opts)
-
-        tmaster = tservers.TestMaster(opts, cls.config)
+        cls.options = cls.get_options()
+        tmaster = tservers.TestMaster(cls.options)
         cls.proxy = tservers.ProxyThread(tmaster)
         cls.proxy.start()
 
diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py
index 4cae756a..562f822c 100644
--- a/test/mitmproxy/proxy/test_server.py
+++ b/test/mitmproxy/proxy/test_server.py
@@ -117,13 +117,12 @@ class TcpMixin:
 
     def _ignore_on(self):
         assert not hasattr(self, "_ignore_backup")
-        self._ignore_backup = self.config.check_ignore
-        self.config.check_ignore = HostMatcher(
-            [".+:%s" % self.server.port] + self.config.check_ignore.patterns)
+        self._ignore_backup = self.options.ignore_hosts
+        self.options.ignore_hosts = [".+:%s" % self.server.port] + self.options.ignore_hosts
 
     def _ignore_off(self):
         assert hasattr(self, "_ignore_backup")
-        self.config.check_ignore = self._ignore_backup
+        self.options.ignore_hosts = self._ignore_backup
         del self._ignore_backup
 
     def test_ignore(self):
@@ -163,13 +162,12 @@ class TcpMixin:
 
     def _tcpproxy_on(self):
         assert not hasattr(self, "_tcpproxy_backup")
-        self._tcpproxy_backup = self.config.check_tcp
-        self.config.check_tcp = HostMatcher(
-            [".+:%s" % self.server.port] + self.config.check_tcp.patterns)
+        self._tcpproxy_backup = self.options.tcp_hosts
+        self.options.tcp_hosts = [".+:%s" % self.server.port] + self.options.tcp_hosts
 
     def _tcpproxy_off(self):
         assert hasattr(self, "_tcpproxy_backup")
-        self.config.check_tcp = self._tcpproxy_backup
+        self.options.tcp_hosts = self._tcpproxy_backup
         del self._tcpproxy_backup
 
     def test_tcp(self):
@@ -194,7 +192,8 @@ class TcpMixin:
             i2_cert = certs.SSLCert(i2.sslinfo.certchain[0])
             n_cert = certs.SSLCert(n.sslinfo.certchain[0])
 
-            assert i_cert == i2_cert == n_cert
+            assert i_cert == i2_cert
+            assert i_cert != n_cert
 
         # Make sure that TCP messages are in the event log.
         # Re-enable and fix this when we start keeping TCPFlows in the state.
@@ -353,22 +352,22 @@ class TestHTTPS(tservers.HTTPProxyTest, CommonMixin, TcpMixin):
 
     def test_clientcert_file(self):
         try:
-            self.config.client_certs = os.path.join(
+            self.options.client_certs = os.path.join(
                 tutils.test_data.path("mitmproxy/data/clientcert"), "client.pem")
             f = self.pathod("304")
             assert f.status_code == 304
             assert self.server.last_log()["request"]["clientcert"]["keyinfo"]
         finally:
-            self.config.client_certs = None
+            self.options.client_certs = None
 
     def test_clientcert_dir(self):
         try:
-            self.config.client_certs = tutils.test_data.path("mitmproxy/data/clientcert")
+            self.options.client_certs = tutils.test_data.path("mitmproxy/data/clientcert")
             f = self.pathod("304")
             assert f.status_code == 304
             assert self.server.last_log()["request"]["clientcert"]["keyinfo"]
         finally:
-            self.config.client_certs = None
+            self.options.client_certs = None
 
     def test_error_post_connect(self):
         p = self.pathoc()
@@ -412,7 +411,7 @@ class TestHTTPSUpstreamServerVerificationWTrustedCert(tservers.HTTPProxyTest):
             return p.request("get:/p/242")
 
     def test_verification_w_cadir(self):
-        self.config.options.update(
+        self.options.update(
             ssl_insecure=False,
             ssl_verify_upstream_trusted_cadir=tutils.test_data.path(
                 "mitmproxy/data/servercert/"
@@ -422,7 +421,7 @@ class TestHTTPSUpstreamServerVerificationWTrustedCert(tservers.HTTPProxyTest):
         assert self._request().status_code == 242
 
     def test_verification_w_pemfile(self):
-        self.config.options.update(
+        self.options.update(
             ssl_insecure=False,
             ssl_verify_upstream_trusted_cadir=None,
             ssl_verify_upstream_trusted_ca=tutils.test_data.path(
@@ -458,7 +457,7 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest):
         return opts
 
     def test_no_verification_w_bad_cert(self):
-        self.config.options.ssl_insecure = True
+        self.options.ssl_insecure = True
         r = self._request()
         assert r.status_code == 242
 
@@ -466,7 +465,7 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxyTest):
         # We only test for a single invalid cert here.
         # Actual testing of different root-causes (invalid hostname, expired, ...)
         # is done in mitmproxy.net.
-        self.config.options.ssl_insecure = False
+        self.options.ssl_insecure = False
         r = self._request()
         assert r.status_code == 502
         assert b"Certificate Verification Error" in r.raw_content
@@ -493,7 +492,7 @@ class TestReverse(tservers.ReverseProxyTest, CommonMixin, TcpMixin):
     reverse = True
 
     def test_host_header(self):
-        self.config.options.keep_host_header = True
+        self.options.keep_host_header = True
         p = self.pathoc()
         with p.connect():
             resp = p.request("get:/p/200:h'Host'='example.com'")
@@ -503,7 +502,7 @@ class TestReverse(tservers.ReverseProxyTest, CommonMixin, TcpMixin):
         assert req.host_header == "example.com"
 
     def test_overridden_host_header(self):
-        self.config.options.keep_host_header = False  # default value
+        self.options.keep_host_header = False  # default value
         p = self.pathoc()
         with p.connect():
             resp = p.request("get:/p/200:h'Host'='example.com'")
diff --git a/test/mitmproxy/test_addonmanager.py b/test/mitmproxy/test_addonmanager.py
index 678bc1b7..3ac74375 100644
--- a/test/mitmproxy/test_addonmanager.py
+++ b/test/mitmproxy/test_addonmanager.py
@@ -6,7 +6,6 @@ from mitmproxy import exceptions
 from mitmproxy import options
 from mitmproxy import command
 from mitmproxy import master
-from mitmproxy import proxy
 from mitmproxy.test import taddons
 from mitmproxy.test import tflow
 
@@ -51,7 +50,7 @@ def test_command():
 
 def test_halt():
     o = options.Options()
-    m = master.Master(o, proxy.DummyServer(o))
+    m = master.Master(o)
     a = addonmanager.AddonManager(m)
     halt = THalt()
     end = TAddon("end")
@@ -68,7 +67,7 @@ def test_halt():
 
 def test_lifecycle():
     o = options.Options()
-    m = master.Master(o, proxy.DummyServer(o))
+    m = master.Master(o)
     a = addonmanager.AddonManager(m)
     a.add(TAddon("one"))
 
@@ -128,7 +127,7 @@ def test_simple():
 
 def test_load_option():
     o = options.Options()
-    m = master.Master(o, proxy.DummyServer(o))
+    m = master.Master(o)
     a = addonmanager.AddonManager(m)
     a.add(AOption())
     assert "custom_option" in m.options._options
@@ -136,7 +135,7 @@ def test_load_option():
 
 def test_nesting():
     o = options.Options()
-    m = master.Master(o, proxy.DummyServer(o))
+    m = master.Master(o)
     a = addonmanager.AddonManager(m)
 
     a.add(
diff --git a/test/mitmproxy/test_controller.py b/test/mitmproxy/test_controller.py
index 2e13d298..e840380a 100644
--- a/test/mitmproxy/test_controller.py
+++ b/test/mitmproxy/test_controller.py
@@ -30,7 +30,8 @@ class TestMaster:
             assert ctx.master.should_exit.is_set()
 
     def test_server_simple(self):
-        m = master.Master(None, proxy.DummyServer(None))
+        m = master.Master(None)
+        m.server = proxy.DummyServer()
         m.start()
         m.shutdown()
         m.start()
diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py
index 0b04c4c1..7f9d577b 100644
--- a/test/mitmproxy/test_flow.py
+++ b/test/mitmproxy/test_flow.py
@@ -6,13 +6,11 @@ from mitmproxy.test import tflow, tutils
 import mitmproxy.io
 from mitmproxy import flowfilter
 from mitmproxy import options
-from mitmproxy.proxy import config
 from mitmproxy.io import tnetstring
 from mitmproxy.exceptions import FlowReadException, ReplayException, ControlException
 from mitmproxy import flow
 from mitmproxy import http
 from mitmproxy.net import http as net_http
-from mitmproxy.proxy.server import DummyServer
 from mitmproxy import master
 from . import tservers
 
@@ -93,8 +91,7 @@ class TestFlowMaster:
         opts = options.Options(
             mode="reverse:https://use-this-domain"
         )
-        conf = config.ProxyConfig(opts)
-        fm = master.Master(opts, DummyServer(conf))
+        fm = master.Master(opts)
         fm.addons.add(s)
         f = tflow.tflow(resp=True)
         fm.load_flow(f)
@@ -102,8 +99,7 @@ class TestFlowMaster:
 
     def test_replay(self):
         opts = options.Options()
-        conf = config.ProxyConfig(opts)
-        fm = master.Master(opts, DummyServer(conf))
+        fm = master.Master(opts)
         f = tflow.tflow(resp=True)
         f.request.content = None
         with pytest.raises(ReplayException, match="missing"):
@@ -131,7 +127,7 @@ class TestFlowMaster:
 
     def test_all(self):
         s = tservers.TestState()
-        fm = master.Master(None, DummyServer())
+        fm = master.Master(None)
         fm.addons.add(s)
         f = tflow.tflow(req=None)
         fm.addons.handle_lifecycle("clientconnect", f.client_conn)
diff --git a/test/mitmproxy/tools/console/test_master.py b/test/mitmproxy/tools/console/test_master.py
index ef357c76..3aa0dc54 100644
--- a/test/mitmproxy/tools/console/test_master.py
+++ b/test/mitmproxy/tools/console/test_master.py
@@ -1,7 +1,6 @@
 import urwid
 
 from mitmproxy import options
-from mitmproxy import proxy
 from mitmproxy.test import tflow
 from mitmproxy.test import tutils
 from mitmproxy.tools import console
@@ -30,7 +29,7 @@ class TestMaster(tservers.MasterTest):
         if "verbosity" not in opts:
             opts["verbosity"] = 'warn'
         o = options.Options(**opts)
-        m = console.master.ConsoleMaster(o, proxy.DummyServer())
+        m = console.master.ConsoleMaster(o)
         m.addons.trigger("configure", o.keys())
         return m
 
diff --git a/test/mitmproxy/tools/console/test_statusbar.py b/test/mitmproxy/tools/console/test_statusbar.py
index 55a3c4a0..2f7825c9 100644
--- a/test/mitmproxy/tools/console/test_statusbar.py
+++ b/test/mitmproxy/tools/console/test_statusbar.py
@@ -1,4 +1,4 @@
-from mitmproxy import options, proxy
+from mitmproxy import options
 from mitmproxy.tools.console import statusbar, master
 
 
@@ -26,7 +26,7 @@ def test_statusbar(monkeypatch):
         scripts=["nonexistent"],
         save_stream_file="foo",
     )
-    m = master.ConsoleMaster(o, proxy.DummyServer())
+    m = master.ConsoleMaster(o)
     monkeypatch.setattr(m.addons.get("clientplayback"), "count", lambda: 42)
     monkeypatch.setattr(m.addons.get("serverplayback"), "count", lambda: 42)
 
diff --git a/test/mitmproxy/tools/test_dump.py b/test/mitmproxy/tools/test_dump.py
index 597333af..952c3f4f 100644
--- a/test/mitmproxy/tools/test_dump.py
+++ b/test/mitmproxy/tools/test_dump.py
@@ -1,7 +1,6 @@
 import pytest
 from unittest import mock
 
-from mitmproxy import proxy
 from mitmproxy import log
 from mitmproxy import controller
 from mitmproxy import options
@@ -13,7 +12,7 @@ from .. import tservers
 class TestDumpMaster(tservers.MasterTest):
     def mkmaster(self, flt, **opts):
         o = options.Options(view_filter=flt, verbosity='error', flow_detail=0, **opts)
-        m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=False, with_dumper=False)
+        m = dump.DumpMaster(o, with_termlog=False, with_dumper=False)
         return m
 
     def test_has_error(self):
@@ -27,12 +26,12 @@ class TestDumpMaster(tservers.MasterTest):
     def test_addons_termlog(self, termlog):
         with mock.patch('sys.stdout'):
             o = options.Options()
-            m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=termlog)
+            m = dump.DumpMaster(o, with_termlog=termlog)
             assert (m.addons.get('termlog') is not None) == termlog
 
     @pytest.mark.parametrize("dumper", [False, True])
     def test_addons_dumper(self, dumper):
         with mock.patch('sys.stdout'):
             o = options.Options()
-            m = dump.DumpMaster(o, proxy.DummyServer(), with_dumper=dumper)
+            m = dump.DumpMaster(o, with_dumper=dumper)
             assert (m.addons.get('dumper') is not None) == dumper
diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py
index 4d290284..2362508f 100644
--- a/test/mitmproxy/tools/web/test_app.py
+++ b/test/mitmproxy/tools/web/test_app.py
@@ -7,7 +7,6 @@ from tornado import httpclient
 from tornado import websocket
 
 from mitmproxy import exceptions
-from mitmproxy import proxy
 from mitmproxy import options
 from mitmproxy.test import tflow
 from mitmproxy.tools.web import app
@@ -21,7 +20,7 @@ def json(resp: httpclient.HTTPResponse):
 class TestApp(tornado.testing.AsyncHTTPTestCase):
     def get_app(self):
         o = options.Options(http2=False)
-        m = webmaster.WebMaster(o, proxy.DummyServer(), with_termlog=False)
+        m = webmaster.WebMaster(o, with_termlog=False)
         f = tflow.tflow(resp=True)
         f.id = "42"
         m.view.add([f])
diff --git a/test/mitmproxy/tools/web/test_master.py b/test/mitmproxy/tools/web/test_master.py
index 27f99a18..2bceb5ca 100644
--- a/test/mitmproxy/tools/web/test_master.py
+++ b/test/mitmproxy/tools/web/test_master.py
@@ -1,7 +1,5 @@
 from mitmproxy.tools.web import master
-from mitmproxy import proxy
 from mitmproxy import options
-from mitmproxy.proxy.config import ProxyConfig
 
 from ... import tservers
 
@@ -9,8 +7,7 @@ from ... import tservers
 class TestWebMaster(tservers.MasterTest):
     def mkmaster(self, **opts):
         o = options.Options(**opts)
-        c = ProxyConfig(o)
-        return master.WebMaster(o, proxy.DummyServer(c))
+        return master.WebMaster(o)
 
     def test_basic(self):
         m = self.mkmaster()
diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py
index 9faaf20e..dd5bb327 100644
--- a/test/mitmproxy/tservers.py
+++ b/test/mitmproxy/tservers.py
@@ -69,9 +69,10 @@ class TestState:
 
 class TestMaster(taddons.RecordingMaster):
 
-    def __init__(self, opts, config):
-        s = ProxyServer(config)
-        super().__init__(opts, s)
+    def __init__(self, opts):
+        super().__init__(opts)
+        config = ProxyConfig(opts)
+        self.server = ProxyServer(config)
 
     def clear_addons(self, addons):
         self.addons.clear()
@@ -129,9 +130,8 @@ class ProxyTestBase:
             ssl=cls.ssl,
             ssloptions=cls.ssloptions)
 
-        opts = cls.get_options()
-        cls.config = ProxyConfig(opts)
-        tmaster = cls.masterclass(opts, cls.config)
+        cls.options = cls.get_options()
+        tmaster = cls.masterclass(cls.options)
         cls.proxy = ProxyThread(tmaster)
         cls.proxy.start()
 
@@ -338,19 +338,16 @@ class ChainProxyTest(ProxyTestBase):
 
     @classmethod
     def setup_class(cls):
+        # We need to initialize the chain first so that the normal server gets a correct config.
         cls.chain = []
-        super().setup_class()
         for _ in range(cls.n):
             opts = cls.get_options()
-            config = ProxyConfig(opts)
-            tmaster = cls.masterclass(opts, config)
+            tmaster = cls.masterclass(opts)
             proxy = ProxyThread(tmaster)
             proxy.start()
             cls.chain.insert(0, proxy)
 
-        # Patch the orginal proxy to upstream mode
-        opts = cls.get_options()
-        cls.config = cls.proxy.tmaster.config = cls.proxy.tmaster.server.config = ProxyConfig(opts)
+        super().setup_class()
 
     @classmethod
     def teardown_class(cls):