diff options
| -rw-r--r-- | libmproxy/protocol/http.py | 43 | ||||
| -rw-r--r-- | test/test_protocol_http.py | 119 | ||||
| -rw-r--r-- | test/test_script.py | 2 | ||||
| -rw-r--r-- | test/test_server.py | 175 | ||||
| -rw-r--r-- | test/tservers.py | 107 |
5 files changed, 223 insertions, 223 deletions
diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 253192dd..90ee127c 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1203,33 +1203,28 @@ class RequestReplayThread(threading.Thread): threading.Thread.__init__(self) def run(self): + r = self.flow.request + form_out_backup = r.form_out try: - r = self.flow.request - form_out_backup = r.form_out - - r.form_out = self.config.http_form_out - server_address, server_ssl = False, False - # If the flow is live, r.host is already the correct upstream server unless modified by a script. - # If modified by a script, we probably want to keep the modified destination. - if self.config.get_upstream_server and not self.flow.live: - try: - # this will fail in transparent mode - upstream_info = self.config.get_upstream_server(self.flow.client_conn) - server_ssl = upstream_info[1] - server_address = upstream_info[2:] - except proxy.ProxyError: - pass - if not server_address: - server_address = (r.host, r.port) - - server = ServerConnection(server_address) - server.connect() - - if server_ssl or r.scheme == "https": - if self.config.http_form_out == "absolute": # form_out == absolute -> forward mode -> send CONNECT + # In all modes, we directly connect to the server displayed + if self.config.http_form_out == "absolute": # form_out == absolute -> forward mode + server_address = self.config.get_upstream_server(self.flow.client_conn)[2:] + server = ServerConnection(server_address) + server.connect() + if r.scheme == "https": send_connect_request(server, r.host, r.port) + server.establish_ssl(self.config.clientcerts, sni=r.host) r.form_out = "relative" - server.establish_ssl(self.config.clientcerts, sni=r.host) + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + if r.scheme == "https": + server.establish_ssl(self.config.clientcerts, sni=r.host) + r.form_out = "relative" + server.send(r._assemble()) self.flow.response = HTTPResponse.from_stream(server.rfile, r.method, body_size_limit=self.config.body_size_limit) diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index 3ca590f1..41019672 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -1,5 +1,4 @@ from libmproxy.protocol.http import * -from libmproxy.protocol import KILL from cStringIO import StringIO import tutils, tservers @@ -134,120 +133,4 @@ class TestInvalidRequests(tservers.HTTPProxTest): p.connect() r = p.request("get:/p/200") assert r.status_code == 400 - assert "Invalid HTTP request form" in r.content - - -class TestProxyChaining(tservers.HTTPChainProxyTest): - def test_all(self): - self.chain[1].tmaster.replacehooks.add("~q", "foo", "bar") # replace in request - self.chain[0].tmaster.replacehooks.add("~q", "foo", "oh noes!") - self.proxy.tmaster.replacehooks.add("~q", "bar", "baz") - self.chain[0].tmaster.replacehooks.add("~s", "baz", "ORLY") # replace in response - - p = self.pathoc() - req = p.request("get:'%s/p/418:b\"foo\"'" % self.server.urlbase) - assert req.content == "ORLY" - assert req.status_code == 418 - -class TestProxyChainingSSL(tservers.HTTPChainProxyTest): - ssl = True - def test_simple(self): - p = self.pathoc() - req = p.request("get:'/p/418:b\"content\"'") - assert req.content == "content" - assert req.status_code == 418 - - assert self.chain[1].tmaster.state.flow_count() == 2 # CONNECT from pathoc to chain[0], - # request from pathoc to chain[0] - assert self.chain[0].tmaster.state.flow_count() == 2 # CONNECT from chain[1] to proxy, - # request from chain[1] to proxy - assert self.proxy.tmaster.state.flow_count() == 1 # request from chain[0] (regular proxy doesn't store CONNECTs) - - def test_closing_connect_response(self): - """ - https://github.com/mitmproxy/mitmproxy/issues/313 - """ - def handle_request(f): - f.request.httpversion = (1, 0) - del f.request.headers["Content-Length"] - f.reply() - _handle_request = self.chain[0].tmaster.handle_request - self.chain[0].tmaster.handle_request = handle_request - try: - assert self.pathoc().request("get:/p/418").status_code == 418 - finally: - self.chain[0].tmaster.handle_request = _handle_request - - def test_sni(self): - p = self.pathoc(sni="foo.com") - req = p.request("get:'/p/418:b\"content\"'") - assert req.content == "content" - assert req.status_code == 418 - -class TestProxyChainingSSLReconnect(tservers.HTTPChainProxyTest): - ssl = True - - def test_reconnect(self): - """ - Tests proper functionality of ConnectionHandler.server_reconnect mock. - If we have a disconnect on a secure connection that's transparently proxified to - an upstream http proxy, we need to send the CONNECT request again. - """ - def kill_requests(master, attr, exclude): - k = [0] # variable scope workaround: put into array - _func = getattr(master, attr) - def handler(f): - k[0] += 1 - if not (k[0] in exclude): - f.client_conn.finish() - f.error = Error("terminated") - f.reply(KILL) - return _func(f) - setattr(master, attr, handler) - - kill_requests(self.proxy.tmaster, "handle_request", - exclude=[ - # fail first request - 2, # allow second request - ]) - - kill_requests(self.chain[0].tmaster, "handle_request", - exclude=[ - 1, # CONNECT - # fail first request - 3, # reCONNECT - 4, # request - ]) - - p = self.pathoc() - req = p.request("get:'/p/418:b\"content\"'") - assert self.chain[1].tmaster.state.flow_count() == 2 # CONNECT and request - assert self.chain[0].tmaster.state.flow_count() == 4 # CONNECT, failing request, - # reCONNECT, request - assert self.proxy.tmaster.state.flow_count() == 2 # failing request, request - # (doesn't store (repeated) CONNECTs from chain[0] - # as it is a regular proxy) - assert req.content == "content" - assert req.status_code == 418 - - assert not self.proxy.tmaster.state._flow_list[0].response # killed - assert self.proxy.tmaster.state._flow_list[1].response - - assert self.chain[1].tmaster.state._flow_list[0].request.form_in == "authority" - assert self.chain[1].tmaster.state._flow_list[1].request.form_in == "relative" - - assert self.chain[0].tmaster.state._flow_list[0].request.form_in == "authority" - assert self.chain[0].tmaster.state._flow_list[1].request.form_in == "relative" - assert self.chain[0].tmaster.state._flow_list[2].request.form_in == "authority" - assert self.chain[0].tmaster.state._flow_list[3].request.form_in == "relative" - - assert self.proxy.tmaster.state._flow_list[0].request.form_in == "relative" - assert self.proxy.tmaster.state._flow_list[1].request.form_in == "relative" - - req = p.request("get:'/p/418:b\"content2\"'") - - assert req.status_code == 502 - assert self.chain[1].tmaster.state.flow_count() == 3 # + new request - assert self.chain[0].tmaster.state.flow_count() == 6 # + new request, repeated CONNECT from chain[1] - # (both terminated) - assert self.proxy.tmaster.state.flow_count() == 2 # nothing happened here + assert "Invalid HTTP request form" in r.content
\ No newline at end of file diff --git a/test/test_script.py b/test/test_script.py index 7c421fde..aed7def1 100644 --- a/test/test_script.py +++ b/test/test_script.py @@ -99,7 +99,7 @@ class TestScript: d = Dummy() assert s.run(hook, d)[0] d.reply() - while (time.time() - t_start) < 5 and m.call_count <= 5: + while (time.time() - t_start) < 20 and m.call_count <= 5: if m.call_count == 5: return time.sleep(0.001) diff --git a/test/test_server.py b/test/test_server.py index a3fff0f1..a6d591ed 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -1,12 +1,10 @@ import socket, time -import mock from libmproxy.proxy.config import ProxyConfig from netlib import tcp, http_auth, http from libpathod import pathoc, pathod from netlib.certutils import SSLCert import tutils, tservers -from libmproxy import flow -from libmproxy.protocol import KILL +from libmproxy.protocol import KILL, Error from libmproxy.protocol.http import CONTENT_MISSING """ @@ -23,8 +21,11 @@ class CommonMixin: def test_replay(self): assert self.pathod("304").status_code == 304 - assert len(self.master.state.view) == 1 - l = self.master.state.view[0] + if isinstance(self, tservers.HTTPUpstreamProxTest) and self.ssl: + assert len(self.master.state.view) == 2 + else: + assert len(self.master.state.view) == 1 + l = self.master.state.view[-1] assert l.response.code == 304 l.request.path = "/p/305" rt = self.master.replay_request(l, block=True) @@ -33,18 +34,28 @@ class CommonMixin: # Disconnect error l.request.path = "/p/305:d0" rt = self.master.replay_request(l, block=True) - assert l.error + assert not rt + if isinstance(self, tservers.HTTPUpstreamProxTest): + assert l.response.code == 502 + else: + assert l.error # Port error l.request.port = 1 - self.master.replay_request(l, block=True) - assert l.error + # In upstream mode, we get a 502 response from the upstream proxy server. + # In upstream mode with ssl, the replay will fail as we cannot establish SSL with the upstream proxy. + rt = self.master.replay_request(l, block=True) + assert not rt + if isinstance(self, tservers.HTTPUpstreamProxTest) and not self.ssl: + assert l.response.code == 502 + else: + assert l.error def test_http(self): f = self.pathod("304") assert f.status_code == 304 - l = self.master.state.view[0] + l = self.master.state.view[-1] # In Upstream mode with SSL, we may already have a previous CONNECT request. assert l.client_conn.address assert "host" in l.request.headers assert l.response.code == 304 @@ -57,6 +68,15 @@ class CommonMixin: line = t.rfile.readline() assert ("Bad Request" in line) or ("Bad Gateway" in line) + def test_sni(self): + if not self.ssl: + return + + f = self.pathod("304", sni="testserver.com") + assert f.status_code == 304 + log = self.server.last_log() + assert log["request"]["sni"] == "testserver.com" + class TcpMixin: def _ignore_on(self): conf = ProxyConfig(ignore=[".+:%s" % self.server.port]) @@ -221,12 +241,6 @@ class TestHTTPS(tservers.HTTPProxTest, CommonMixin, TcpMixin): assert f.status_code == 304 assert self.server.last_log()["request"]["clientcert"]["keyinfo"] - def test_sni(self): - f = self.pathod("304", sni="testserver.com") - assert f.status_code == 304 - l = self.server.last_log() - assert self.server.last_log()["request"]["sni"] == "testserver.com" - def test_error_post_connect(self): p = self.pathoc() assert p.request("get:/:i0,'invalid\r\n\r\n'").status_code == 400 @@ -264,11 +278,6 @@ class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin): class TestTransparentSSL(tservers.TransparentProxTest, CommonMixin, TcpMixin): ssl = True - def test_sni(self): - f = self.pathod("304", sni="testserver.com") - assert f.status_code == 304 - l = self.server.last_log() - assert l["request"]["sni"] == "testserver.com" def test_sslerr(self): p = pathoc.Pathoc(("localhost", self.proxy.port)) @@ -538,6 +547,132 @@ class TestIncompleteResponse(tservers.HTTPProxTest): class TestCertForward(tservers.HTTPProxTest): certforward = True ssl = True + def test_app_err(self): tutils.raises("handshake error", self.pathod, "200:b@100") + +class TestUpstreamProxy(tservers.HTTPUpstreamProxTest, CommonMixin, AppMixin): + ssl = False + + def test_order(self): + self.proxy.tmaster.replacehooks.add("~q", "foo", "bar") # replace in request + self.chain[0].tmaster.replacehooks.add("~q", "bar", "baz") + self.chain[1].tmaster.replacehooks.add("~q", "foo", "oh noes!") + self.chain[0].tmaster.replacehooks.add("~s", "baz", "ORLY") # replace in response + + p = self.pathoc() + req = p.request("get:'%s/p/418:b\"foo\"'" % self.server.urlbase) + assert req.content == "ORLY" + assert req.status_code == 418 + + +class TestUpstreamProxySSL(tservers.HTTPUpstreamProxTest, CommonMixin, TcpMixin): + ssl = True + + def _ignore_on(self): + super(TestUpstreamProxySSL, self)._ignore_on() + conf = ProxyConfig(ignore=[".+:%s" % self.server.port]) + for proxy in self.chain: + proxy.tmaster.server.config.ignore.append(conf.ignore[0]) + + def _ignore_off(self): + super(TestUpstreamProxySSL, self)._ignore_off() + for proxy in self.chain: + proxy.tmaster.server.config.ignore.pop() + + def test_simple(self): + p = self.pathoc() + req = p.request("get:'/p/418:b\"content\"'") + assert req.content == "content" + assert req.status_code == 418 + + assert self.proxy.tmaster.state.flow_count() == 2 # CONNECT from pathoc to chain[0], + # request from pathoc to chain[0] + assert self.chain[0].tmaster.state.flow_count() == 2 # CONNECT from proxy to chain[1], + # request from proxy to chain[1] + assert self.chain[1].tmaster.state.flow_count() == 1 # request from chain[0] (regular proxy doesn't store CONNECTs) + + def test_closing_connect_response(self): + """ + https://github.com/mitmproxy/mitmproxy/issues/313 + """ + def handle_request(f): + f.request.httpversion = (1, 0) + del f.request.headers["Content-Length"] + f.reply() + _handle_request = self.chain[0].tmaster.handle_request + self.chain[0].tmaster.handle_request = handle_request + try: + assert self.pathoc().request("get:/p/418").status_code == 418 + finally: + self.chain[0].tmaster.handle_request = _handle_request + + +class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxTest): + ssl = True + + def test_reconnect(self): + """ + Tests proper functionality of ConnectionHandler.server_reconnect mock. + If we have a disconnect on a secure connection that's transparently proxified to + an upstream http proxy, we need to send the CONNECT request again. + """ + def kill_requests(master, attr, exclude): + k = [0] # variable scope workaround: put into array + _func = getattr(master, attr) + def handler(f): + k[0] += 1 + if not (k[0] in exclude): + f.client_conn.finish() + f.error = Error("terminated") + f.reply(KILL) + return _func(f) + setattr(master, attr, handler) + + kill_requests(self.chain[1].tmaster, "handle_request", + exclude=[ + # fail first request + 2, # allow second request + ]) + + kill_requests(self.chain[0].tmaster, "handle_request", + exclude=[ + 1, # CONNECT + # fail first request + 3, # reCONNECT + 4, # request + ]) + + p = self.pathoc() + req = p.request("get:'/p/418:b\"content\"'") + assert self.proxy.tmaster.state.flow_count() == 2 # CONNECT and request + assert self.chain[0].tmaster.state.flow_count() == 4 # CONNECT, failing request, + # reCONNECT, request + assert self.chain[1].tmaster.state.flow_count() == 2 # failing request, request + # (doesn't store (repeated) CONNECTs from chain[0] + # as it is a regular proxy) + assert req.content == "content" + assert req.status_code == 418 + + assert not self.chain[1].tmaster.state._flow_list[0].response # killed + assert self.chain[1].tmaster.state._flow_list[1].response + + assert self.proxy.tmaster.state._flow_list[0].request.form_in == "authority" + assert self.proxy.tmaster.state._flow_list[1].request.form_in == "relative" + + assert self.chain[0].tmaster.state._flow_list[0].request.form_in == "authority" + assert self.chain[0].tmaster.state._flow_list[1].request.form_in == "relative" + assert self.chain[0].tmaster.state._flow_list[2].request.form_in == "authority" + assert self.chain[0].tmaster.state._flow_list[3].request.form_in == "relative" + + assert self.chain[1].tmaster.state._flow_list[0].request.form_in == "relative" + assert self.chain[1].tmaster.state._flow_list[1].request.form_in == "relative" + + req = p.request("get:'/p/418:b\"content2\"'") + + assert req.status_code == 502 + assert self.proxy.tmaster.state.flow_count() == 3 # + new request + assert self.chain[0].tmaster.state.flow_count() == 6 # + new request, repeated CONNECT from chain[1] + # (both terminated) + assert self.chain[1].tmaster.state.flow_count() == 2 # nothing happened here diff --git a/test/tservers.py b/test/tservers.py index 9f2abbe1..8a2e72a4 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -84,29 +84,19 @@ class ProxTestBase(object): masterclass = TestMaster externalapp = False certforward = False + @classmethod def setupAll(cls): cls.server = libpathod.test.Daemon(ssl=cls.ssl, ssloptions=cls.ssloptions) cls.server2 = libpathod.test.Daemon(ssl=cls.ssl, ssloptions=cls.ssloptions) - pconf = cls.get_proxy_config() - cls.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy") - cls.config = ProxyConfig( - no_upstream_cert = cls.no_upstream_cert, - confdir = cls.confdir, - authenticator = cls.authenticator, - certforward = cls.certforward, - ssl_ports=([cls.server.port, cls.server2.port] if cls.ssl else []), - **pconf - ) + + cls.config = ProxyConfig(**cls.get_proxy_config()) + tmaster = cls.masterclass(cls.config) tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp) cls.proxy = ProxyThread(tmaster) cls.proxy.start() - @property - def master(cls): - return cls.proxy.tmaster - @classmethod def teardownAll(cls): shutil.rmtree(cls.confdir) @@ -121,24 +111,20 @@ class ProxTestBase(object): self.server2.clear_log() @property - def scheme(self): - return "https" if self.ssl else "http" - - @property - def proxies(self): - """ - The URL base for the server instance. - """ - return ( - (self.scheme, ("127.0.0.1", self.proxy.port)) - ) + def master(self): + return self.proxy.tmaster @classmethod def get_proxy_config(cls): - d = dict() - if cls.clientcerts: - d["clientcerts"] = tutils.test_data.path("data/clientcert") - return d + cls.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy") + return dict( + no_upstream_cert = cls.no_upstream_cert, + confdir = cls.confdir, + authenticator = cls.authenticator, + certforward = cls.certforward, + ssl_ports=([cls.server.port, cls.server2.port] if cls.ssl else []), + clientcerts = tutils.test_data.path("data/clientcert") if cls.clientcerts else None + ) class HTTPProxTest(ProxTestBase): @@ -265,49 +251,50 @@ class ReverseProxTest(ProxTestBase): class ChainProxTest(ProxTestBase): """ - Chain n instances of mitmproxy in a row - because we can. + Chain three instances of mitmproxy in a row to test upstream mode. + Proxy order is cls.proxy -> cls.chain[0] -> cls.chain[1] + cls.proxy and cls.chain[0] are in upstream mode, + cls.chain[1] is in regular mode. """ + chain = None n = 2 - chain_config = [lambda port, sslports: ProxyConfig( - upstream_server= (False, False, "127.0.0.1", port), - http_form_in = "absolute", - http_form_out = "absolute", - ssl_ports=sslports - )] * n + @classmethod def setupAll(cls): - super(ChainProxTest, cls).setupAll() cls.chain = [] - for i in range(cls.n): - sslports = [cls.server.port, cls.server2.port] - config = cls.chain_config[i](cls.proxy.port if i == 0 else cls.chain[-1].port, - sslports) + super(ChainProxTest, cls).setupAll() + for _ in range(cls.n): + config = ProxyConfig(**cls.get_proxy_config()) tmaster = cls.masterclass(config) - tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp) - cls.chain.append(ProxyThread(tmaster)) - cls.chain[-1].start() + proxy = ProxyThread(tmaster) + proxy.start() + cls.chain.insert(0, proxy) + + # Patch the orginal proxy to upstream mode + cls.config = cls.proxy.tmaster.config = cls.proxy.tmaster.server.config = ProxyConfig(**cls.get_proxy_config()) + @classmethod def teardownAll(cls): super(ChainProxTest, cls).teardownAll() - for p in cls.chain: - p.tmaster.shutdown() + for proxy in cls.chain: + proxy.shutdown() def setUp(self): super(ChainProxTest, self).setUp() - for p in self.chain: - p.tmaster.clear_log() - p.tmaster.state.clear() + for proxy in self.chain: + proxy.tmaster.clear_log() + proxy.tmaster.state.clear() + @classmethod + def get_proxy_config(cls): + d = super(ChainProxTest, cls).get_proxy_config() + if cls.chain: # First proxy is in normal mode. + d.update( + mode="upstream", + upstream_server=(False, False, "127.0.0.1", cls.chain[0].port) + ) + return d -class HTTPChainProxyTest(ChainProxTest): - def pathoc(self, sni=None): - """ - Returns a connected Pathoc instance. - """ - p = libpathod.pathoc.Pathoc(("localhost", self.chain[-1].port), ssl=self.ssl, sni=sni) - if self.ssl: - p.connect(("127.0.0.1", self.server.port)) - else: - p.connect() - return p +class HTTPUpstreamProxTest(ChainProxTest, HTTPProxTest): + pass
\ No newline at end of file |