diff options
Diffstat (limited to 'test')
| -rw-r--r-- | test/requirements.txt | 5 | ||||
| -rw-r--r-- | test/test_dump.py | 20 | ||||
| -rw-r--r-- | test/test_examples.py | 4 | ||||
| -rw-r--r-- | test/test_flow.py | 130 | ||||
| -rw-r--r-- | test/test_proxy.py | 20 | ||||
| -rw-r--r-- | test/test_server.py | 101 | ||||
| -rw-r--r-- | test/tools/passive_close.py | 21 | ||||
| -rw-r--r-- | test/tservers.py | 6 | ||||
| -rw-r--r-- | test/tutils.py | 4 |
9 files changed, 227 insertions, 84 deletions
diff --git a/test/requirements.txt b/test/requirements.txt deleted file mode 100644 index 89e4aa0a..00000000 --- a/test/requirements.txt +++ /dev/null @@ -1,5 +0,0 @@ -mock>=1.0.1 -nose>=1.3.0 -nose-cov>=1.6 -coveralls>=0.4.1 -pathod>=0.10
\ No newline at end of file diff --git a/test/test_dump.py b/test/test_dump.py index fd93cc03..2e58e073 100644 --- a/test/test_dump.py +++ b/test/test_dump.py @@ -35,8 +35,8 @@ class TestDumpMaster: def _dummy_cycle(self, n, filt, content, **options): cs = StringIO() - o = dump.Options(**options) - m = dump.DumpMaster(None, o, filt, outfile=cs) + o = dump.Options(filtstr=filt, **options) + m = dump.DumpMaster(None, o, outfile=cs) for i in range(n): self._cycle(m, content) m.shutdown() @@ -52,7 +52,7 @@ class TestDumpMaster: def test_error(self): cs = StringIO() o = dump.Options(flow_detail=1) - m = dump.DumpMaster(None, o, None, outfile=cs) + m = dump.DumpMaster(None, o, outfile=cs) f = tutils.tflow(err=True) m.handle_request(f) assert m.handle_error(f) @@ -62,24 +62,24 @@ class TestDumpMaster: cs = StringIO() o = dump.Options(server_replay="nonexistent", kill=True) - tutils.raises(dump.DumpError, dump.DumpMaster, None, o, None, outfile=cs) + tutils.raises(dump.DumpError, dump.DumpMaster, None, o, outfile=cs) with tutils.tmpdir() as t: p = os.path.join(t, "rep") self._flowfile(p) o = dump.Options(server_replay=p, kill=True) - m = dump.DumpMaster(None, o, None, outfile=cs) + m = dump.DumpMaster(None, o, outfile=cs) self._cycle(m, "content") self._cycle(m, "content") o = dump.Options(server_replay=p, kill=False) - m = dump.DumpMaster(None, o, None, outfile=cs) + m = dump.DumpMaster(None, o, outfile=cs) self._cycle(m, "nonexistent") o = dump.Options(client_replay=p, kill=False) - m = dump.DumpMaster(None, o, None, outfile=cs) + m = dump.DumpMaster(None, o, outfile=cs) def test_read(self): with tutils.tmpdir() as t: @@ -105,18 +105,18 @@ class TestDumpMaster: def test_app(self): o = dump.Options(app=True) s = mock.MagicMock() - m = dump.DumpMaster(s, o, None) + m = dump.DumpMaster(s, o) assert len(m.apps.apps) == 1 def test_replacements(self): o = dump.Options(replacements=[(".*", "content", "foo")]) - m = dump.DumpMaster(None, o, None) + m = dump.DumpMaster(None, o) f = self._cycle(m, "content") assert f.request.content == "foo" def test_setheader(self): o = dump.Options(setheaders=[(".*", "one", "two")]) - m = dump.DumpMaster(None, o, None) + m = dump.DumpMaster(None, o) f = self._cycle(m, "content") assert f.request.headers["one"] == ["two"] diff --git a/test/test_examples.py b/test/test_examples.py index d557080e..fd42e6f0 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -1,11 +1,11 @@ import glob +import mock from libmproxy import utils, script from libmproxy.proxy import config import tservers - +@mock.patch.dict("sys.modules", {"bs4": mock.Mock()}) def test_load_scripts(): - example_dir = utils.Data("libmproxy").path("../examples") scripts = glob.glob("%s/*.py" % example_dir) diff --git a/test/test_flow.py b/test/test_flow.py index 399c8827..6ed279c2 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -5,6 +5,7 @@ import mock from libmproxy import filt, protocol, controller, utils, tnetstring, flow from libmproxy.protocol.primitives import Error, Flow from libmproxy.protocol.http import decoded, CONTENT_MISSING +from libmproxy.proxy.config import HostMatcher from libmproxy.proxy import ProxyConfig from libmproxy.proxy.server import DummyServer from libmproxy.proxy.connection import ClientConnection @@ -104,7 +105,7 @@ class TestClientPlaybackState: q = Queue.Queue() fm.state.clear() - fm.tick(q) + fm.tick(q, timeout=0) fm.stop_client_playback() assert not fm.client_playback @@ -112,7 +113,7 @@ class TestClientPlaybackState: class TestServerPlaybackState: def test_hash(self): - s = flow.ServerPlaybackState(None, [], False, False) + s = flow.ServerPlaybackState(None, [], False, False, None, False) r = tutils.tflow() r2 = tutils.tflow() @@ -124,7 +125,7 @@ class TestServerPlaybackState: assert s._hash(r) != s._hash(r2) def test_headers(self): - s = flow.ServerPlaybackState(["foo"], [], False, False) + s = flow.ServerPlaybackState(["foo"], [], False, False, None, False) r = tutils.tflow(resp=True) r.request.headers["foo"] = ["bar"] r2 = tutils.tflow(resp=True) @@ -145,7 +146,7 @@ class TestServerPlaybackState: r2 = tutils.tflow(resp=True) r2.request.headers["key"] = ["two"] - s = flow.ServerPlaybackState(None, [r, r2], False, False) + s = flow.ServerPlaybackState(None, [r, r2], False, False, None, False) assert s.count() == 2 assert len(s.fmap.keys()) == 1 @@ -166,28 +167,68 @@ class TestServerPlaybackState: r2 = tutils.tflow(resp=True) r2.request.headers["key"] = ["two"] - s = flow.ServerPlaybackState(None, [r, r2], False, True) + s = flow.ServerPlaybackState(None, [r, r2], False, True, None, False) assert s.count() == 2 s.next_flow(r) assert s.count() == 2 + def test_ignore_params(self): + s = flow.ServerPlaybackState(None, [], False, False, ["param1", "param2"], False) + r = tutils.tflow(resp=True) + r.request.path="/test?param1=1" + r2 = tutils.tflow(resp=True) + r2.request.path="/test" + assert s._hash(r) == s._hash(r2) + r2.request.path="/test?param1=2" + assert s._hash(r) == s._hash(r2) + r2.request.path="/test?param2=1" + assert s._hash(r) == s._hash(r2) + r2.request.path="/test?param3=2" + assert not s._hash(r) == s._hash(r2) + + def test_ignore_content(self): + s = flow.ServerPlaybackState(None, [], False, False, None, False) + r = tutils.tflow(resp=True) + r2 = tutils.tflow(resp=True) + + r.request.content = "foo" + r2.request.content = "foo" + assert s._hash(r) == s._hash(r2) + r2.request.content = "bar" + assert not s._hash(r) == s._hash(r2) + + #now ignoring content + s = flow.ServerPlaybackState(None, [], False, False, None, True) + r = tutils.tflow(resp=True) + r2 = tutils.tflow(resp=True) + r.request.content = "foo" + r2.request.content = "foo" + assert s._hash(r) == s._hash(r2) + r2.request.content = "bar" + assert s._hash(r) == s._hash(r2) + r2.request.content = "" + assert s._hash(r) == s._hash(r2) + r2.request.content = None + assert s._hash(r) == s._hash(r2) + + class TestFlow: def test_copy(self): f = tutils.tflow(resp=True) - a0 = f._get_state() + a0 = f.get_state() f2 = f.copy() - a = f._get_state() - b = f2._get_state() - assert f._get_state() == f2._get_state() + a = f.get_state() + b = f2.get_state() + assert f.get_state() == f2.get_state() assert not f == f2 assert not f is f2 - assert f.request == f2.request + assert f.request.get_state() == f2.request.get_state() assert not f.request is f2.request assert f.request.headers == f2.request.headers assert not f.request.headers is f2.request.headers - assert f.response == f2.response + assert f.response.get_state() == f2.response.get_state() assert not f.response is f2.response f = tutils.tflow(err=True) @@ -196,7 +237,7 @@ class TestFlow: assert not f.request is f2.request assert f.request.headers == f2.request.headers assert not f.request.headers is f2.request.headers - assert f.error == f2.error + assert f.error.get_state() == f2.error.get_state() assert not f.error is f2.error def test_match(self): @@ -230,21 +271,21 @@ class TestFlow: def test_getset_state(self): f = tutils.tflow(resp=True) - state = f._get_state() - assert f._get_state() == protocol.http.HTTPFlow._from_state(state)._get_state() + state = f.get_state() + assert f.get_state() == protocol.http.HTTPFlow.from_state(state).get_state() f.response = None f.error = Error("error") - state = f._get_state() - assert f._get_state() == protocol.http.HTTPFlow._from_state(state)._get_state() + state = f.get_state() + assert f.get_state() == protocol.http.HTTPFlow.from_state(state).get_state() f2 = f.copy() - assert f._get_state() == f2._get_state() + assert f.get_state() == f2.get_state() assert not f == f2 f2.error = Error("e2") assert not f == f2 - f._load_state(f2._get_state()) - assert f._get_state() == f2._get_state() + f.load_state(f2.get_state()) + assert f.get_state() == f2.get_state() def test_kill(self): s = flow.State() @@ -482,7 +523,7 @@ class TestSerialize: assert len(l) == 1 f2 = l[0] - assert f2._get_state() == f._get_state() + assert f2.get_state() == f.get_state() assert f2.request.assemble() == f.request.assemble() def test_load_flows(self): @@ -530,7 +571,7 @@ class TestSerialize: def test_versioncheck(self): f = tutils.tflow() - d = f._get_state() + d = f.get_state() d["version"] = (0, 0) sio = StringIO() tnetstring.dump(d, sio) @@ -553,11 +594,11 @@ class TestFlowMaster: def test_getset_ignore(self): p = mock.Mock() - p.config.ignore = [] + p.config.check_ignore = HostMatcher() fm = flow.FlowMaster(p, flow.State()) - assert not fm.get_ignore() - fm.set_ignore(["^apple\.com:", ":443$"]) - assert fm.get_ignore() + assert not fm.get_ignore_filter() + fm.set_ignore_filter(["^apple\.com:", ":443$"]) + assert fm.get_ignore_filter() def test_replay(self): s = flow.State() @@ -569,6 +610,9 @@ class TestFlowMaster: f.intercepting = True assert "intercepting" in fm.replay_request(f) + f.live = True + assert "live" in fm.replay_request(f) + def test_script_reqerr(self): s = flow.State() fm = flow.FlowMaster(None, s) @@ -649,12 +693,12 @@ class TestFlowMaster: f = tutils.tflow(resp=True) pb = [tutils.tflow(resp=True), f] fm = flow.FlowMaster(None, s) - assert not fm.start_server_playback(pb, False, [], False, False) + assert not fm.start_server_playback(pb, False, [], False, False, None, False) assert not fm.start_client_playback(pb, False) q = Queue.Queue() assert not fm.state.flow_count() - fm.tick(q) + fm.tick(q, 0) assert fm.state.flow_count() f.error = Error("error") @@ -671,18 +715,18 @@ class TestFlowMaster: fm.refresh_server_playback = True assert not fm.do_server_playback(tutils.tflow()) - fm.start_server_playback(pb, False, [], False, False) + fm.start_server_playback(pb, False, [], False, False, None, False) assert fm.do_server_playback(tutils.tflow()) - fm.start_server_playback(pb, False, [], True, False) + fm.start_server_playback(pb, False, [], True, False, None, False) r = tutils.tflow() r.request.content = "gibble" assert not fm.do_server_playback(r) assert fm.do_server_playback(tutils.tflow()) - fm.start_server_playback(pb, False, [], True, False) + fm.start_server_playback(pb, False, [], True, False, None, False) q = Queue.Queue() - fm.tick(q) + fm.tick(q, 0) assert fm.should_exit.is_set() fm.stop_server_playback() @@ -695,7 +739,7 @@ class TestFlowMaster: pb = [f] fm = flow.FlowMaster(None, s) fm.refresh_server_playback = True - fm.start_server_playback(pb, True, [], False, False) + fm.start_server_playback(pb, True, [], False, False, None, False) f = tutils.tflow() f.request.host = "nonexistent" @@ -779,7 +823,7 @@ class TestRequest: assert r.size() == len(r.assemble()) r2 = r.copy() - assert r == r2 + assert r.get_state() == r2.get_state() r.content = None assert r.assemble() @@ -988,7 +1032,7 @@ class TestResponse: assert resp.size() == len(resp.assemble()) resp2 = resp.copy() - assert resp2 == resp + assert resp2.get_state() == resp.get_state() resp.content = None assert resp.assemble() @@ -1131,37 +1175,37 @@ class TestResponse: class TestError: def test_getset_state(self): e = Error("Error") - state = e._get_state() - assert Error._from_state(state) == e + state = e.get_state() + assert Error.from_state(state).get_state() == e.get_state() assert e.copy() e2 = Error("bar") assert not e == e2 - e._load_state(e2._get_state()) - assert e == e2 - + e.load_state(e2.get_state()) + assert e.get_state() == e2.get_state() e3 = e.copy() - assert e3 == e + assert e3.get_state() == e.get_state() class TestClientConnection: def test_state(self): c = tutils.tclient_conn() - assert ClientConnection._from_state(c._get_state()) == c + assert ClientConnection.from_state(c.get_state()).get_state() ==\ + c.get_state() c2 = tutils.tclient_conn() c2.address.address = (c2.address.host, 4242) assert not c == c2 c2.timestamp_start = 42 - c._load_state(c2._get_state()) + c.load_state(c2.get_state()) assert c.timestamp_start == 42 c3 = c.copy() - assert c3 == c + assert c3.get_state() == c.get_state() assert str(c) diff --git a/test/test_proxy.py b/test/test_proxy.py index 5f1b83f6..c396183b 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -1,6 +1,6 @@ import argparse from libmproxy import cmdline -from libmproxy.proxy.config import process_proxy_options +from libmproxy.proxy import ProxyConfig, process_proxy_options from libmproxy.proxy.connection import ServerConnection from libmproxy.proxy.primitives import ProxyError from libmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler @@ -119,16 +119,16 @@ class TestProcessProxyOptions: class TestProxyServer: @tutils.SkipWindows # binding to 0.0.0.0:1 works without special permissions on Windows def test_err(self): - parser = argparse.ArgumentParser() - cmdline.common_options(parser) - opts = parser.parse_args(args=[]) - tutils.raises("error starting proxy server", ProxyServer, opts, 1) + conf = ProxyConfig( + port=1 + ) + tutils.raises("error starting proxy server", ProxyServer, conf) def test_err_2(self): - parser = argparse.ArgumentParser() - cmdline.common_options(parser) - opts = parser.parse_args(args=[]) - tutils.raises("error starting proxy server", ProxyServer, opts, 8080, "invalidhost") + conf = ProxyConfig( + host="invalidhost" + ) + tutils.raises("error starting proxy server", ProxyServer, conf) class TestDummyServer: @@ -142,6 +142,6 @@ class TestConnectionHandler: def test_fatal_error(self): config = mock.Mock() config.mode.get_upstream_server.side_effect = RuntimeError - c = ConnectionHandler(config, mock.MagicMock(), ("127.0.0.1", 8080), None, mock.MagicMock(), None) + c = ConnectionHandler(config, mock.MagicMock(), ("127.0.0.1", 8080), None, mock.MagicMock()) with tutils.capture_stderr(c.handle) as output: assert "mitmproxy has crashed" in output diff --git a/test/test_server.py b/test/test_server.py index 0ce5d056..c81eab2b 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -1,5 +1,5 @@ import socket, time -from libmproxy.proxy.config import parse_host_pattern +from libmproxy.proxy.config import HostMatcher from netlib import tcp, http_auth, http from libpathod import pathoc, pathod from netlib.certutils import SSLCert @@ -19,6 +19,17 @@ class CommonMixin: def test_large(self): assert len(self.pathod("200:b@50k").content) == 1024*50 + @staticmethod + def wait_until_not_live(flow): + """ + Race condition: We don't want to replay the flow while it is still live. + """ + s = time.time() + while flow.live: + time.sleep(0.001) + if time.time() - s > 5: + raise RuntimeError("Flow is live for too long.") + def test_replay(self): assert self.pathod("304").status_code == 304 if isinstance(self, tservers.HTTPUpstreamProxTest) and self.ssl: @@ -28,6 +39,7 @@ class CommonMixin: l = self.master.state.view[-1] assert l.response.code == 304 l.request.path = "/p/305" + self.wait_until_not_live(l) rt = self.master.replay_request(l, block=True) assert l.response.code == 305 @@ -79,11 +91,14 @@ class CommonMixin: class TcpMixin: def _ignore_on(self): - ignore = parse_host_pattern([".+:%s" % self.server.port])[0] - self.config.ignore.append(ignore) + assert not hasattr(self, "_ignore_backup") + self._ignore_backup = self.config.check_ignore + self.config.check_ignore = HostMatcher([".+:%s" % self.server.port] + self.config.check_ignore.patterns) def _ignore_off(self): - self.config.ignore.pop() + assert hasattr(self, "_ignore_backup") + self.config.check_ignore = self._ignore_backup + del self._ignore_backup def test_ignore(self): spec = '304:h"Alternate-Protocol"="mitmproxy-will-remove-this"' @@ -114,6 +129,40 @@ class TcpMixin: tutils.raises("invalid server response", self.pathod, spec) # pathoc tries to parse answer as HTTP self._ignore_off() + def _tcpproxy_on(self): + assert not hasattr(self, "_tcpproxy_backup") + self._tcpproxy_backup = self.config.check_tcp + self.config.check_tcp = HostMatcher([".+:%s" % self.server.port] + self.config.check_tcp.patterns) + + def _tcpproxy_off(self): + assert hasattr(self, "_tcpproxy_backup") + self.config.check_ignore = self._tcpproxy_backup + del self._tcpproxy_backup + + + def test_tcp(self): + spec = '304:h"Alternate-Protocol"="mitmproxy-will-remove-this"' + n = self.pathod(spec) + self._tcpproxy_on() + i = self.pathod(spec) + i2 = self.pathod(spec) + self._tcpproxy_off() + + assert i.status_code == i2.status_code == n.status_code == 304 + assert "Alternate-Protocol" in i.headers + assert "Alternate-Protocol" in i2.headers + assert "Alternate-Protocol" not in n.headers + + # Test that we get the original SSL cert + if self.ssl: + i_cert = SSLCert(i.sslinfo.certchain[0]) + i2_cert = SSLCert(i2.sslinfo.certchain[0]) + n_cert = SSLCert(n.sslinfo.certchain[0]) + + assert i_cert == i2_cert == n_cert + + # Make sure that TCP messages are in the event log. + assert any("mitmproxy-will-remove-this" in m for m in self.master.log) class AppMixin: def test_app(self): @@ -579,16 +628,50 @@ class TestUpstreamProxy(tservers.HTTPUpstreamProxTest, CommonMixin, AppMixin): class TestUpstreamProxySSL(tservers.HTTPUpstreamProxTest, CommonMixin, TcpMixin): ssl = True + def _host_pattern_on(self, attr): + """ + Updates config.check_tcp or check_ignore, depending on attr. + """ + assert not hasattr(self, "_ignore_%s_backup" % attr) + backup = [] + for proxy in self.chain: + old_matcher = getattr(proxy.tmaster.server.config, "check_%s" % attr) + backup.append(old_matcher) + setattr( + proxy.tmaster.server.config, + "check_%s" % attr, + HostMatcher([".+:%s" % self.server.port] + old_matcher.patterns) + ) + + setattr(self, "_ignore_%s_backup" % attr, backup) + + def _host_pattern_off(self, attr): + backup = getattr(self, "_ignore_%s_backup" % attr) + for proxy in reversed(self.chain): + setattr( + proxy.tmaster.server.config, + "check_%s" % attr, + backup.pop() + ) + + assert not backup + delattr(self, "_ignore_%s_backup" % attr) + def _ignore_on(self): super(TestUpstreamProxySSL, self)._ignore_on() - ignore = parse_host_pattern([".+:%s" % self.server.port])[0] - for proxy in self.chain: - proxy.tmaster.server.config.ignore.append(ignore) + self._host_pattern_on("ignore") def _ignore_off(self): super(TestUpstreamProxySSL, self)._ignore_off() - for proxy in self.chain: - proxy.tmaster.server.config.ignore.pop() + self._host_pattern_off("ignore") + + def _tcpproxy_on(self): + super(TestUpstreamProxySSL, self)._tcpproxy_on() + self._host_pattern_on("tcp") + + def _tcpproxy_off(self): + super(TestUpstreamProxySSL, self)._tcpproxy_off() + self._host_pattern_off("tcp") def test_simple(self): p = self.pathoc() diff --git a/test/tools/passive_close.py b/test/tools/passive_close.py new file mode 100644 index 00000000..d0b36e7f --- /dev/null +++ b/test/tools/passive_close.py @@ -0,0 +1,21 @@ +import SocketServer +from threading import Thread +from time import sleep + +class service(SocketServer.BaseRequestHandler): + def handle(self): + data = 'dummy' + print "Client connected with ", self.client_address + while True: + self.request.send("HTTP/1.1 200 OK\r\nConnection: close\r\nContent-Length: 7\r\n\r\ncontent") + data = self.request.recv(1024) + if not len(data): + print "Connection closed by remote: ", self.client_address + sleep(3600) + + +class ThreadedTCPServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer): + pass + +server = ThreadedTCPServer(('',1520), service) +server.serve_forever() diff --git a/test/tservers.py b/test/tservers.py index 009a3c92..93c8a80a 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -29,7 +29,8 @@ def errapp(environ, start_response): class TestMaster(flow.FlowMaster): def __init__(self, config): - s = ProxyServer(config, 0) + config.port = 0 + s = ProxyServer(config) state = flow.State() flow.FlowMaster.__init__(self, s, state) self.apps.add(testapp, "testapp", 80) @@ -82,7 +83,6 @@ class ProxTestBase(object): no_upstream_cert = False authenticator = None masterclass = TestMaster - externalapp = False certforward = False @classmethod @@ -93,7 +93,7 @@ class ProxTestBase(object): cls.config = ProxyConfig(**cls.get_proxy_config()) tmaster = cls.masterclass(cls.config) - tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp) + tmaster.start_app(APP_HOST, APP_PORT) cls.proxy = ProxyThread(tmaster) cls.proxy.start() diff --git a/test/tutils.py b/test/tutils.py index 69f79a91..e7720d33 100644 --- a/test/tutils.py +++ b/test/tutils.py @@ -55,7 +55,7 @@ def tclient_conn(): """ @return: libmproxy.proxy.connection.ClientConnection """ - c = ClientConnection._from_state(dict( + c = ClientConnection.from_state(dict( address=dict(address=("address", 22), use_ipv6=True), clientcert=None )) @@ -67,7 +67,7 @@ def tserver_conn(): """ @return: libmproxy.proxy.connection.ServerConnection """ - c = ServerConnection._from_state(dict( + c = ServerConnection.from_state(dict( address=dict(address=("address", 22), use_ipv6=True), state=[], source_address=dict(address=("address", 22), use_ipv6=True), |