From a2d45193546962e2e14d1959e1bf008c83b9f3cf Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 24 Mar 2018 12:03:50 +1300 Subject: asyncio: brutally rip out our old queue mechanism --- test/mitmproxy/proxy/protocol/test_http2.py | 4 +- test/mitmproxy/proxy/protocol/test_websocket.py | 4 +- test/mitmproxy/proxy/test_server.py | 8 -- test/mitmproxy/test_controller.py | 140 ++++++++++++++---------- test/mitmproxy/test_flow.py | 7 +- test/mitmproxy/tservers.py | 40 +++---- 6 files changed, 106 insertions(+), 97 deletions(-) (limited to 'test') diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index d9aa03b4..8b929995 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -90,9 +90,7 @@ class _Http2TestBase: @classmethod def setup_class(cls): cls.options = cls.get_options() - tmaster = tservers.TestMaster(cls.options) - tmaster.addons.add(core.Core()) - cls.proxy = tservers.ProxyThread(tmaster) + cls.proxy = tservers.ProxyThread(tservers.TestMaster, cls.options) cls.proxy.start() @classmethod diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py index 661605b7..2a343450 100644 --- a/test/mitmproxy/proxy/protocol/test_websocket.py +++ b/test/mitmproxy/proxy/protocol/test_websocket.py @@ -52,9 +52,7 @@ class _WebSocketTestBase: @classmethod def setup_class(cls): cls.options = cls.get_options() - tmaster = tservers.TestMaster(cls.options) - tmaster.addons.add(core.Core()) - cls.proxy = tservers.ProxyThread(tmaster) + cls.proxy = tservers.ProxyThread(tservers.TestMaster, cls.options) cls.proxy.start() @classmethod diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 986dfb39..4cfaa523 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -21,14 +21,6 @@ from pathod import pathod from .. import tservers from ...conftest import skip_appveyor -""" - Note that the choice of response code in these tests matters more than you - might think. libcurl treats a 304 response code differently from, say, a - 200 response code - it will correctly terminate a 304 response with no - content-length header, whereas it will block forever waiting for content - for a 200 response. -""" - class CommonMixin: diff --git a/test/mitmproxy/test_controller.py b/test/mitmproxy/test_controller.py index e840380a..e27f6baf 100644 --- a/test/mitmproxy/test_controller.py +++ b/test/mitmproxy/test_controller.py @@ -1,7 +1,9 @@ +import asyncio from threading import Thread, Event from unittest.mock import Mock import queue import pytest +import sys from mitmproxy.exceptions import Kill, ControlException from mitmproxy import controller @@ -14,69 +16,87 @@ class TMsg: pass -class TestMaster: - def test_simple(self): - class tAddon: - def log(self, _): - ctx.master.should_exit.set() +def test_master(): + class tAddon: + def log(self, _): + ctx.master.should_exit.set() + + with taddons.context() as ctx: + ctx.master.addons.add(tAddon()) + assert not ctx.master.should_exit.is_set() - with taddons.context() as ctx: - ctx.master.addons.add(tAddon()) - assert not ctx.master.should_exit.is_set() + async def test(): msg = TMsg() msg.reply = controller.DummyReply() - ctx.master.event_queue.put(("log", msg)) - ctx.master.run() - assert ctx.master.should_exit.is_set() - - def test_server_simple(self): - m = master.Master(None) - m.server = proxy.DummyServer() - m.start() - m.shutdown() - m.start() - m.shutdown() - - -class TestServerThread: - def test_simple(self): - m = Mock() - t = master.ServerThread(m) - t.run() - assert m.serve_forever.called - - -class TestChannel: - def test_tell(self): - q = queue.Queue() - channel = controller.Channel(q, Event()) - m = Mock(name="test_tell") - channel.tell("test", m) - assert q.get() == ("test", m) - assert m.reply - - def test_ask_simple(self): - q = queue.Queue() - - def reply(): - m, obj = q.get() - assert m == "test" - obj.reply.send(42) - obj.reply.take() - obj.reply.commit() - - Thread(target=reply).start() - - channel = controller.Channel(q, Event()) - assert channel.ask("test", Mock(name="test_ask_simple")) == 42 - - def test_ask_shutdown(self): - q = queue.Queue() - done = Event() - done.set() - channel = controller.Channel(q, done) - with pytest.raises(Kill): - channel.ask("test", Mock(name="test_ask_shutdown")) + await ctx.master.event_queue.put(("log", msg)) + + ctx.master.run(inject=test) + + +# class TestMaster: +# # def test_simple(self): +# # class tAddon: +# # def log(self, _): +# # ctx.master.should_exit.set() + +# # with taddons.context() as ctx: +# # ctx.master.addons.add(tAddon()) +# # assert not ctx.master.should_exit.is_set() +# # msg = TMsg() +# # msg.reply = controller.DummyReply() +# # ctx.master.event_queue.put(("log", msg)) +# # ctx.master.run() +# # assert ctx.master.should_exit.is_set() + +# # def test_server_simple(self): +# # m = master.Master(None) +# # m.server = proxy.DummyServer() +# # m.start() +# # m.shutdown() +# # m.start() +# # m.shutdown() +# pass + + +# class TestServerThread: +# def test_simple(self): +# m = Mock() +# t = master.ServerThread(m) +# t.run() +# assert m.serve_forever.called + + +# class TestChannel: +# def test_tell(self): +# q = queue.Queue() +# channel = controller.Channel(q, Event()) +# m = Mock(name="test_tell") +# channel.tell("test", m) +# assert q.get() == ("test", m) +# assert m.reply + +# def test_ask_simple(self): +# q = queue.Queue() + +# def reply(): +# m, obj = q.get() +# assert m == "test" +# obj.reply.send(42) +# obj.reply.take() +# obj.reply.commit() + +# Thread(target=reply).start() + +# channel = controller.Channel(q, Event()) +# assert channel.ask("test", Mock(name="test_ask_simple")) == 42 + +# def test_ask_shutdown(self): +# q = queue.Queue() +# done = Event() +# done.set() +# channel = controller.Channel(q, done) +# with pytest.raises(Kill): +# channel.ask("test", Mock(name="test_ask_shutdown")) class TestReply: diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 8cc11a16..9f1fb213 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -169,9 +169,10 @@ class TestFlowMaster: f.error = flow.Error("msg") fm.addons.handle_lifecycle("error", f) - fm.tell("foo", f) - with pytest.raises(ControlException): - fm.tick(timeout=1) + # FIXME: This no longer works, because we consume on the main loop. + # fm.tell("foo", f) + # with pytest.raises(ControlException): + # fm.addons.trigger("unknown") fm.shutdown() diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 0040b023..7be31a28 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -2,6 +2,7 @@ import os.path import threading import tempfile import sys +import time from unittest import mock import mitmproxy.platform @@ -62,11 +63,6 @@ class TestState: if f not in self.flows: self.flows.append(f) - # TODO: add TCP support? - # def tcp_start(self, f): - # if f not in self.flows: - # self.flows.append(f) - class TestMaster(taddons.RecordingMaster): @@ -90,13 +86,11 @@ class TestMaster(taddons.RecordingMaster): class ProxyThread(threading.Thread): - def __init__(self, tmaster): + def __init__(self, masterclass, options): threading.Thread.__init__(self) - self.tmaster = tmaster - self.name = "ProxyThread (%s:%s)" % ( - tmaster.server.address[0], - tmaster.server.address[1], - ) + self.masterclass = masterclass + self.options = options + self.tmaster = None controller.should_exit = False @property @@ -107,12 +101,18 @@ class ProxyThread(threading.Thread): def tlog(self): return self.tmaster.logs - def run(self): - self.tmaster.run() - def shutdown(self): self.tmaster.shutdown() + def run(self): + self.tmaster = self.masterclass(self.options) + self.tmaster.addons.add(core.Core()) + self.name = "ProxyThread (%s:%s)" % ( + self.tmaster.server.address[0], + self.tmaster.server.address[1], + ) + self.tmaster.run() + class ProxyTestBase: # Test Configuration @@ -132,10 +132,12 @@ class ProxyTestBase: ssloptions=cls.ssloptions) cls.options = cls.get_options() - tmaster = cls.masterclass(cls.options) - tmaster.addons.add(core.Core()) - cls.proxy = ProxyThread(tmaster) + cls.proxy = ProxyThread(cls.masterclass, cls.options) cls.proxy.start() + while True: + if cls.proxy.tmaster: + break + time.sleep(0.01) @classmethod def teardown_class(cls): @@ -344,9 +346,7 @@ class ChainProxyTest(ProxyTestBase): cls.chain = [] for _ in range(cls.n): opts = cls.get_options() - tmaster = cls.masterclass(opts) - tmaster.addons.add(core.Core()) - proxy = ProxyThread(tmaster) + proxy = ProxyThread(cls.masterclass, opts) proxy.start() cls.chain.insert(0, proxy) -- cgit v1.2.3 From 976b2018a3fc320272ac4f588250977fc08cf9b5 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 31 Mar 2018 17:06:09 +1300 Subject: asyncio: clean up event loop acquisition We now acquire the event loop through asyncio.get_event_loop, avoiding having to pass the loop explicity in a bunch of places. This function does not return the currently running loop from within coroutines in versions of Python prior to 3.6. --- test/mitmproxy/test_controller.py | 89 ++++----------------------------------- 1 file changed, 9 insertions(+), 80 deletions(-) (limited to 'test') diff --git a/test/mitmproxy/test_controller.py b/test/mitmproxy/test_controller.py index e27f6baf..f7c64ed9 100644 --- a/test/mitmproxy/test_controller.py +++ b/test/mitmproxy/test_controller.py @@ -1,102 +1,31 @@ import asyncio -from threading import Thread, Event -from unittest.mock import Mock import queue import pytest -import sys from mitmproxy.exceptions import Kill, ControlException from mitmproxy import controller -from mitmproxy import master -from mitmproxy import proxy from mitmproxy.test import taddons -class TMsg: - pass +@pytest.mark.asyncio +async def test_master(): + class TMsg: + pass - -def test_master(): class tAddon: def log(self, _): ctx.master.should_exit.set() - with taddons.context() as ctx: - ctx.master.addons.add(tAddon()) + with taddons.context(tAddon()) as ctx: assert not ctx.master.should_exit.is_set() async def test(): msg = TMsg() msg.reply = controller.DummyReply() - await ctx.master.event_queue.put(("log", msg)) - - ctx.master.run(inject=test) - - -# class TestMaster: -# # def test_simple(self): -# # class tAddon: -# # def log(self, _): -# # ctx.master.should_exit.set() - -# # with taddons.context() as ctx: -# # ctx.master.addons.add(tAddon()) -# # assert not ctx.master.should_exit.is_set() -# # msg = TMsg() -# # msg.reply = controller.DummyReply() -# # ctx.master.event_queue.put(("log", msg)) -# # ctx.master.run() -# # assert ctx.master.should_exit.is_set() - -# # def test_server_simple(self): -# # m = master.Master(None) -# # m.server = proxy.DummyServer() -# # m.start() -# # m.shutdown() -# # m.start() -# # m.shutdown() -# pass - - -# class TestServerThread: -# def test_simple(self): -# m = Mock() -# t = master.ServerThread(m) -# t.run() -# assert m.serve_forever.called - - -# class TestChannel: -# def test_tell(self): -# q = queue.Queue() -# channel = controller.Channel(q, Event()) -# m = Mock(name="test_tell") -# channel.tell("test", m) -# assert q.get() == ("test", m) -# assert m.reply - -# def test_ask_simple(self): -# q = queue.Queue() - -# def reply(): -# m, obj = q.get() -# assert m == "test" -# obj.reply.send(42) -# obj.reply.take() -# obj.reply.commit() - -# Thread(target=reply).start() - -# channel = controller.Channel(q, Event()) -# assert channel.ask("test", Mock(name="test_ask_simple")) == 42 - -# def test_ask_shutdown(self): -# q = queue.Queue() -# done = Event() -# done.set() -# channel = controller.Channel(q, done) -# with pytest.raises(Kill): -# channel.ask("test", Mock(name="test_ask_shutdown")) + await ctx.master.channel.tell("log", msg) + + asyncio.ensure_future(test()) + assert not ctx.master.should_exit.is_set() class TestReply: -- cgit v1.2.3 From 54170ee6572e8ba38b94a4e51f3c8e832e5f9ac7 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 31 Mar 2018 19:27:26 +1300 Subject: asyncio: start a test loop for unit tests Also move signal handling out of master. These only work in the main thread, and properly belong in main.py. --- test/mitmproxy/test_fuzzing.py | 12 +----------- test/mitmproxy/tservers.py | 2 ++ 2 files changed, 3 insertions(+), 11 deletions(-) (limited to 'test') diff --git a/test/mitmproxy/test_fuzzing.py b/test/mitmproxy/test_fuzzing.py index 905ba1cd..57d0ca55 100644 --- a/test/mitmproxy/test_fuzzing.py +++ b/test/mitmproxy/test_fuzzing.py @@ -25,14 +25,4 @@ class TestFuzzy(tservers.HTTPProxyTest): p = self.pathoc() with p.connect(): resp = p.request(req % self.server.port) - assert resp.status_code == 400 - - # def test_invalid_upstream(self): - # req = r"get:'http://localhost:%s/p/200:i10,\x27+\x27'" - # p = self.pathoc() - # assert p.request(req % self.server.port).status_code == 502 - - # def test_upstream_disconnect(self): - # req = r'200:d0' - # p = self.pathod(req) - # assert p.status_code == 502 + assert resp.status_code == 400 \ No newline at end of file diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 7be31a28..9e6cef97 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -4,6 +4,7 @@ import tempfile import sys import time from unittest import mock +import asyncio import mitmproxy.platform from mitmproxy.addons import core @@ -105,6 +106,7 @@ class ProxyThread(threading.Thread): self.tmaster.shutdown() def run(self): + asyncio.set_event_loop(asyncio.new_event_loop()) self.tmaster = self.masterclass(self.options) self.tmaster.addons.add(core.Core()) self.name = "ProxyThread (%s:%s)" % ( -- cgit v1.2.3 From 3cc5d81a4a6cdb507b71fce0ce6c2a23fed0c208 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 1 Apr 2018 11:37:35 +1200 Subject: asyncio: fix channel interface and tests We now need to synthesize a tick event when changing addons in tests, because tick is asynchronously called on the event loop. --- test/mitmproxy/proxy/test_server.py | 34 ++++++++++++++++------------------ test/mitmproxy/tservers.py | 8 +++++++- 2 files changed, 23 insertions(+), 19 deletions(-) (limited to 'test') diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 4cfaa523..9e276294 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -276,10 +276,9 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin): s = script.Script( tutils.test_data.path("mitmproxy/data/addonscripts/stream_modify.py") ) - self.master.addons.add(s) + self.set_addons(s) d = self.pathod('200:b"foo"') assert d.content == b"bar" - self.master.addons.remove(s) def test_first_line_rewrite(self): """ @@ -583,12 +582,11 @@ class TestTransparent(tservers.TransparentProxyTest, CommonMixin, TcpMixin): s = script.Script( tutils.test_data.path("mitmproxy/data/addonscripts/tcp_stream_modify.py") ) - self.master.addons.add(s) + self.set_addons(s) self._tcpproxy_on() d = self.pathod('200:b"foo"') self._tcpproxy_off() assert d.content == b"bar" - self.master.addons.remove(s) class TestTransparentSSL(tservers.TransparentProxyTest, CommonMixin, TcpMixin): @@ -731,7 +729,7 @@ class TestRedirectRequest(tservers.HTTPProxyTest): This test verifies that the original destination is restored for the third request. """ - self.proxy.tmaster.addons.add(ARedirectRequest(self.server2.port)) + self.set_addons(ARedirectRequest(self.server2.port)) p = self.pathoc() with p.connect(): @@ -770,7 +768,7 @@ class AStreamRequest: class TestStreamRequest(tservers.HTTPProxyTest): def test_stream_simple(self): - self.proxy.tmaster.addons.add(AStreamRequest()) + self.set_addons(AStreamRequest()) p = self.pathoc() with p.connect(): # a request with 100k of data but without content-length @@ -779,7 +777,7 @@ class TestStreamRequest(tservers.HTTPProxyTest): assert len(r1.content) > 100000 def test_stream_multiple(self): - self.proxy.tmaster.addons.add(AStreamRequest()) + self.set_addons(AStreamRequest()) p = self.pathoc() with p.connect(): # simple request with streaming turned on @@ -791,7 +789,7 @@ class TestStreamRequest(tservers.HTTPProxyTest): assert r1.status_code == 201 def test_stream_chunked(self): - self.proxy.tmaster.addons.add(AStreamRequest()) + self.set_addons(AStreamRequest()) connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) connection.connect(("127.0.0.1", self.proxy.port)) fconn = connection.makefile("rb") @@ -820,7 +818,7 @@ class AFakeResponse: class TestFakeResponse(tservers.HTTPProxyTest): def test_fake(self): - self.proxy.tmaster.addons.add(AFakeResponse()) + self.set_addons(AFakeResponse()) f = self.pathod("200") assert "header-response" in f.headers @@ -836,7 +834,7 @@ class TestServerConnect(tservers.HTTPProxyTest): def test_unnecessary_serverconnect(self): """A replayed/fake response with no upstream_cert should not connect to an upstream server""" - self.proxy.tmaster.addons.add(AFakeResponse()) + self.set_addons(AFakeResponse()) assert self.pathod("200").status_code == 200 assert not self.proxy.tmaster.has_log("serverconnect") @@ -849,7 +847,7 @@ class AKillRequest: class TestKillRequest(tservers.HTTPProxyTest): def test_kill(self): - self.proxy.tmaster.addons.add(AKillRequest()) + self.set_addons(AKillRequest()) with pytest.raises(exceptions.HttpReadDisconnect): self.pathod("200") # Nothing should have hit the server @@ -863,7 +861,7 @@ class AKillResponse: class TestKillResponse(tservers.HTTPProxyTest): def test_kill(self): - self.proxy.tmaster.addons.add(AKillResponse()) + self.set_addons(AKillResponse()) with pytest.raises(exceptions.HttpReadDisconnect): self.pathod("200") # The server should have seen a request @@ -886,7 +884,7 @@ class AIncomplete: class TestIncompleteResponse(tservers.HTTPProxyTest): def test_incomplete(self): - self.proxy.tmaster.addons.add(AIncomplete()) + self.set_addons(AIncomplete()) assert self.pathod("200").status_code == 502 @@ -969,7 +967,7 @@ class TestUpstreamProxySSL( def test_change_upstream_proxy_connect(self): # skip chain[0]. - self.proxy.tmaster.addons.add( + self.set_addons( UpstreamProxyChanger( ("127.0.0.1", self.chain[1].port) ) @@ -988,8 +986,8 @@ class TestUpstreamProxySSL( Client <- HTTPS -> Proxy <- HTTP -> Proxy <- HTTPS -> Server """ - self.proxy.tmaster.addons.add(RewriteToHttp()) - self.chain[1].tmaster.addons.add(RewriteToHttps()) + self.set_addons(RewriteToHttp()) + self.set_addons(RewriteToHttps()) p = self.pathoc() with p.connect(): resp = p.request("get:'/p/418'") @@ -1063,8 +1061,8 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest): http1obj.server_conn.wfile.write(headers) http1obj.server_conn.wfile.flush() - self.chain[0].tmaster.addons.add(RequestKiller([1, 2])) - self.chain[1].tmaster.addons.add(RequestKiller([1])) + self.chain[0].set_addons(RequestKiller([1, 2])) + self.chain[1].set_addons(RequestKiller([1])) p = self.pathoc() with p.connect(): diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 9e6cef97..f3270e5c 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -92,6 +92,7 @@ class ProxyThread(threading.Thread): self.masterclass = masterclass self.options = options self.tmaster = None + self.event_loop = None controller.should_exit = False @property @@ -106,7 +107,8 @@ class ProxyThread(threading.Thread): self.tmaster.shutdown() def run(self): - asyncio.set_event_loop(asyncio.new_event_loop()) + self.event_loop = asyncio.new_event_loop() + asyncio.set_event_loop(self.event_loop) self.tmaster = self.masterclass(self.options) self.tmaster.addons.add(core.Core()) self.name = "ProxyThread (%s:%s)" % ( @@ -177,6 +179,10 @@ class ProxyTestBase: ssl_insecure=True, ) + def set_addons(self, *addons): + self.proxy.tmaster.reset(addons) + self.proxy.tmaster.addons.trigger("tick") + def addons(self): """ Can be over-ridden to add a standard set of addons to tests. -- cgit v1.2.3 From 28a8ddc0e827149c76a738e6363713ed2ebc0eac Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 2 Apr 2018 09:46:02 +1200 Subject: asyncio: fix a few remaining issues in proxy/test_server.py --- test/mitmproxy/proxy/test_server.py | 2 +- test/mitmproxy/tservers.py | 21 +++++++++++++-------- 2 files changed, 14 insertions(+), 9 deletions(-) (limited to 'test') diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 9e276294..aed4a774 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -987,7 +987,7 @@ class TestUpstreamProxySSL( Client <- HTTPS -> Proxy <- HTTP -> Proxy <- HTTPS -> Server """ self.set_addons(RewriteToHttp()) - self.set_addons(RewriteToHttps()) + self.chain[1].set_addons(RewriteToHttps()) p = self.pathoc() with p.connect(): resp = p.request("get:'/p/418'") diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index f3270e5c..18e4f27d 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -117,6 +117,11 @@ class ProxyThread(threading.Thread): ) self.tmaster.run() + def set_addons(self, *addons): + self.tmaster.reset(addons) + self.tmaster.addons.trigger("tick") + + class ProxyTestBase: # Test Configuration @@ -180,8 +185,7 @@ class ProxyTestBase: ) def set_addons(self, *addons): - self.proxy.tmaster.reset(addons) - self.proxy.tmaster.addons.trigger("tick") + self.proxy.set_addons(*addons) def addons(self): """ @@ -337,8 +341,7 @@ class SocksModeTest(HTTPProxyTest): return opts -class ChainProxyTest(ProxyTestBase): - +class HTTPUpstreamProxyTest(HTTPProxyTest): """ Chain three instances of mitmproxy in a row to test upstream mode. Proxy order is cls.proxy -> cls.chain[0] -> cls.chain[1] @@ -357,6 +360,12 @@ class ChainProxyTest(ProxyTestBase): proxy = ProxyThread(cls.masterclass, opts) proxy.start() cls.chain.insert(0, proxy) + while 1: + if( + proxy.event_loop and + proxy.event_loop.is_running() + ): + break super().setup_class() @@ -380,7 +389,3 @@ class ChainProxyTest(ProxyTestBase): mode="upstream:" + s, ) return opts - - -class HTTPUpstreamProxyTest(ChainProxyTest, HTTPProxyTest): - pass -- cgit v1.2.3 From 2b040ff09374ad50b0878c18296beb4d9147d075 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 2 Apr 2018 11:03:05 +1200 Subject: asyncio: make http2 tests pass --- test/mitmproxy/proxy/protocol/test_http2.py | 7 ++++++- test/mitmproxy/tservers.py | 1 - 2 files changed, 6 insertions(+), 2 deletions(-) (limited to 'test') diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index 8b929995..1cc36402 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -6,11 +6,11 @@ import tempfile import traceback import pytest import h2 +import time from mitmproxy import options import mitmproxy.net -from mitmproxy.addons import core from ...net import tservers as net_tservers from mitmproxy import exceptions from mitmproxy.net.http import http1, http2 @@ -92,6 +92,10 @@ class _Http2TestBase: cls.options = cls.get_options() cls.proxy = tservers.ProxyThread(tservers.TestMaster, cls.options) cls.proxy.start() + while True: + if cls.proxy.tmaster: + break + time.sleep(0.01) @classmethod def teardown_class(cls): @@ -118,6 +122,7 @@ class _Http2TestBase: def teardown(self): if self.client: self.client.close() + self.server.server.wait_for_silence() def setup_connection(self): self.client = mitmproxy.net.tcp.TCPClient(("127.0.0.1", self.proxy.port)) diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 18e4f27d..d72ca138 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -122,7 +122,6 @@ class ProxyThread(threading.Thread): self.tmaster.addons.trigger("tick") - class ProxyTestBase: # Test Configuration ssl = None -- cgit v1.2.3 From a3da43d3e5d5b2ca243971e586aeee4969b6d053 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 2 Apr 2018 14:51:14 +1200 Subject: asyncio: test cleanup Also silence asyncio logs. We sometimes end up with messages on the queue that need to be ignored when the proxy shuts down, and asyncio complains loudly about this. --- test/mitmproxy/data/addonscripts/shutdown.py | 2 +- test/mitmproxy/proxy/protocol/test_websocket.py | 14 +++++++++----- test/mitmproxy/tools/test_main.py | 10 ++++++++-- 3 files changed, 18 insertions(+), 8 deletions(-) (limited to 'test') diff --git a/test/mitmproxy/data/addonscripts/shutdown.py b/test/mitmproxy/data/addonscripts/shutdown.py index 51a99b5c..3da4d03e 100644 --- a/test/mitmproxy/data/addonscripts/shutdown.py +++ b/test/mitmproxy/data/addonscripts/shutdown.py @@ -1,5 +1,5 @@ from mitmproxy import ctx -def running(): +def tick(): ctx.master.shutdown() diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py index 2a343450..014490b7 100644 --- a/test/mitmproxy/proxy/protocol/test_websocket.py +++ b/test/mitmproxy/proxy/protocol/test_websocket.py @@ -3,10 +3,10 @@ import os import struct import tempfile import traceback +import time from mitmproxy import options from mitmproxy import exceptions -from mitmproxy.addons import core from mitmproxy.http import HTTPFlow from mitmproxy.websocket import WebSocketFlow @@ -54,6 +54,10 @@ class _WebSocketTestBase: cls.options = cls.get_options() cls.proxy = tservers.ProxyThread(tservers.TestMaster, cls.options) cls.proxy.start() + while True: + if cls.proxy.tmaster: + break + time.sleep(0.01) @classmethod def teardown_class(cls): @@ -161,7 +165,7 @@ class TestSimple(_WebSocketTest): def websocket_start(self, f): f.stream = streaming - self.master.addons.add(Stream()) + self.proxy.set_addons(Stream()) self.setup_connection() frame = websockets.Frame.from_file(self.client.rfile) @@ -202,7 +206,7 @@ class TestSimple(_WebSocketTest): def websocket_message(self, f): f.messages[-1].content = "foo" - self.master.addons.add(Addon()) + self.proxy.set_addons(Addon()) self.setup_connection() frame = websockets.Frame.from_file(self.client.rfile) @@ -233,7 +237,7 @@ class TestKillFlow(_WebSocketTest): def websocket_message(self, f): f.kill() - self.master.addons.add(KillFlow()) + self.proxy.set_addons(KillFlow()) self.setup_connection() with pytest.raises(exceptions.TcpDisconnect): @@ -403,7 +407,7 @@ class TestStreaming(_WebSocketTest): def websocket_start(self, f): f.stream = streaming - self.master.addons.add(Stream()) + self.proxy.set_addons(Stream()) self.setup_connection() frame = None diff --git a/test/mitmproxy/tools/test_main.py b/test/mitmproxy/tools/test_main.py index 88e2fe86..a514df74 100644 --- a/test/mitmproxy/tools/test_main.py +++ b/test/mitmproxy/tools/test_main.py @@ -1,19 +1,25 @@ +import pytest + from mitmproxy.test import tutils from mitmproxy.tools import main shutdown_script = tutils.test_data.path("mitmproxy/data/addonscripts/shutdown.py") -def test_mitmweb(): +@pytest.mark.asyncio +async def test_mitmweb(): main.mitmweb([ "--no-web-open-browser", "-q", + "-p", "0", "-s", shutdown_script ]) -def test_mitmdump(): +@pytest.mark.asyncio +async def test_mitmdump(): main.mitmdump([ "-q", + "-p", "0", "-s", shutdown_script ]) -- cgit v1.2.3 From 14f461c5d59e95e84658ea43567b3ca9f7d0a108 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 2 Apr 2018 15:47:23 +1200 Subject: asyncio: cleanup and lint Also fix a racy websocket test. --- test/mitmproxy/proxy/protocol/test_websocket.py | 7 ++++++- test/mitmproxy/test_flow.py | 2 +- 2 files changed, 7 insertions(+), 2 deletions(-) (limited to 'test') diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py index 014490b7..f64f500c 100644 --- a/test/mitmproxy/proxy/protocol/test_websocket.py +++ b/test/mitmproxy/proxy/protocol/test_websocket.py @@ -331,7 +331,12 @@ class TestPong(_WebSocketTest): assert frame.header.opcode == websockets.OPCODE.PONG assert frame.payload == b'foobar' - assert self.master.has_log("Pong Received from server", "info") + for i in range(20): + if self.master.has_log("Pong Received from server", "info"): + break + time.sleep(0.01) + else: + raise AssertionError("No pong seen") class TestClose(_WebSocketTest): diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 9f1fb213..4042de5b 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -7,7 +7,7 @@ import mitmproxy.io from mitmproxy import flowfilter from mitmproxy import options from mitmproxy.io import tnetstring -from mitmproxy.exceptions import FlowReadException, ReplayException, ControlException +from mitmproxy.exceptions import FlowReadException, ReplayException from mitmproxy import flow from mitmproxy import http from mitmproxy.net import http as net_http -- cgit v1.2.3 From 7e73e5fa51fb59ee1937bd8f634834bb420ff903 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 3 Apr 2018 09:46:11 +1200 Subject: asyncio: factor out test server startup wait --- test/mitmproxy/proxy/protocol/test_http2.py | 4 ---- test/mitmproxy/proxy/protocol/test_websocket.py | 4 ---- test/mitmproxy/tservers.py | 11 +++++++---- 3 files changed, 7 insertions(+), 12 deletions(-) (limited to 'test') diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index 1cc36402..58114904 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -92,10 +92,6 @@ class _Http2TestBase: cls.options = cls.get_options() cls.proxy = tservers.ProxyThread(tservers.TestMaster, cls.options) cls.proxy.start() - while True: - if cls.proxy.tmaster: - break - time.sleep(0.01) @classmethod def teardown_class(cls): diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py index f64f500c..e5ed8e9d 100644 --- a/test/mitmproxy/proxy/protocol/test_websocket.py +++ b/test/mitmproxy/proxy/protocol/test_websocket.py @@ -54,10 +54,6 @@ class _WebSocketTestBase: cls.options = cls.get_options() cls.proxy = tservers.ProxyThread(tservers.TestMaster, cls.options) cls.proxy.start() - while True: - if cls.proxy.tmaster: - break - time.sleep(0.01) @classmethod def teardown_class(cls): diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index d72ca138..7feb90b7 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -121,6 +121,13 @@ class ProxyThread(threading.Thread): self.tmaster.reset(addons) self.tmaster.addons.trigger("tick") + def start(self): + super().start() + while True: + if self.tmaster: + break + time.sleep(0.01) + class ProxyTestBase: # Test Configuration @@ -142,10 +149,6 @@ class ProxyTestBase: cls.options = cls.get_options() cls.proxy = ProxyThread(cls.masterclass, cls.options) cls.proxy.start() - while True: - if cls.proxy.tmaster: - break - time.sleep(0.01) @classmethod def teardown_class(cls): -- cgit v1.2.3 From 659fceb697054d28e427c3a1169e07c210049159 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 3 Apr 2018 09:54:29 +1200 Subject: minor fixes --- test/mitmproxy/proxy/protocol/test_http2.py | 1 - test/mitmproxy/tservers.py | 13 ++++--------- 2 files changed, 4 insertions(+), 10 deletions(-) (limited to 'test') diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index 58114904..13f28728 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -6,7 +6,6 @@ import tempfile import traceback import pytest import h2 -import time from mitmproxy import options diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 7feb90b7..2d102a5d 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -14,6 +14,7 @@ from mitmproxy import controller from mitmproxy import options from mitmproxy import exceptions from mitmproxy import io +from mitmproxy.utils import human import pathod.test import pathod.pathoc @@ -111,10 +112,7 @@ class ProxyThread(threading.Thread): asyncio.set_event_loop(self.event_loop) self.tmaster = self.masterclass(self.options) self.tmaster.addons.add(core.Core()) - self.name = "ProxyThread (%s:%s)" % ( - self.tmaster.server.address[0], - self.tmaster.server.address[1], - ) + self.name = "ProxyThread (%s)" % human.format_address(self.tmaster.server.address) self.tmaster.run() def set_addons(self, *addons): @@ -362,11 +360,8 @@ class HTTPUpstreamProxyTest(HTTPProxyTest): proxy = ProxyThread(cls.masterclass, opts) proxy.start() cls.chain.insert(0, proxy) - while 1: - if( - proxy.event_loop and - proxy.event_loop.is_running() - ): + while True: + if proxy.event_loop and proxy.event_loop.is_running(): break super().setup_class() -- cgit v1.2.3