From 4501c8a0a158f15386d9dfe4884a9ffacb8ffd0b Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 24 Jan 2016 23:53:32 +0100 Subject: add http2 full-stack test --- test/test_protocol_http2.py | 117 ++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 117 insertions(+) create mode 100644 test/test_protocol_http2.py (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py new file mode 100644 index 00000000..17b8506d --- /dev/null +++ b/test/test_protocol_http2.py @@ -0,0 +1,117 @@ +from __future__ import (absolute_import, print_function, division) + +import inspect +import socket +from io import BytesIO + +import logging +logging.getLogger("hyper.packages.hpack.hpack").setLevel(logging.WARNING) + +import netlib +from netlib import tservers as netlib_tservers + +import h2 + +from libmproxy import utils +from . import tservers + +class SimpleHttp2Server(netlib_tservers.ServerTestBase): + ssl = dict( + alpn_select=b'h2', + ) + + class handler(netlib.tcp.BaseHandler): + def handle(self): + h2_conn = h2.connection.H2Connection(client_side=False) + + preamble = self.rfile.read(24) + h2_conn.initiate_connection() + h2_conn.receive_data(preamble) + self.wfile.write(h2_conn.data_to_send()) + self.wfile.flush() + + while True: + events = h2_conn.receive_data(utils.http2_read_frame(self.rfile)) + self.wfile.write(h2_conn.data_to_send()) + self.wfile.flush() + + for event in events: + if isinstance(event, h2.events.RequestReceived): + h2_conn.send_headers(1, [ + (':status', '200'), + ('foo', 'bar'), + ]) + h2_conn.send_data(1, b'foobar') + h2_conn.end_stream(1) + self.wfile.write(h2_conn.data_to_send()) + self.wfile.flush() + elif isinstance(event, h2.events.ConnectionTerminated): + return + + +class TestHttp2(tservers.ProxTestBase): + def _setup_connection(self): + self.config.http2 = True + + client = netlib.tcp.TCPClient(("127.0.0.1", self.proxy.port)) + client.connect() + + # send CONNECT request + client.wfile.write( + b"CONNECT localhost:%d HTTP/1.1\r\n" + b"Host: localhost:%d\r\n" + b"\r\n" % (self.server.port, self.server.port) + ) + client.wfile.flush() + + # read CONNECT response + while client.rfile.readline() != "\r\n": + pass + + client.convert_to_ssl(alpn_protos=[b'h2']) + + h2_conn = h2.connection.H2Connection(client_side=True) + h2_conn.initiate_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + return client, h2_conn + + def _send_request(self, wfile, h2_conn, stream_id=1, headers=[], end_stream=True): + h2_conn.send_headers( + stream_id=stream_id, + headers=headers, + end_stream=end_stream, + ) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + + def test_simple(self): + self.server = SimpleHttp2Server() + self.server.setup_class() + + client, h2_conn = self._setup_connection() + + self._send_request(client.wfile, h2_conn, headers=[ + (':authority', "127.0.0.1:%s" % self.server.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ]) + + done = False + while not done: + events = h2_conn.receive_data(utils.http2_read_frame(client.rfile)) + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + done = True + + self.server.teardown_class() + + assert len(self.master.state.flows) == 1 + assert self.master.state.flows[0].response.status_code == 200 + assert self.master.state.flows[0].response.headers['foo'] == 'bar' + assert self.master.state.flows[0].response.body == b'foobar' -- cgit v1.2.3 From bfc7d3967c0978b22faeaedc653a695e65156b34 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 25 Jan 2016 19:09:14 +0100 Subject: exclude tests if no alpn support present --- test/test_protocol_http2.py | 8 ++++++++ 1 file changed, 8 insertions(+) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index 17b8506d..d3725e81 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -2,6 +2,8 @@ from __future__ import (absolute_import, print_function, division) import inspect import socket +import OpenSSL +import pytest from io import BytesIO import logging @@ -15,6 +17,11 @@ import h2 from libmproxy import utils from . import tservers +requires_alpn = pytest.mark.skipif( + not OpenSSL._util.lib.Cryptography_HAS_ALPN, + reason="requires OpenSSL with ALPN support") + + class SimpleHttp2Server(netlib_tservers.ServerTestBase): ssl = dict( alpn_select=b'h2', @@ -49,6 +56,7 @@ class SimpleHttp2Server(netlib_tservers.ServerTestBase): return +@requires_alpn class TestHttp2(tservers.ProxTestBase): def _setup_connection(self): self.config.http2 = True -- cgit v1.2.3 From 47cf27c01146e87b8bbf315f70e9752a38ba8edd Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 25 Jan 2016 19:20:44 +0100 Subject: silence 3rd party module loggers --- test/test_protocol_http2.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index d3725e81..a7e8978a 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -8,6 +8,11 @@ from io import BytesIO import logging logging.getLogger("hyper.packages.hpack.hpack").setLevel(logging.WARNING) +logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(logging.WARNING) +logging.getLogger("passlib.utils.compat").setLevel(logging.WARNING) +logging.getLogger("passlib.registry").setLevel(logging.WARNING) +logging.getLogger("PIL.Image").setLevel(logging.WARNING) +logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING) import netlib from netlib import tservers as netlib_tservers -- cgit v1.2.3 From 735c79a2edb3b31a35ed3e484744807bb626ab77 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 25 Jan 2016 19:41:22 +0100 Subject: increase coverage --- test/test_protocol_http2.py | 4 ++++ 1 file changed, 4 insertions(+) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index a7e8978a..e72113c4 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -122,6 +122,10 @@ class TestHttp2(tservers.ProxTestBase): if isinstance(event, h2.events.StreamEnded): done = True + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + self.server.teardown_class() assert len(self.master.state.flows) == 1 -- cgit v1.2.3 From 41f4197a0dd73a2b00ea8485608ba9b05a605dd4 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 25 Jan 2016 21:14:58 +0100 Subject: test PushPromise support --- test/test_protocol_http2.py | 91 +++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 88 insertions(+), 3 deletions(-) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index e72113c4..4fa2c701 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -28,9 +28,7 @@ requires_alpn = pytest.mark.skipif( class SimpleHttp2Server(netlib_tservers.ServerTestBase): - ssl = dict( - alpn_select=b'h2', - ) + ssl = dict(alpn_select=b'h2') class handler(netlib.tcp.BaseHandler): def handle(self): @@ -61,6 +59,59 @@ class SimpleHttp2Server(netlib_tservers.ServerTestBase): return +class PushHttp2Server(netlib_tservers.ServerTestBase): + ssl = dict(alpn_select=b'h2') + + class handler(netlib.tcp.BaseHandler): + def handle(self): + h2_conn = h2.connection.H2Connection(client_side=False) + + preamble = self.rfile.read(24) + h2_conn.initiate_connection() + h2_conn.receive_data(preamble) + self.wfile.write(h2_conn.data_to_send()) + self.wfile.flush() + + while True: + events = h2_conn.receive_data(utils.http2_read_frame(self.rfile)) + self.wfile.write(h2_conn.data_to_send()) + self.wfile.flush() + + for event in events: + if isinstance(event, h2.events.RequestReceived): + h2_conn.send_headers(1, [(':status', '200')]) + h2_conn.push_stream(1, 2, [ + (':authority', "127.0.0.1:%s" % self.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/pushed_stream_foo'), + ('foo', 'bar') + ]) + h2_conn.push_stream(1, 4, [ + (':authority', "127.0.0.1:%s" % self.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/pushed_stream_bar'), + ('foo', 'bar') + ]) + self.wfile.write(h2_conn.data_to_send()) + self.wfile.flush() + + h2_conn.send_headers(2, [(':status', '202')]) + h2_conn.send_headers(4, [(':status', '204')]) + h2_conn.send_data(1, b'regular_stream') + h2_conn.send_data(2, b'pushed_stream_foo') + h2_conn.send_data(4, b'pushed_stream_bar') + h2_conn.end_stream(1) + h2_conn.end_stream(2) + h2_conn.end_stream(4) + self.wfile.write(h2_conn.data_to_send()) + self.wfile.flush() + print("HERE") + elif isinstance(event, h2.events.ConnectionTerminated): + return + + @requires_alpn class TestHttp2(tservers.ProxTestBase): def _setup_connection(self): @@ -132,3 +183,37 @@ class TestHttp2(tservers.ProxTestBase): assert self.master.state.flows[0].response.status_code == 200 assert self.master.state.flows[0].response.headers['foo'] == 'bar' assert self.master.state.flows[0].response.body == b'foobar' + + def test_pushed_streams(self): + self.server = PushHttp2Server() + self.server.setup_class() + + client, h2_conn = self._setup_connection() + + self._send_request(client.wfile, h2_conn, headers=[ + (':authority', "127.0.0.1:%s" % self.server.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ('foo', 'bar') + ]) + + ended_streams = 0 + while ended_streams != 3: + try: + events = h2_conn.receive_data(utils.http2_read_frame(client.rfile)) + except: + break + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + ended_streams += 1 + + self.server.teardown_class() + + assert len(self.master.state.flows) == 3 + assert self.master.state.flows[0].response.body == b'regular_stream' + assert self.master.state.flows[1].response.body == b'pushed_stream_foo' + assert self.master.state.flows[2].response.body == b'pushed_stream_bar' -- cgit v1.2.3 From 187691e65bf4a18de3567d6d801d78aa721b9fa5 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 25 Jan 2016 23:04:15 +0100 Subject: remove print --- test/test_protocol_http2.py | 1 - 1 file changed, 1 deletion(-) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index 4fa2c701..17687b45 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -107,7 +107,6 @@ class PushHttp2Server(netlib_tservers.ServerTestBase): h2_conn.end_stream(4) self.wfile.write(h2_conn.data_to_send()) self.wfile.flush() - print("HERE") elif isinstance(event, h2.events.ConnectionTerminated): return -- cgit v1.2.3 From 276817e40e99dbb2ddc7638839bd74e944fd704e Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 26 Jan 2016 13:15:20 +0100 Subject: refactor http2 tests --- test/test_protocol_http2.py | 234 ++++++++++++++++++++++++++++---------------- 1 file changed, 149 insertions(+), 85 deletions(-) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index 17687b45..b42b86cb 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -4,8 +4,16 @@ import inspect import socket import OpenSSL import pytest +import traceback +import os +import tempfile + from io import BytesIO +from libmproxy.proxy.config import ProxyConfig +from libmproxy.proxy.server import ProxyServer +from libmproxy.cmdline import APP_HOST, APP_PORT + import logging logging.getLogger("hyper.packages.hpack.hpack").setLevel(logging.WARNING) logging.getLogger("requests.packages.urllib3.connectionpool").setLevel(logging.WARNING) @@ -18,6 +26,7 @@ import netlib from netlib import tservers as netlib_tservers import h2 +from hyperframe.frame import Frame from libmproxy import utils from . import tservers @@ -26,8 +35,7 @@ requires_alpn = pytest.mark.skipif( not OpenSSL._util.lib.Cryptography_HAS_ALPN, reason="requires OpenSSL with ALPN support") - -class SimpleHttp2Server(netlib_tservers.ServerTestBase): +class _Http2ServerBase(netlib_tservers.ServerTestBase): ssl = dict(alpn_select=b'h2') class handler(netlib.tcp.BaseHandler): @@ -41,78 +49,56 @@ class SimpleHttp2Server(netlib_tservers.ServerTestBase): self.wfile.flush() while True: - events = h2_conn.receive_data(utils.http2_read_frame(self.rfile)) + raw_frame = utils.http2_read_frame(self.rfile) + events = h2_conn.receive_data(raw_frame) self.wfile.write(h2_conn.data_to_send()) self.wfile.flush() for event in events: - if isinstance(event, h2.events.RequestReceived): - h2_conn.send_headers(1, [ - (':status', '200'), - ('foo', 'bar'), - ]) - h2_conn.send_data(1, b'foobar') - h2_conn.end_stream(1) - self.wfile.write(h2_conn.data_to_send()) - self.wfile.flush() - elif isinstance(event, h2.events.ConnectionTerminated): - return - - -class PushHttp2Server(netlib_tservers.ServerTestBase): - ssl = dict(alpn_select=b'h2') - - class handler(netlib.tcp.BaseHandler): - def handle(self): - h2_conn = h2.connection.H2Connection(client_side=False) - - preamble = self.rfile.read(24) - h2_conn.initiate_connection() - h2_conn.receive_data(preamble) - self.wfile.write(h2_conn.data_to_send()) - self.wfile.flush() - - while True: - events = h2_conn.receive_data(utils.http2_read_frame(self.rfile)) - self.wfile.write(h2_conn.data_to_send()) - self.wfile.flush() - - for event in events: - if isinstance(event, h2.events.RequestReceived): - h2_conn.send_headers(1, [(':status', '200')]) - h2_conn.push_stream(1, 2, [ - (':authority', "127.0.0.1:%s" % self.address.port), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/pushed_stream_foo'), - ('foo', 'bar') - ]) - h2_conn.push_stream(1, 4, [ - (':authority', "127.0.0.1:%s" % self.address.port), - (':method', 'GET'), - (':scheme', 'https'), - (':path', '/pushed_stream_bar'), - ('foo', 'bar') - ]) - self.wfile.write(h2_conn.data_to_send()) - self.wfile.flush() - - h2_conn.send_headers(2, [(':status', '202')]) - h2_conn.send_headers(4, [(':status', '204')]) - h2_conn.send_data(1, b'regular_stream') - h2_conn.send_data(2, b'pushed_stream_foo') - h2_conn.send_data(4, b'pushed_stream_bar') - h2_conn.end_stream(1) - h2_conn.end_stream(2) - h2_conn.end_stream(4) - self.wfile.write(h2_conn.data_to_send()) - self.wfile.flush() - elif isinstance(event, h2.events.ConnectionTerminated): - return + try: + if not self.server.handle_server_event(event, h2_conn, self.rfile, self.wfile): + break + except Exception as e: + print(repr(e)) + print(traceback.format_exc()) + break + + def handle_server_event(self, h2_conn, rfile, wfile): + raise NotImplementedError() + + +class _Http2TestBase(object): + @classmethod + def setup_class(self): + self.config = ProxyConfig(**self.get_proxy_config()) + + tmaster = tservers.TestMaster(self.config) + tmaster.start_app(APP_HOST, APP_PORT) + self.proxy = tservers.ProxyThread(tmaster) + self.proxy.start() + + @classmethod + def teardown_class(cls): + cls.proxy.shutdown() + + @property + def master(self): + return self.proxy.tmaster + + @classmethod + def get_proxy_config(cls): + cls.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy") + return dict( + no_upstream_cert = False, + cadir = cls.cadir, + authenticator = None, + ) + def setup(self): + self.master.clear_log() + self.master.state.clear() + self.server.server.handle_server_event = self.handle_server_event -@requires_alpn -class TestHttp2(tservers.ProxTestBase): def _setup_connection(self): self.config.http2 = True @@ -123,7 +109,7 @@ class TestHttp2(tservers.ProxTestBase): client.wfile.write( b"CONNECT localhost:%d HTTP/1.1\r\n" b"Host: localhost:%d\r\n" - b"\r\n" % (self.server.port, self.server.port) + b"\r\n" % (self.server.server.address.port, self.server.server.address.port) ) client.wfile.flush() @@ -149,14 +135,40 @@ class TestHttp2(tservers.ProxTestBase): wfile.write(h2_conn.data_to_send()) wfile.flush() - def test_simple(self): - self.server = SimpleHttp2Server() - self.server.setup_class() +@requires_alpn +class TestSimple(_Http2TestBase, _Http2ServerBase): + @classmethod + def setup_class(self): + _Http2TestBase.setup_class() + _Http2ServerBase.setup_class() + + @classmethod + def teardown_class(self): + _Http2TestBase.teardown_class() + _Http2ServerBase.teardown_class() + + @classmethod + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.RequestReceived): + h2_conn.send_headers(1, [ + (':status', '200'), + ('foo', 'bar'), + ]) + h2_conn.send_data(1, b'foobar') + h2_conn.end_stream(1) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + + return True + + def test_simple(self): client, h2_conn = self._setup_connection() self._send_request(client.wfile, h2_conn, headers=[ - (':authority', "127.0.0.1:%s" % self.server.port), + (':authority', "127.0.0.1:%s" % self.server.server.address.port), (':method', 'GET'), (':scheme', 'https'), (':path', '/'), @@ -176,21 +188,69 @@ class TestHttp2(tservers.ProxTestBase): client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() - self.server.teardown_class() - assert len(self.master.state.flows) == 1 assert self.master.state.flows[0].response.status_code == 200 assert self.master.state.flows[0].response.headers['foo'] == 'bar' assert self.master.state.flows[0].response.body == b'foobar' - def test_pushed_streams(self): - self.server = PushHttp2Server() - self.server.setup_class() +@requires_alpn +class TestPushPromise(_Http2TestBase, _Http2ServerBase): + @classmethod + def setup_class(self): + _Http2TestBase.setup_class() + _Http2ServerBase.setup_class() + + @classmethod + def teardown_class(self): + _Http2TestBase.teardown_class() + _Http2ServerBase.teardown_class() + + @classmethod + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + elif isinstance(event, h2.events.RequestReceived): + if event.stream_id != 1: + # ignore requests initiated by push promises + return True + + h2_conn.send_headers(1, [(':status', '200')]) + h2_conn.push_stream(1, 2, [ + (':authority', "127.0.0.1:%s" % self.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/pushed_stream_foo'), + ('foo', 'bar') + ]) + h2_conn.push_stream(1, 4, [ + (':authority', "127.0.0.1:%s" % self.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/pushed_stream_bar'), + ('foo', 'bar') + ]) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + + h2_conn.send_headers(2, [(':status', '202')]) + h2_conn.send_headers(4, [(':status', '204')]) + h2_conn.send_data(1, b'regular_stream') + h2_conn.send_data(2, b'pushed_stream_foo') + h2_conn.send_data(4, b'pushed_stream_bar') + h2_conn.end_stream(1) + h2_conn.end_stream(2) + h2_conn.end_stream(4) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + + return True + + def test_push_promise(self): client, h2_conn = self._setup_connection() - self._send_request(client.wfile, h2_conn, headers=[ - (':authority', "127.0.0.1:%s" % self.server.port), + self._send_request(client.wfile, h2_conn, stream_id=1, headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), (':method', 'GET'), (':scheme', 'https'), (':path', '/'), @@ -198,6 +258,7 @@ class TestHttp2(tservers.ProxTestBase): ]) ended_streams = 0 + pushed_streams = 0 while ended_streams != 3: try: events = h2_conn.receive_data(utils.http2_read_frame(client.rfile)) @@ -209,10 +270,13 @@ class TestHttp2(tservers.ProxTestBase): for event in events: if isinstance(event, h2.events.StreamEnded): ended_streams += 1 + elif isinstance(event, h2.events.PushedStreamReceived): + pushed_streams += 1 - self.server.teardown_class() + assert pushed_streams == 2 - assert len(self.master.state.flows) == 3 - assert self.master.state.flows[0].response.body == b'regular_stream' - assert self.master.state.flows[1].response.body == b'pushed_stream_foo' - assert self.master.state.flows[2].response.body == b'pushed_stream_bar' + bodies = [flow.response.body for flow in self.master.state.flows] + assert len(bodies) == 3 + assert b'regular_stream' in bodies + assert b'pushed_stream_foo' in bodies + assert b'pushed_stream_bar' in bodies -- cgit v1.2.3 From bd1d9e28e4aa942b6fdaac0d327d9147ee911ab1 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 26 Jan 2016 19:38:29 +0100 Subject: test stream resets in push promise --- test/test_protocol_http2.py | 37 +++++++++++++++++++++++++++++++++++++ 1 file changed, 37 insertions(+) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index b42b86cb..b1c88b27 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -235,6 +235,9 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): h2_conn.send_headers(2, [(':status', '202')]) h2_conn.send_headers(4, [(':status', '204')]) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + h2_conn.send_data(1, b'regular_stream') h2_conn.send_data(2, b'pushed_stream_foo') h2_conn.send_data(4, b'pushed_stream_bar') @@ -280,3 +283,37 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): assert b'regular_stream' in bodies assert b'pushed_stream_foo' in bodies assert b'pushed_stream_bar' in bodies + + def test_push_promise_reset(self): + client, h2_conn = self._setup_connection() + + self._send_request(client.wfile, h2_conn, stream_id=1, headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ('foo', 'bar') + ]) + + done = False + while not done: + try: + events = h2_conn.receive_data(utils.http2_read_frame(client.rfile)) + except: + break + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded) and event.stream_id == 1: + done = True + elif isinstance(event, h2.events.PushedStreamReceived): + h2_conn.reset_stream(event.pushed_stream_id) + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + bodies = [flow.response.body for flow in self.master.state.flows] + assert len(bodies) == 3 + assert b'regular_stream' in bodies + assert b'pushed_stream_foo' in bodies + assert b'pushed_stream_bar' in bodies -- cgit v1.2.3 From 44f83b594701f9756418cf8208c30a9ba5ac4aad Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 26 Jan 2016 20:44:53 +0100 Subject: add more tests, improve coverage --- test/test_protocol_http2.py | 79 +++++++++++++++++++++++++++++++++++++++++---- 1 file changed, 72 insertions(+), 7 deletions(-) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index b1c88b27..3554fa4d 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -7,7 +7,7 @@ import pytest import traceback import os import tempfile - +import time from io import BytesIO from libmproxy.proxy.config import ProxyConfig @@ -126,12 +126,15 @@ class _Http2TestBase(object): return client, h2_conn - def _send_request(self, wfile, h2_conn, stream_id=1, headers=[], end_stream=True): + def _send_request(self, wfile, h2_conn, stream_id=1, headers=[], body=b''): h2_conn.send_headers( stream_id=stream_id, headers=headers, - end_stream=end_stream, + end_stream=(len(body) == 0), ) + if body: + h2_conn.send_data(stream_id, body) + h2_conn.end_stream(stream_id) wfile.write(h2_conn.data_to_send()) wfile.flush() @@ -172,7 +175,7 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): (':method', 'GET'), (':scheme', 'https'), (':path', '/'), - ]) + ], body='my request body echoed back to me') done = False while not done: @@ -194,6 +197,69 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): assert self.master.state.flows[0].response.body == b'foobar' +@requires_alpn +class TestWithBodies(_Http2TestBase, _Http2ServerBase): + tmp_data_buffer_foobar = b'' + + @classmethod + def setup_class(self): + _Http2TestBase.setup_class() + _Http2ServerBase.setup_class() + + @classmethod + def teardown_class(self): + _Http2TestBase.teardown_class() + _Http2ServerBase.teardown_class() + + @classmethod + def handle_server_event(self, event, h2_conn, rfile, wfile): + if isinstance(event, h2.events.ConnectionTerminated): + return False + if isinstance(event, h2.events.DataReceived): + self.tmp_data_buffer_foobar += event.data + elif isinstance(event, h2.events.StreamEnded): + h2_conn.send_headers(1, [ + (':status', '200'), + ]) + h2_conn.send_data(1, self.tmp_data_buffer_foobar) + h2_conn.end_stream(1) + wfile.write(h2_conn.data_to_send()) + wfile.flush() + + return True + + def test_with_bodies(self): + client, h2_conn = self._setup_connection() + + self._send_request( + client.wfile, + h2_conn, + headers=[ + (':authority', "127.0.0.1:%s" % self.server.server.address.port), + (':method', 'GET'), + (':scheme', 'https'), + (':path', '/'), + ], + body='foobar with request body', + ) + + done = False + while not done: + events = h2_conn.receive_data(utils.http2_read_frame(client.rfile)) + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + for event in events: + if isinstance(event, h2.events.StreamEnded): + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert self.master.state.flows[0].response.body == b'foobar with request body' + + @requires_alpn class TestPushPromise(_Http2TestBase, _Http2ServerBase): @classmethod @@ -308,12 +374,11 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): if isinstance(event, h2.events.StreamEnded) and event.stream_id == 1: done = True elif isinstance(event, h2.events.PushedStreamReceived): - h2_conn.reset_stream(event.pushed_stream_id) + h2_conn.reset_stream(event.pushed_stream_id, error_code=0x8) client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() bodies = [flow.response.body for flow in self.master.state.flows] assert len(bodies) == 3 assert b'regular_stream' in bodies - assert b'pushed_stream_foo' in bodies - assert b'pushed_stream_bar' in bodies + # the other two bodies might not be transmitted before the reset -- cgit v1.2.3 From 6d3b3994e27b3cd97c74612bc64c7e0457aeb448 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 27 Jan 2016 12:52:18 +0100 Subject: code formatting --- test/test_protocol_http2.py | 9 +++++---- 1 file changed, 5 insertions(+), 4 deletions(-) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index 3554fa4d..6da8cd31 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -1,14 +1,10 @@ from __future__ import (absolute_import, print_function, division) -import inspect -import socket import OpenSSL import pytest import traceback import os import tempfile -import time -from io import BytesIO from libmproxy.proxy.config import ProxyConfig from libmproxy.proxy.server import ProxyServer @@ -35,10 +31,12 @@ requires_alpn = pytest.mark.skipif( not OpenSSL._util.lib.Cryptography_HAS_ALPN, reason="requires OpenSSL with ALPN support") + class _Http2ServerBase(netlib_tservers.ServerTestBase): ssl = dict(alpn_select=b'h2') class handler(netlib.tcp.BaseHandler): + def handle(self): h2_conn = h2.connection.H2Connection(client_side=False) @@ -68,6 +66,7 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): class _Http2TestBase(object): + @classmethod def setup_class(self): self.config = ProxyConfig(**self.get_proxy_config()) @@ -141,6 +140,7 @@ class _Http2TestBase(object): @requires_alpn class TestSimple(_Http2TestBase, _Http2ServerBase): + @classmethod def setup_class(self): _Http2TestBase.setup_class() @@ -262,6 +262,7 @@ class TestWithBodies(_Http2TestBase, _Http2ServerBase): @requires_alpn class TestPushPromise(_Http2TestBase, _Http2ServerBase): + @classmethod def setup_class(self): _Http2TestBase.setup_class() -- cgit v1.2.3 From d8ae7c3e29bcf117eb051f9e74b76fea733c1c64 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 1 Feb 2016 23:03:15 +0100 Subject: fix tests and use netlib utils --- test/test_protocol_http2.py | 14 +++++++------- 1 file changed, 7 insertions(+), 7 deletions(-) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index 6da8cd31..831f70ab 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -20,6 +20,7 @@ logging.getLogger("PIL.PngImagePlugin").setLevel(logging.WARNING) import netlib from netlib import tservers as netlib_tservers +from netlib.utils import http2_read_raw_frame import h2 from hyperframe.frame import Frame @@ -47,8 +48,7 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): self.wfile.flush() while True: - raw_frame = utils.http2_read_frame(self.rfile) - events = h2_conn.receive_data(raw_frame) + events = h2_conn.receive_data(b''.join(http2_read_raw_frame(self.rfile))) self.wfile.write(h2_conn.data_to_send()) self.wfile.flush() @@ -179,7 +179,7 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): done = False while not done: - events = h2_conn.receive_data(utils.http2_read_frame(client.rfile)) + events = h2_conn.receive_data(b''.join(http2_read_raw_frame(client.rfile))) client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() @@ -245,7 +245,7 @@ class TestWithBodies(_Http2TestBase, _Http2ServerBase): done = False while not done: - events = h2_conn.receive_data(utils.http2_read_frame(client.rfile)) + events = h2_conn.receive_data(b''.join(http2_read_raw_frame(client.rfile))) client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() @@ -331,7 +331,7 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): pushed_streams = 0 while ended_streams != 3: try: - events = h2_conn.receive_data(utils.http2_read_frame(client.rfile)) + events = h2_conn.receive_data(b''.join(http2_read_raw_frame(client.rfile))) except: break client.wfile.write(h2_conn.data_to_send()) @@ -365,7 +365,7 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): done = False while not done: try: - events = h2_conn.receive_data(utils.http2_read_frame(client.rfile)) + events = h2_conn.receive_data(b''.join(http2_read_raw_frame(client.rfile))) except: break client.wfile.write(h2_conn.data_to_send()) @@ -379,7 +379,7 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() - bodies = [flow.response.body for flow in self.master.state.flows] + bodies = [flow.response.body for flow in self.master.state.flows if flow.response] assert len(bodies) == 3 assert b'regular_stream' in bodies # the other two bodies might not be transmitted before the reset -- cgit v1.2.3 From 738094e1674ae78f92ff32020e608510ff4af45a Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 1 Feb 2016 23:27:50 +0100 Subject: improve test reliability --- test/test_protocol_http2.py | 12 ++++++++---- 1 file changed, 8 insertions(+), 4 deletions(-) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index 831f70ab..dce8c5da 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -48,7 +48,10 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): self.wfile.flush() while True: - events = h2_conn.receive_data(b''.join(http2_read_raw_frame(self.rfile))) + try: + events = h2_conn.receive_data(b''.join(http2_read_raw_frame(self.rfile))) + except: + break self.wfile.write(h2_conn.data_to_send()) self.wfile.flush() @@ -362,8 +365,8 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): ('foo', 'bar') ]) - done = False - while not done: + streams = 0 + while streams != 3: try: events = h2_conn.receive_data(b''.join(http2_read_raw_frame(client.rfile))) except: @@ -373,8 +376,9 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): for event in events: if isinstance(event, h2.events.StreamEnded) and event.stream_id == 1: - done = True + streams += 1 elif isinstance(event, h2.events.PushedStreamReceived): + streams += 1 h2_conn.reset_stream(event.pushed_stream_id, error_code=0x8) client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() -- cgit v1.2.3 From ca5cc34d0b70f3306f62004be7ceb3f0c2053da7 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 2 Feb 2016 17:48:09 +0100 Subject: cleanup --- test/test_protocol_http2.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index dce8c5da..cc62f734 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -384,6 +384,6 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): client.wfile.flush() bodies = [flow.response.body for flow in self.master.state.flows if flow.response] - assert len(bodies) == 3 + assert len(bodies) >= 1 assert b'regular_stream' in bodies # the other two bodies might not be transmitted before the reset -- cgit v1.2.3 From cf8c063773b70ad37ab0a2125f5ed03c35e17336 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 2 Feb 2016 23:54:35 +0100 Subject: fix http2 race condition --- test/test_protocol_http2.py | 66 ++++++++++++++++++++++++++++++++++++--------- 1 file changed, 54 insertions(+), 12 deletions(-) (limited to 'test/test_protocol_http2.py') diff --git a/test/test_protocol_http2.py b/test/test_protocol_http2.py index cc62f734..38cfdfc3 100644 --- a/test/test_protocol_http2.py +++ b/test/test_protocol_http2.py @@ -5,6 +5,7 @@ import pytest import traceback import os import tempfile +import sys from libmproxy.proxy.config import ProxyConfig from libmproxy.proxy.server import ProxyServer @@ -47,9 +48,11 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): self.wfile.write(h2_conn.data_to_send()) self.wfile.flush() - while True: + done = False + while not done: try: - events = h2_conn.receive_data(b''.join(http2_read_raw_frame(self.rfile))) + raw = b''.join(http2_read_raw_frame(self.rfile)) + events = h2_conn.receive_data(raw) except: break self.wfile.write(h2_conn.data_to_send()) @@ -58,10 +61,12 @@ class _Http2ServerBase(netlib_tservers.ServerTestBase): for event in events: try: if not self.server.handle_server_event(event, h2_conn, self.rfile, self.wfile): + done = True break except Exception as e: print(repr(e)) print(traceback.format_exc()) + done = True break def handle_server_event(self, h2_conn, rfile, wfile): @@ -182,7 +187,10 @@ class TestSimple(_Http2TestBase, _Http2ServerBase): done = False while not done: - events = h2_conn.receive_data(b''.join(http2_read_raw_frame(client.rfile))) + try: + events = h2_conn.receive_data(b''.join(http2_read_raw_frame(client.rfile))) + except: + break client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() @@ -248,7 +256,10 @@ class TestWithBodies(_Http2TestBase, _Http2ServerBase): done = False while not done: - events = h2_conn.receive_data(b''.join(http2_read_raw_frame(client.rfile))) + try: + events = h2_conn.receive_data(b''.join(http2_read_raw_frame(client.rfile))) + except: + break client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() @@ -303,14 +314,16 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): wfile.write(h2_conn.data_to_send()) wfile.flush() - h2_conn.send_headers(2, [(':status', '202')]) - h2_conn.send_headers(4, [(':status', '204')]) + h2_conn.send_headers(2, [(':status', '200')]) + h2_conn.send_headers(4, [(':status', '200')]) wfile.write(h2_conn.data_to_send()) wfile.flush() h2_conn.send_data(1, b'regular_stream') h2_conn.send_data(2, b'pushed_stream_foo') h2_conn.send_data(4, b'pushed_stream_bar') + wfile.write(h2_conn.data_to_send()) + wfile.flush() h2_conn.end_stream(1) h2_conn.end_stream(2) h2_conn.end_stream(4) @@ -330,11 +343,14 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): ('foo', 'bar') ]) + done = False ended_streams = 0 pushed_streams = 0 - while ended_streams != 3: + responses = 0 + while not done: try: - events = h2_conn.receive_data(b''.join(http2_read_raw_frame(client.rfile))) + raw = b''.join(http2_read_raw_frame(client.rfile)) + events = h2_conn.receive_data(raw) except: break client.wfile.write(h2_conn.data_to_send()) @@ -345,7 +361,19 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): ended_streams += 1 elif isinstance(event, h2.events.PushedStreamReceived): pushed_streams += 1 + elif isinstance(event, h2.events.ResponseReceived): + responses += 1 + if isinstance(event, h2.events.ConnectionTerminated): + done = True + if responses == 3 and ended_streams == 3 and pushed_streams == 2: + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() + + assert ended_streams == 3 assert pushed_streams == 2 bodies = [flow.response.body for flow in self.master.state.flows] @@ -365,8 +393,11 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): ('foo', 'bar') ]) - streams = 0 - while streams != 3: + done = False + ended_streams = 0 + pushed_streams = 0 + responses = 0 + while not done: try: events = h2_conn.receive_data(b''.join(http2_read_raw_frame(client.rfile))) except: @@ -376,12 +407,23 @@ class TestPushPromise(_Http2TestBase, _Http2ServerBase): for event in events: if isinstance(event, h2.events.StreamEnded) and event.stream_id == 1: - streams += 1 + ended_streams += 1 elif isinstance(event, h2.events.PushedStreamReceived): - streams += 1 + pushed_streams += 1 h2_conn.reset_stream(event.pushed_stream_id, error_code=0x8) client.wfile.write(h2_conn.data_to_send()) client.wfile.flush() + elif isinstance(event, h2.events.ResponseReceived): + responses += 1 + if isinstance(event, h2.events.ConnectionTerminated): + done = True + + if responses >= 1 and ended_streams >= 1 and pushed_streams == 2: + done = True + + h2_conn.close_connection() + client.wfile.write(h2_conn.data_to_send()) + client.wfile.flush() bodies = [flow.response.body for flow in self.master.state.flows if flow.response] assert len(bodies) >= 1 -- cgit v1.2.3