From eeaed93a83fbe14762e263e9f25b5361088daa15 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 11 Jun 2015 15:37:17 +0200 Subject: improve ALPN integration --- netlib/tcp.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) diff --git a/netlib/tcp.py b/netlib/tcp.py index 9a980035..98b17c50 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -404,16 +404,17 @@ class _Connection(object): context.set_info_callback(log_ssl_key) if OpenSSL._util.lib.Cryptography_HAS_ALPN: - # advertise application layer protocols if alpn_protos is not None: + # advertise application layer protocols context.set_alpn_protos(alpn_protos) - - # select application layer protocol - if alpn_select is not None: - def alpn_select_f(conn, options): - return bytes(alpn_select) - - context.set_alpn_select_callback(alpn_select_f) + elif alpn_select is not None: + # select application layer protocol + def alpn_select_callback(conn, options): + if alpn_select in options: + return bytes(alpn_select) + else: + return options[0] + context.set_alpn_select_callback(alpn_select_callback) return context @@ -612,6 +613,12 @@ class BaseHandler(_Connection): def settimeout(self, n): self.connection.settimeout(n) + def get_alpn_proto_negotiated(self): + if OpenSSL._util.lib.Cryptography_HAS_ALPN and self.ssl_established: + return self.connection.get_alpn_proto_negotiated() + else: # pragma no cover + return None + class TCPServer(object): request_queue_size = 20 -- cgit v1.2.3 From 8ea157775debeccfa0f2fab3aa7e009d13ce4391 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 11 Jun 2015 15:38:32 +0200 Subject: http2: general improvements --- netlib/http2/protocol.py | 63 ++++++++++++++++++++++++++------------- test/http2/test_http2_protocol.py | 41 +++++++++++++++++++++---- 2 files changed, 78 insertions(+), 26 deletions(-) diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index feac220c..4b69764f 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -26,12 +26,13 @@ class HTTP2Protocol(object): ) # "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" - CLIENT_CONNECTION_PREFACE = '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a' + CLIENT_CONNECTION_PREFACE = '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex') ALPN_PROTO_H2 = 'h2' - def __init__(self, tcp_client): - self.tcp_client = tcp_client + def __init__(self, tcp_handler, is_server=False): + self.tcp_handler = tcp_handler + self.is_server = is_server self.http2_settings = frame.HTTP2_DEFAULT_SETTINGS.copy() self.current_stream_id = None @@ -39,28 +40,39 @@ class HTTP2Protocol(object): self.decoder = Decoder() def check_alpn(self): - alp = self.tcp_client.get_alpn_proto_negotiated() + alp = self.tcp_handler.get_alpn_proto_negotiated() if alp != self.ALPN_PROTO_H2: raise NotImplementedError( "HTTP2Protocol can not handle unknown ALP: %s" % alp) return True - def perform_connection_preface(self): - self.tcp_client.wfile.write( - bytes(self.CLIENT_CONNECTION_PREFACE.decode('hex'))) - self.send_frame(frame.SettingsFrame(state=self)) - - # read server settings frame - frm = frame.Frame.from_file(self.tcp_client.rfile, self) + def _receive_settings(self): + frm = frame.Frame.from_file(self.tcp_handler.rfile, self) assert isinstance(frm, frame.SettingsFrame) self._apply_settings(frm.settings) - # read setting ACK frame + def _read_settings_ack(self): settings_ack_frame = self.read_frame() assert isinstance(settings_ack_frame, frame.SettingsFrame) assert settings_ack_frame.flags & frame.Frame.FLAG_ACK assert len(settings_ack_frame.settings) == 0 + def perform_server_connection_preface(self): + magic_length = len(self.CLIENT_CONNECTION_PREFACE) + magic = self.tcp_handler.rfile.safe_read(magic_length) + assert magic == self.CLIENT_CONNECTION_PREFACE + + self.send_frame(frame.SettingsFrame(state=self)) + self._receive_settings() + self._read_settings_ack() + + def perform_client_connection_preface(self): + self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE) + + self.send_frame(frame.SettingsFrame(state=self)) + self._receive_settings() + self._read_settings_ack() + def next_stream_id(self): if self.current_stream_id is None: self.current_stream_id = 1 @@ -70,11 +82,11 @@ class HTTP2Protocol(object): def send_frame(self, frame): raw_bytes = frame.to_bytes() - self.tcp_client.wfile.write(raw_bytes) - self.tcp_client.wfile.flush() + self.tcp_handler.wfile.write(raw_bytes) + self.tcp_handler.wfile.flush() def read_frame(self): - frm = frame.Frame.from_file(self.tcp_client.rfile, self) + frm = frame.Frame.from_file(self.tcp_handler.rfile, self) if isinstance(frm, frame.SettingsFrame): self._apply_settings(frm.settings) @@ -139,25 +151,36 @@ class HTTP2Protocol(object): self._create_body(body, stream_id))) def read_response(self): + headers, body = self._receive_transmission() + return headers[':status'], headers, body + + def read_request(self): + return self._receive_transmission() + + def _receive_transmission(self): + body_expected = True + header_block_fragment = b'' body = b'' while True: frm = self.read_frame() - if isinstance(frm, frame.HeadersFrame): + if isinstance(frm, frame.HeadersFrame) or isinstance(frm, frame.ContinuationFrame): header_block_fragment += frm.header_block_fragment - if frm.flags | frame.Frame.FLAG_END_HEADERS: + if frm.flags & frame.Frame.FLAG_END_HEADERS: + if frm.flags & frame.Frame.FLAG_END_STREAM: + body_expected = False break - while True: + while body_expected: frm = self.read_frame() if isinstance(frm, frame.DataFrame): body += frm.payload - if frm.flags | frame.Frame.FLAG_END_STREAM: + if frm.flags & frame.Frame.FLAG_END_STREAM: break headers = {} for header, value in self.decoder.decode(header_block_fragment): headers[header] = value - return headers[':status'], headers, body + return headers, body diff --git a/test/http2/test_http2_protocol.py b/test/http2/test_http2_protocol.py index cb46bc68..1591edd8 100644 --- a/test/http2/test_http2_protocol.py +++ b/test/http2/test_http2_protocol.py @@ -50,7 +50,39 @@ class TestCheckALPNMismatch(test.ServerTestBase): tutils.raises(NotImplementedError, protocol.check_alpn) -class TestPerformConnectionPreface(test.ServerTestBase): +class TestPerformServerConnectionPreface(test.ServerTestBase): + class handler(tcp.BaseHandler): + + def handle(self): + # send magic + self.wfile.write(\ + '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex')) + self.wfile.flush() + + # send empty settings frame + self.wfile.write('000000040000000000'.decode('hex')) + self.wfile.flush() + + # check empty settings frame + assert self.rfile.read(9) ==\ + '000000040000000000'.decode('hex') + + # check settings acknowledgement + assert self.rfile.read(9) == \ + '000000040100000000'.decode('hex') + + # send settings acknowledgement + self.wfile.write('000000040100000000'.decode('hex')) + self.wfile.flush() + + def test_perform_server_connection_preface(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + protocol = http2.HTTP2Protocol(c) + protocol.perform_server_connection_preface() + + +class TestPerformClientConnectionPreface(test.ServerTestBase): class handler(tcp.BaseHandler): def handle(self): @@ -74,14 +106,11 @@ class TestPerformConnectionPreface(test.ServerTestBase): self.wfile.write('000000040100000000'.decode('hex')) self.wfile.flush() - ssl = True - - def test_perform_connection_preface(self): + def test_perform_client_connection_preface(self): c = tcp.TCPClient(("127.0.0.1", self.port)) c.connect() - c.convert_to_ssl() protocol = http2.HTTP2Protocol(c) - protocol.perform_connection_preface() + protocol.perform_client_connection_preface() class TestStreamIds(): -- cgit v1.2.3 From a901bc3032747faf00adf82c3187d38213c070ca Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 12 Jun 2015 14:41:54 +0200 Subject: http2: add response creation --- netlib/http2/protocol.py | 56 ++++++++++++++++++++++++++++----------- test/http2/test_http2_protocol.py | 2 +- 2 files changed, 42 insertions(+), 16 deletions(-) diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 4b69764f..56aee490 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -26,7 +26,8 @@ class HTTP2Protocol(object): ) # "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" - CLIENT_CONNECTION_PREFACE = '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex') + CLIENT_CONNECTION_PREFACE =\ + '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex') ALPN_PROTO_H2 = 'h2' @@ -38,6 +39,7 @@ class HTTP2Protocol(object): self.current_stream_id = None self.encoder = Encoder() self.decoder = Decoder() + self.connection_preface_performed = False def check_alpn(self): alp = self.tcp_handler.get_alpn_proto_negotiated() @@ -57,25 +59,36 @@ class HTTP2Protocol(object): assert settings_ack_frame.flags & frame.Frame.FLAG_ACK assert len(settings_ack_frame.settings) == 0 - def perform_server_connection_preface(self): - magic_length = len(self.CLIENT_CONNECTION_PREFACE) - magic = self.tcp_handler.rfile.safe_read(magic_length) - assert magic == self.CLIENT_CONNECTION_PREFACE + def perform_server_connection_preface(self, force=False): + if force or not self.connection_preface_performed: + self.connection_preface_performed = True - self.send_frame(frame.SettingsFrame(state=self)) - self._receive_settings() - self._read_settings_ack() + magic_length = len(self.CLIENT_CONNECTION_PREFACE) + magic = self.tcp_handler.rfile.safe_read(magic_length) + assert magic == self.CLIENT_CONNECTION_PREFACE - def perform_client_connection_preface(self): - self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE) + self.send_frame(frame.SettingsFrame(state=self)) + self._receive_settings() + self._read_settings_ack() - self.send_frame(frame.SettingsFrame(state=self)) - self._receive_settings() - self._read_settings_ack() + def perform_client_connection_preface(self, force=False): + if force or not self.connection_preface_performed: + self.connection_preface_performed = True + + self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE) + + self.send_frame(frame.SettingsFrame(state=self)) + self._receive_settings() + self._read_settings_ack() def next_stream_id(self): if self.current_stream_id is None: - self.current_stream_id = 1 + if self.is_server: + # servers must use even stream ids + self.current_stream_id = 2 + else: + # clients must use odd stream ids + self.current_stream_id = 1 else: self.current_stream_id += 2 return self.current_stream_id @@ -165,7 +178,8 @@ class HTTP2Protocol(object): while True: frm = self.read_frame() - if isinstance(frm, frame.HeadersFrame) or isinstance(frm, frame.ContinuationFrame): + if isinstance(frm, frame.HeadersFrame)\ + or isinstance(frm, frame.ContinuationFrame): header_block_fragment += frm.header_block_fragment if frm.flags & frame.Frame.FLAG_END_HEADERS: if frm.flags & frame.Frame.FLAG_END_STREAM: @@ -184,3 +198,15 @@ class HTTP2Protocol(object): headers[header] = value return headers, body + + def create_response(self, code, headers=None, body=None): + if headers is None: + headers = [] + + headers = [(b':status', bytes(str(code)))] + headers + + stream_id = self.next_stream_id() + + return list(itertools.chain( + self._create_headers(headers, stream_id, end_stream=(body is None)), + self._create_body(body, stream_id))) diff --git a/test/http2/test_http2_protocol.py b/test/http2/test_http2_protocol.py index 1591edd8..76a0ffe9 100644 --- a/test/http2/test_http2_protocol.py +++ b/test/http2/test_http2_protocol.py @@ -55,7 +55,7 @@ class TestPerformServerConnectionPreface(test.ServerTestBase): def handle(self): # send magic - self.wfile.write(\ + self.wfile.write( '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex')) self.wfile.flush() -- cgit v1.2.3 From 5fab755a05f2ddd1b3e8e446e10fdcbded894e70 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 12 Jun 2015 15:21:23 +0200 Subject: add more tests --- netlib/tcp.py | 8 ++-- test/http2/test_http2_protocol.py | 87 +++++++++++++++++++++++++++++++++++++-- test/test_tcp.py | 5 +++ 3 files changed, 92 insertions(+), 8 deletions(-) diff --git a/netlib/tcp.py b/netlib/tcp.py index 98b17c50..eb8a523f 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -412,7 +412,7 @@ class _Connection(object): def alpn_select_callback(conn, options): if alpn_select in options: return bytes(alpn_select) - else: + else: # pragma no cover return options[0] context.set_alpn_select_callback(alpn_select_callback) @@ -500,9 +500,9 @@ class TCPClient(_Connection): return self.connection.gettimeout() def get_alpn_proto_negotiated(self): - if OpenSSL._util.lib.Cryptography_HAS_ALPN: + if OpenSSL._util.lib.Cryptography_HAS_ALPN and self.ssl_established: return self.connection.get_alpn_proto_negotiated() - else: # pragma no cover + else: return None @@ -616,7 +616,7 @@ class BaseHandler(_Connection): def get_alpn_proto_negotiated(self): if OpenSSL._util.lib.Cryptography_HAS_ALPN and self.ssl_established: return self.connection.get_alpn_proto_negotiated() - else: # pragma no cover + else: return None diff --git a/test/http2/test_http2_protocol.py b/test/http2/test_http2_protocol.py index 76a0ffe9..ebd2c9a7 100644 --- a/test/http2/test_http2_protocol.py +++ b/test/http2/test_http2_protocol.py @@ -1,4 +1,3 @@ - import OpenSSL from netlib import http2 @@ -113,11 +112,11 @@ class TestPerformClientConnectionPreface(test.ServerTestBase): protocol.perform_client_connection_preface() -class TestStreamIds(): +class TestClientStreamIds(): c = tcp.TCPClient(("127.0.0.1", 0)) protocol = http2.HTTP2Protocol(c) - def test_stream_ids(self): + def test_client_stream_ids(self): assert self.protocol.current_stream_id is None assert self.protocol.next_stream_id() == 1 assert self.protocol.current_stream_id == 1 @@ -127,6 +126,20 @@ class TestStreamIds(): assert self.protocol.current_stream_id == 5 +class TestServerStreamIds(): + c = tcp.TCPClient(("127.0.0.1", 0)) + protocol = http2.HTTP2Protocol(c, is_server=True) + + def test_server_stream_ids(self): + assert self.protocol.current_stream_id is None + assert self.protocol.next_stream_id() == 2 + assert self.protocol.current_stream_id == 2 + assert self.protocol.next_stream_id() == 4 + assert self.protocol.current_stream_id == 4 + assert self.protocol.next_stream_id() == 6 + assert self.protocol.current_stream_id == 6 + + class TestApplySettings(test.ServerTestBase): class handler(tcp.BaseHandler): @@ -242,5 +255,71 @@ class TestReadResponse(test.ServerTestBase): status, headers, body = protocol.read_response() assert headers == {':status': '200', 'etag': 'foobar'} - assert status == '200' + assert status == "200" + assert body == b'foobar' + + +class TestReadEmptyResponse(test.ServerTestBase): + class handler(tcp.BaseHandler): + + def handle(self): + self.wfile.write( + b'00000801050000000188628594e78c767f'.decode('hex')) + self.wfile.flush() + + ssl = True + + def test_read_empty_response(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + c.convert_to_ssl() + protocol = http2.HTTP2Protocol(c) + + status, headers, body = protocol.read_response() + + assert headers == {':status': '200', 'etag': 'foobar'} + assert status == "200" + assert body == b'' + + +class TestReadRequest(test.ServerTestBase): + class handler(tcp.BaseHandler): + + def handle(self): + self.wfile.write( + b'000003010400000001828487'.decode('hex')) + self.wfile.write( + b'000006000100000001666f6f626172'.decode('hex')) + self.wfile.flush() + + ssl = True + + def test_read_request(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + c.convert_to_ssl() + protocol = http2.HTTP2Protocol(c, is_server=True) + + headers, body = protocol.read_request() + + assert headers == {':method': 'GET', ':path': '/', ':scheme': 'https'} assert body == b'foobar' + + +class TestCreateResponse(): + c = tcp.TCPClient(("127.0.0.1", 0)) + + def test_create_request_simple(self): + bytes = http2.HTTP2Protocol(self.c, is_server=True).create_response(200) + assert len(bytes) == 1 + assert bytes[0] ==\ + '00000101050000000288'.decode('hex') + + def test_create_request_with_body(self): + bytes = http2.HTTP2Protocol(self.c, is_server=True).create_response( + 200, [(b'foo', b'bar')], 'foobar') + assert len(bytes) == 2 + assert bytes[0] ==\ + '00000901040000000288408294e7838c767f'.decode('hex') + assert bytes[1] ==\ + '000006000100000002666f6f626172'.decode('hex') diff --git a/test/test_tcp.py b/test/test_tcp.py index d5506556..8aa34d2b 100644 --- a/test/test_tcp.py +++ b/test/test_tcp.py @@ -376,6 +376,11 @@ class TestALPN(test.ServerTestBase): c.convert_to_ssl(alpn_protos=["foobar"]) assert c.get_alpn_proto_negotiated() == "foobar" + def test_no_alpn(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + assert c.get_alpn_proto_negotiated() == None + else: def test_none_alpn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) -- cgit v1.2.3 From 9c6d237d02290c2388f19ec8f215827d4f921e4b Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 12 Jun 2015 16:03:01 +0200 Subject: add new TLS methods --- netlib/tcp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/netlib/tcp.py b/netlib/tcp.py index eb8a523f..74fe70d4 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -19,6 +19,9 @@ SSLv2_METHOD = SSL.SSLv2_METHOD SSLv3_METHOD = SSL.SSLv3_METHOD SSLv23_METHOD = SSL.SSLv23_METHOD TLSv1_METHOD = SSL.TLSv1_METHOD +TLSv1_1_METHOD = SSL.TLSv1_1_METHOD +TLSv1_2_METHOD = SSL.TLSv1_2_METHOD + OP_NO_SSLv2 = SSL.OP_NO_SSLv2 OP_NO_SSLv3 = SSL.OP_NO_SSLv3 @@ -376,7 +379,7 @@ class _Connection(object): alpn_select=None, ): """ - :param method: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, TLSv1_METHOD or TLSv1_1_METHOD + :param method: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, TLSv1_METHOD, TLSv1_1_METHOD, or TLSv1_2_METHOD :param options: A bit field consisting of OpenSSL.SSL.OP_* values :param cipher_list: A textual OpenSSL cipher list, see https://www.openssl.org/docs/apps/ciphers.html :rtype : SSL.Context -- cgit v1.2.3 From 8d71a5b4aba8248b97918b11b12275bbf5197337 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 14 Jun 2015 19:17:34 +0200 Subject: http2: add authority header --- netlib/http2/protocol.py | 6 +++++- test/http2/test_http2_protocol.py | 4 ++-- 2 files changed, 7 insertions(+), 3 deletions(-) diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 56aee490..1e722dfb 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -152,10 +152,13 @@ class HTTP2Protocol(object): if headers is None: headers = [] + authority = self.tcp_handler.sni if self.tcp_handler.sni else self.tcp_handler.address.host headers = [ (b':method', bytes(method)), (b':path', bytes(path)), - (b':scheme', b'https')] + headers + (b':scheme', b'https'), + (b':authority', authority), + ] + headers stream_id = self.next_stream_id() @@ -192,6 +195,7 @@ class HTTP2Protocol(object): body += frm.payload if frm.flags & frame.Frame.FLAG_END_STREAM: break + # TODO: implement window update & flow headers = {} for header, value in self.decoder.decode(header_block_fragment): diff --git a/test/http2/test_http2_protocol.py b/test/http2/test_http2_protocol.py index ebd2c9a7..34c69fa9 100644 --- a/test/http2/test_http2_protocol.py +++ b/test/http2/test_http2_protocol.py @@ -222,14 +222,14 @@ class TestCreateRequest(): def test_create_request_simple(self): bytes = http2.HTTP2Protocol(self.c).create_request('GET', '/') assert len(bytes) == 1 - assert bytes[0] == '000003010500000001828487'.decode('hex') + assert bytes[0] == '00000c0105000000018284874187089d5c0b8170ff'.decode('hex') def test_create_request_with_body(self): bytes = http2.HTTP2Protocol(self.c).create_request( 'GET', '/', [(b'foo', b'bar')], 'foobar') assert len(bytes) == 2 assert bytes[0] ==\ - '00000b010400000001828487408294e7838c767f'.decode('hex') + '0000140104000000018284874187089d5c0b8170ff408294e7838c767f'.decode('hex') assert bytes[1] ==\ '000006000100000001666f6f626172'.decode('hex') -- cgit v1.2.3 From 0d137eac6f4c00a72d3aa4d11fce7d1ea15f0f21 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 14 Jun 2015 19:50:35 +0200 Subject: simplify ALPN --- netlib/tcp.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) diff --git a/netlib/tcp.py b/netlib/tcp.py index 74fe70d4..897e3e65 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -535,7 +535,6 @@ class BaseHandler(_Connection): request_client_cert=None, chain_file=None, dhparams=None, - alpn_select=None, **sslctx_kwargs): """ cert: A certutils.SSLCert object. @@ -562,9 +561,7 @@ class BaseHandler(_Connection): until then we're conservative. """ - context = self._create_ssl_context( - alpn_select=alpn_select, - **sslctx_kwargs) + context = self._create_ssl_context(**sslctx_kwargs) context.use_privatekey(key) context.use_certificate(cert.x509) @@ -589,7 +586,7 @@ class BaseHandler(_Connection): return context - def convert_to_ssl(self, cert, key, alpn_select=None, **sslctx_kwargs): + def convert_to_ssl(self, cert, key, **sslctx_kwargs): """ Convert connection to SSL. For a list of parameters, see BaseHandler._create_ssl_context(...) @@ -598,7 +595,6 @@ class BaseHandler(_Connection): context = self.create_ssl_context( cert, key, - alpn_select=alpn_select, **sslctx_kwargs) self.connection = SSL.Connection(context, self.connection) self.connection.set_accept_state() -- cgit v1.2.3 From 08f988e9f65d8628657cf2018fd36ab82a4d0789 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 11:58:24 +0200 Subject: improve meta code --- check_coding_style.sh | 4 ++-- setup.cfg | 1 - 2 files changed, 2 insertions(+), 3 deletions(-) diff --git a/check_coding_style.sh b/check_coding_style.sh index 5b38e003..a1c94e03 100755 --- a/check_coding_style.sh +++ b/check_coding_style.sh @@ -5,7 +5,7 @@ if [[ -n "$(git status -s)" ]]; then echo "autopep8 yielded the following changes:" git status -s git --no-pager diff - exit 1 + exit 0 # don't be so strict about coding style errors fi autoflake -i -r --remove-all-unused-imports --remove-unused-variables . @@ -13,7 +13,7 @@ if [[ -n "$(git status -s)" ]]; then echo "autoflake yielded the following changes:" git status -s git --no-pager diff - exit 1 + exit 0 # don't be so strict about coding style errors fi echo "Coding style seems to be ok." diff --git a/setup.cfg b/setup.cfg index bc980d56..4207020e 100644 --- a/setup.cfg +++ b/setup.cfg @@ -4,6 +4,5 @@ max-complexity = 15 [pep8] max-line-length = 80 -max-complexity = 15 exclude = */contrib/* ignore = E251,E309 -- cgit v1.2.3 From fe764cde5229046b8447062971c61fac745d2d58 Mon Sep 17 00:00:00 2001 From: Kyle Morton Date: Mon, 15 Jun 2015 10:16:44 -0700 Subject: Adding support for upstream certificate validation when using SSL/TLS with an instance of TCPClient. --- netlib/tcp.py | 23 +++++++++++++++++++++ test/data/not-server.crt | 15 ++++++++++++++ test/test_tcp.py | 53 ++++++++++++++++++++++++++++++++++++++++++++++++ 3 files changed, 91 insertions(+) create mode 100644 test/data/not-server.crt diff --git a/netlib/tcp.py b/netlib/tcp.py index 9a980035..ca948514 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -21,6 +21,7 @@ SSLv23_METHOD = SSL.SSLv23_METHOD TLSv1_METHOD = SSL.TLSv1_METHOD OP_NO_SSLv2 = SSL.OP_NO_SSLv2 OP_NO_SSLv3 = SSL.OP_NO_SSLv3 +VERIFY_NONE = SSL.VERIFY_NONE class NetLibError(Exception): @@ -371,6 +372,9 @@ class _Connection(object): def _create_ssl_context(self, method=SSLv23_METHOD, options=(OP_NO_SSLv2 | OP_NO_SSLv3), + verify_options=VERIFY_NONE, + ca_path=None, + ca_pemfile=None, cipher_list=None, alpn_protos=None, alpn_select=None, @@ -378,6 +382,9 @@ class _Connection(object): """ :param method: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, TLSv1_METHOD or TLSv1_1_METHOD :param options: A bit field consisting of OpenSSL.SSL.OP_* values + :param verify_options: A bit field consisting of OpenSSL.SSL.VERIFY_* values + :param ca_path: Path to a directory of trusted CA certificates prepared using the c_rehash tool + :param ca_pemfile: Path to a PEM formatted trusted CA certificate :param cipher_list: A textual OpenSSL cipher list, see https://www.openssl.org/docs/apps/ciphers.html :rtype : SSL.Context """ @@ -386,6 +393,19 @@ class _Connection(object): if options is not None: context.set_options(options) + # Verify Options (NONE/PEER/PEER|FAIL_IF_... and trusted CAs) + if verify_options is not None and verify_options is not VERIFY_NONE: + def verify_cert(conn, cert, errno, err_depth, is_cert_verified): + if is_cert_verified: + return True + raise NetLibError( + "Upstream certificate validation failed at depth: %s with error number: %s" % + (err_depth, errno)) + + context.set_verify(verify_options, verify_cert) + if ca_path is not None or ca_pemfile is not None: + context.load_verify_locations(ca_pemfile, ca_path) + # Workaround for # https://github.com/pyca/pyopenssl/issues/190 # https://github.com/mitmproxy/mitmproxy/issues/472 @@ -458,6 +478,9 @@ class TCPClient(_Connection): cert: Path to a file containing both client cert and private key. options: A bit field consisting of OpenSSL.SSL.OP_* values + verify_options: A bit field consisting of OpenSSL.SSL.VERIFY_* values + ca_path: Path to a directory of trusted CA certificates prepared using the c_rehash tool + ca_pemfile: Path to a PEM formatted trusted CA certificate """ context = self.create_ssl_context( alpn_protos=alpn_protos, diff --git a/test/data/not-server.crt b/test/data/not-server.crt new file mode 100644 index 00000000..08c015c2 --- /dev/null +++ b/test/data/not-server.crt @@ -0,0 +1,15 @@ +-----BEGIN CERTIFICATE----- +MIICRTCCAa4CCQD/j4qq1h3iCjANBgkqhkiG9w0BAQUFADBnMQswCQYDVQQGEwJV +UzELMAkGA1UECBMCQ0ExETAPBgNVBAcTCFNvbWVDaXR5MRcwFQYDVQQKEw5Ob3RU +aGVSaWdodE9yZzELMAkGA1UECxMCTkExEjAQBgNVBAMTCU5vdFNlcnZlcjAeFw0x +NTA2MTMwMTE2MDZaFw0yNTA2MTAwMTE2MDZaMGcxCzAJBgNVBAYTAlVTMQswCQYD +VQQIEwJDQTERMA8GA1UEBxMIU29tZUNpdHkxFzAVBgNVBAoTDk5vdFRoZVJpZ2h0 +T3JnMQswCQYDVQQLEwJOQTESMBAGA1UEAxMJTm90U2VydmVyMIGfMA0GCSqGSIb3 +DQEBAQUAA4GNADCBiQKBgQDPkJlXAOCMKF0R7aDn5QJ7HtrJgOUDk/LpbhKhRZZR +dRGnJ4/HQxYYHh9k/4yZamYcvQPUxvFJt7UJUocf+84LUcIusUk7GvJMgsMVtFMq +7UKNXBN5tl3oOtoFDWGMZ8ksaIxS6oW3V/9v2WgU23PfvwE0EZqy+QhMLZZP5GOH +RwIDAQABMA0GCSqGSIb3DQEBBQUAA4GBAJI6UtMKdCS2ghjqhAek2W1rt9u+Wuvx +776WYm5VyrJEtBDc/axLh0OteXzy/A31JrYe15fnVWIeFbDF0Ief9/Ezv6Jn+Pk8 +DErw5IHk2B399O4K3L3Eig06piu7uf3vE4l8ZanY02ZEnw7DyL6kmG9lX98VGenF +uXPfu3yxKbR4 +-----END CERTIFICATE----- diff --git a/test/test_tcp.py b/test/test_tcp.py index d5506556..081c83a7 100644 --- a/test/test_tcp.py +++ b/test/test_tcp.py @@ -171,6 +171,59 @@ class TestSSLv3Only(test.ServerTestBase): tutils.raises(tcp.NetLibError, c.convert_to_ssl, sni="foo.com") +class TestSSLUpstreamCertVerification(test.ServerTestBase): + handler = EchoHandler + + ssl = dict( + cert=tutils.test_data.path("data/server.crt") + ) + + def test_mode_default(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + + c.convert_to_ssl() + + testval = "echo!\n" + c.wfile.write(testval) + c.wfile.flush() + assert c.rfile.readline() == testval + + def test_mode_none(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + + c.convert_to_ssl(verify_options=SSL.VERIFY_NONE) + + testval = "echo!\n" + c.wfile.write(testval) + c.wfile.flush() + assert c.rfile.readline() == testval + + def test_mode_strict_w_bad_cert(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + + tutils.raises( + tcp.NetLibError, + c.convert_to_ssl, + verify_options=SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, + ca_pemfile=tutils.test_data.path("data/not-server.crt")) + + def test_mode_strict_w_cert(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + + c.convert_to_ssl( + verify_options=SSL.VERIFY_PEER | SSL.VERIFY_FAIL_IF_NO_PEER_CERT, + ca_pemfile=tutils.test_data.path("data/server.crt")) + + testval = "echo!\n" + c.wfile.write(testval) + c.wfile.flush() + assert c.rfile.readline() == testval + + class TestSSLClientCert(test.ServerTestBase): class handler(tcp.BaseHandler): -- cgit v1.2.3 From 9089226d661793e2eb5d3cf6f1bbe916578e5b7b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 16 Jun 2015 02:31:47 +0200 Subject: explicitly state that we only support 2.7 --- setup.py | 1 + 1 file changed, 1 insertion(+) diff --git a/setup.py b/setup.py index 0051ea77..dc19a870 100644 --- a/setup.py +++ b/setup.py @@ -49,6 +49,7 @@ setup( "Operating System :: POSIX", "Programming Language :: Python", "Programming Language :: Python :: 2", + "Programming Language :: Python :: 2.7", "Programming Language :: Python :: Implementation :: CPython", "Programming Language :: Python :: Implementation :: PyPy", "Topic :: Internet", -- cgit v1.2.3 From d8db9330a01cfab2603c8ad465c0ba00e8310994 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 16 Jun 2015 02:52:07 +0200 Subject: update badges --- README.mkd | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/README.mkd b/README.mkd index 79e7f803..33d50b29 100644 --- a/README.mkd +++ b/README.mkd @@ -1,8 +1,9 @@ -[![Build Status](https://travis-ci.org/mitmproxy/netlib.svg?branch=master)](https://travis-ci.org/mitmproxy/netlib) -[![Coverage Status](https://coveralls.io/repos/mitmproxy/netlib/badge.svg?branch=master)](https://coveralls.io/r/mitmproxy/netlib) -[![Latest Version](https://pypip.in/version/netlib/badge.svg?style=flat)](https://pypi.python.org/pypi/netlib) -[![Supported Python versions](https://pypip.in/py_versions/netlib/badge.svg?style=flat)](https://pypi.python.org/pypi/netlib) -[![Supported Python implementations](https://pypip.in/implementation/netlib/badge.svg?style=flat)](https://pypi.python.org/pypi/netlib) +[![Build Status](https://img.shields.io/travis/mitmproxy/netlib/master.svg)](https://travis-ci.org/mitmproxy/netlib) +[![Coverage Status](https://img.shields.io/coveralls/mitmproxy/netlib/master.svg)](https://coveralls.io/r/mitmproxy/netlib) +[![Downloads](https://img.shields.io/pypi/dm/netlib.svg?color=orange)](https://pypi.python.org/pypi/netlib) +[![Latest Version](https://img.shields.io/pypi/v/netlib.svg)](https://pypi.python.org/pypi/netlib) +[![Supported Python versions](https://img.shields.io/pypi/pyversions/netlib.svg)](https://pypi.python.org/pypi/netlib) +[![Supported Python implementations](https://img.shields.io/pypi/implementation/netlib.svg)](https://pypi.python.org/pypi/netlib) Netlib is a collection of network utility classes, used by the pathod and mitmproxy projects. It differs from other projects in some fundamental @@ -15,4 +16,4 @@ Requirements ------------ * [Python](http://www.python.org) 2.7.x. -* Third-party packages listed in [setup.py](https://github.com/mitmproxy/netlib/blob/master/setup.py) \ No newline at end of file +* Third-party packages listed in [setup.py](https://github.com/mitmproxy/netlib/blob/master/setup.py) -- cgit v1.2.3 From 1f0c55a942ef1e36d21e2d8006a1585ad4cf2700 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 16 Jun 2015 03:30:34 +0200 Subject: add hacking section --- README.mkd | 5 +++++ 1 file changed, 5 insertions(+) diff --git a/README.mkd b/README.mkd index 33d50b29..7039e203 100644 --- a/README.mkd +++ b/README.mkd @@ -17,3 +17,8 @@ Requirements * [Python](http://www.python.org) 2.7.x. * Third-party packages listed in [setup.py](https://github.com/mitmproxy/netlib/blob/master/setup.py) + +Hacking +------- + +If you'd like to work on netlib, check out the instructions in mitmproxy's [README](https://github.com/mitmproxy/mitmproxy#hacking). -- cgit v1.2.3 From 12702b9a01fb6baf4d675d6f974c140581982843 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 13:15:06 +0200 Subject: http2: improve frame output --- netlib/http2/frame.py | 11 +++------ netlib/http2/protocol.py | 61 ++++++++++++++++++++++++++++-------------------- 2 files changed, 39 insertions(+), 33 deletions(-) diff --git a/netlib/http2/frame.py b/netlib/http2/frame.py index 4a305d82..3e285cba 100644 --- a/netlib/http2/frame.py +++ b/netlib/http2/frame.py @@ -113,16 +113,11 @@ class Frame(object): def payload_human_readable(self): # pragma: no cover raise NotImplementedError() - def human_readable(self): + def human_readable(self, direction="-"): return "\n".join([ - "============================================================", - "length: %d bytes" % self.length, - "type: %s (%#x)" % (self.__class__.__name__, self.TYPE), - "flags: %#x" % self.flags, - "stream_id: %#x" % self.stream_id, - "------------------------------------------------------------", + "%s: %s | length: %d | flags: %#x | stream_id: %d" % (direction, self.__class__.__name__, self.length, self.flags, self.stream_id), self.payload_human_readable(), - "============================================================", + "===============================================================", ]) def __eq__(self, other): diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 1e722dfb..7bf68602 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -48,13 +48,12 @@ class HTTP2Protocol(object): "HTTP2Protocol can not handle unknown ALP: %s" % alp) return True - def _receive_settings(self): - frm = frame.Frame.from_file(self.tcp_handler.rfile, self) + def _receive_settings(self, hide=False): + frm = self.read_frame(hide) assert isinstance(frm, frame.SettingsFrame) - self._apply_settings(frm.settings) - def _read_settings_ack(self): - settings_ack_frame = self.read_frame() + def _read_settings_ack(self, hide=False): + settings_ack_frame = self.read_frame(hide) assert isinstance(settings_ack_frame, frame.SettingsFrame) assert settings_ack_frame.flags & frame.Frame.FLAG_ACK assert len(settings_ack_frame.settings) == 0 @@ -67,9 +66,8 @@ class HTTP2Protocol(object): magic = self.tcp_handler.rfile.safe_read(magic_length) assert magic == self.CLIENT_CONNECTION_PREFACE - self.send_frame(frame.SettingsFrame(state=self)) - self._receive_settings() - self._read_settings_ack() + self.send_frame(frame.SettingsFrame(state=self), hide=True) + self._receive_settings(hide=True) def perform_client_connection_preface(self, force=False): if force or not self.connection_preface_performed: @@ -77,9 +75,8 @@ class HTTP2Protocol(object): self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE) - self.send_frame(frame.SettingsFrame(state=self)) - self._receive_settings() - self._read_settings_ack() + self.send_frame(frame.SettingsFrame(state=self), hide=True) + self._receive_settings(hide=True) def next_stream_id(self): if self.current_stream_id is None: @@ -93,30 +90,35 @@ class HTTP2Protocol(object): self.current_stream_id += 2 return self.current_stream_id - def send_frame(self, frame): - raw_bytes = frame.to_bytes() + def send_frame(self, frm, hide=False): + raw_bytes = frm.to_bytes() self.tcp_handler.wfile.write(raw_bytes) self.tcp_handler.wfile.flush() + if not hide and self.tcp_handler.http2_framedump: + print(frm.human_readable(">>")) - def read_frame(self): + def read_frame(self, hide=False): frm = frame.Frame.from_file(self.tcp_handler.rfile, self) - if isinstance(frm, frame.SettingsFrame): - self._apply_settings(frm.settings) + if not hide and self.tcp_handler.http2_framedump: + print(frm.human_readable("<<")) + if isinstance(frm, frame.SettingsFrame) and not frm.flags & frame.Frame.FLAG_ACK: + self._apply_settings(frm.settings, hide) return frm - def _apply_settings(self, settings): + def _apply_settings(self, settings, hide=False): for setting, value in settings.items(): old_value = self.http2_settings[setting] if not old_value: old_value = '-' - self.http2_settings[setting] = value self.send_frame( frame.SettingsFrame( state=self, - flags=frame.Frame.FLAG_ACK)) + flags=frame.Frame.FLAG_ACK), + hide) + self._read_settings_ack(hide) def _create_headers(self, headers, stream_id, end_stream=True): # TODO: implement max frame size checks and sending in chunks @@ -127,12 +129,16 @@ class HTTP2Protocol(object): header_block_fragment = self.encoder.encode(headers) - bytes = frame.HeadersFrame( + frm = frame.HeadersFrame( state=self, flags=flags, stream_id=stream_id, - header_block_fragment=header_block_fragment).to_bytes() - return [bytes] + header_block_fragment=header_block_fragment) + + if self.tcp_handler.http2_framedump: + print(frm.human_readable(">>")) + + return [frm.to_bytes()] def _create_body(self, body, stream_id): if body is None or len(body) == 0: @@ -141,12 +147,17 @@ class HTTP2Protocol(object): # TODO: implement max frame size checks and sending in chunks # TODO: implement flow-control window - bytes = frame.DataFrame( + frm = frame.DataFrame( state=self, flags=frame.Frame.FLAG_END_STREAM, stream_id=stream_id, - payload=body).to_bytes() - return [bytes] + payload=body) + + if self.tcp_handler.http2_framedump: + print(frm.human_readable(">>")) + + return [frm.to_bytes()] + def create_request(self, method, path, headers=None, body=None): if headers is None: -- cgit v1.2.3 From 79ff43993018209a76a2a7cff995e912eb20d4c3 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 09:47:43 +0200 Subject: add elliptic curve during TLS handshake --- netlib/tcp.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) diff --git a/netlib/tcp.py b/netlib/tcp.py index 953cef6e..2e847d83 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -22,11 +22,6 @@ TLSv1_METHOD = SSL.TLSv1_METHOD TLSv1_1_METHOD = SSL.TLSv1_1_METHOD TLSv1_2_METHOD = SSL.TLSv1_2_METHOD -OP_NO_SSLv2 = SSL.OP_NO_SSLv2 -OP_NO_SSLv3 = SSL.OP_NO_SSLv3 -VERIFY_NONE = SSL.VERIFY_NONE - - class NetLibError(Exception): pass @@ -374,8 +369,8 @@ class _Connection(object): def _create_ssl_context(self, method=SSLv23_METHOD, - options=(OP_NO_SSLv2 | OP_NO_SSLv3), - verify_options=VERIFY_NONE, + options=(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3 | SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_COMPRESSION), + verify_options=SSL.VERIFY_NONE, ca_path=None, ca_pemfile=None, cipher_list=None, @@ -397,7 +392,7 @@ class _Connection(object): context.set_options(options) # Verify Options (NONE/PEER/PEER|FAIL_IF_... and trusted CAs) - if verify_options is not None and verify_options is not VERIFY_NONE: + if verify_options is not None and verify_options is not SSL.VERIFY_NONE: def verify_cert(conn, cert, errno, err_depth, is_cert_verified): if is_cert_verified: return True @@ -426,6 +421,8 @@ class _Connection(object): if log_ssl_key: context.set_info_callback(log_ssl_key) + context.set_tmp_ecdh(OpenSSL.crypto.get_elliptic_curve('prime256v1')) + if OpenSSL._util.lib.Cryptography_HAS_ALPN: if alpn_protos is not None: # advertise application layer protocols -- cgit v1.2.3 From e3db241a2fa47a38fcb85532ed52eeecf1a7b965 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 13:43:23 +0200 Subject: http2: improve frame output --- netlib/http2/protocol.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 7bf68602..24fcb712 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -31,7 +31,7 @@ class HTTP2Protocol(object): ALPN_PROTO_H2 = 'h2' - def __init__(self, tcp_handler, is_server=False): + def __init__(self, tcp_handler, is_server=False, dump_frames=False): self.tcp_handler = tcp_handler self.is_server = is_server @@ -40,6 +40,7 @@ class HTTP2Protocol(object): self.encoder = Encoder() self.decoder = Decoder() self.connection_preface_performed = False + self.dump_frames = dump_frames def check_alpn(self): alp = self.tcp_handler.get_alpn_proto_negotiated() @@ -94,12 +95,12 @@ class HTTP2Protocol(object): raw_bytes = frm.to_bytes() self.tcp_handler.wfile.write(raw_bytes) self.tcp_handler.wfile.flush() - if not hide and self.tcp_handler.http2_framedump: + if not hide and self.dump_frames: print(frm.human_readable(">>")) def read_frame(self, hide=False): frm = frame.Frame.from_file(self.tcp_handler.rfile, self) - if not hide and self.tcp_handler.http2_framedump: + if not hide and self.dump_frames: print(frm.human_readable("<<")) if isinstance(frm, frame.SettingsFrame) and not frm.flags & frame.Frame.FLAG_ACK: self._apply_settings(frm.settings, hide) @@ -135,7 +136,7 @@ class HTTP2Protocol(object): stream_id=stream_id, header_block_fragment=header_block_fragment) - if self.tcp_handler.http2_framedump: + if self.dump_frames: print(frm.human_readable(">>")) return [frm.to_bytes()] @@ -153,7 +154,7 @@ class HTTP2Protocol(object): stream_id=stream_id, payload=body) - if self.tcp_handler.http2_framedump: + if self.dump_frames: print(frm.human_readable(">>")) return [frm.to_bytes()] -- cgit v1.2.3 From d0a9d3cdda6d1f784a23ea4bd9efd3134e292628 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 14:21:34 +0200 Subject: http2: only first headers frame as END_STREAM flag --- netlib/http2/protocol.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 24fcb712..682b7863 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -196,9 +196,9 @@ class HTTP2Protocol(object): if isinstance(frm, frame.HeadersFrame)\ or isinstance(frm, frame.ContinuationFrame): header_block_fragment += frm.header_block_fragment + if frm.flags & frame.Frame.FLAG_END_STREAM: + body_expected = False if frm.flags & frame.Frame.FLAG_END_HEADERS: - if frm.flags & frame.Frame.FLAG_END_STREAM: - body_expected = False break while body_expected: -- cgit v1.2.3 From 1c124421e34d310c6e0577f20b595413d639a5c3 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 15:31:58 +0200 Subject: http2: fix header_block_fragments and length --- netlib/http2/frame.py | 13 +++++++++++-- netlib/http2/protocol.py | 23 +++++++++++++++-------- 2 files changed, 26 insertions(+), 10 deletions(-) diff --git a/netlib/http2/frame.py b/netlib/http2/frame.py index 3e285cba..98ced904 100644 --- a/netlib/http2/frame.py +++ b/netlib/http2/frame.py @@ -114,6 +114,8 @@ class Frame(object): raise NotImplementedError() def human_readable(self, direction="-"): + self.length = len(self.payload_bytes()) + return "\n".join([ "%s: %s | length: %d | flags: %#x | stream_id: %d" % (direction, self.__class__.__name__, self.length, self.flags, self.stream_id), self.payload_human_readable(), @@ -456,7 +458,10 @@ class PushPromiseFrame(Frame): s.append("padding: %d" % self.pad_length) s.append("promised stream: %#x" % self.promised_stream) - s.append("header_block_fragment: %s" % str(self.header_block_fragment)) + s.append( + "header_block_fragment: %s" % + self.header_block_fragment.encode('hex')) + return "\n".join(s) @@ -600,7 +605,11 @@ class ContinuationFrame(Frame): return self.header_block_fragment def payload_human_readable(self): - return "header_block_fragment: %s" % str(self.header_block_fragment) + s = [] + s.append( + "header_block_fragment: %s" % + self.header_block_fragment.encode('hex')) + return "\n".join(s) _FRAME_CLASSES = [ DataFrame, diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 682b7863..f17f998f 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -50,14 +50,18 @@ class HTTP2Protocol(object): return True def _receive_settings(self, hide=False): - frm = self.read_frame(hide) - assert isinstance(frm, frame.SettingsFrame) + while True: + frm = self.read_frame(hide) + if isinstance(frm, frame.SettingsFrame): + break def _read_settings_ack(self, hide=False): - settings_ack_frame = self.read_frame(hide) - assert isinstance(settings_ack_frame, frame.SettingsFrame) - assert settings_ack_frame.flags & frame.Frame.FLAG_ACK - assert len(settings_ack_frame.settings) == 0 + while True: + frm = self.read_frame(hide) + if isinstance(frm, frame.SettingsFrame): + assert settings_ack_frame.flags & frame.Frame.FLAG_ACK + assert len(settings_ack_frame.settings) == 0 + break def perform_server_connection_preface(self, force=False): if force or not self.connection_preface_performed: @@ -119,7 +123,7 @@ class HTTP2Protocol(object): state=self, flags=frame.Frame.FLAG_ACK), hide) - self._read_settings_ack(hide) + # self._read_settings_ack(hide) def _create_headers(self, headers, stream_id, end_stream=True): # TODO: implement max frame size checks and sending in chunks @@ -219,10 +223,13 @@ class HTTP2Protocol(object): if headers is None: headers = [] + body='foobar' + headers = [(b':status', bytes(str(code)))] + headers stream_id = self.next_stream_id() return list(itertools.chain( self._create_headers(headers, stream_id, end_stream=(body is None)), - self._create_body(body, stream_id))) + self._create_body(body, stream_id), + )) -- cgit v1.2.3 From 20c136e070cee0e93e870bf32199cb36b1b85275 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 15:51:40 +0200 Subject: http2: return stream_id from request for response --- netlib/http2/protocol.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index f17f998f..a77edd9b 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -183,7 +183,7 @@ class HTTP2Protocol(object): self._create_body(body, stream_id))) def read_response(self): - headers, body = self._receive_transmission() + stream_id, headers, body = self._receive_transmission() return headers[':status'], headers, body def read_request(self): @@ -192,6 +192,7 @@ class HTTP2Protocol(object): def _receive_transmission(self): body_expected = True + stream_id = 0 header_block_fragment = b'' body = b'' @@ -199,6 +200,7 @@ class HTTP2Protocol(object): frm = self.read_frame() if isinstance(frm, frame.HeadersFrame)\ or isinstance(frm, frame.ContinuationFrame): + stream_id = frm.stream_id header_block_fragment += frm.header_block_fragment if frm.flags & frame.Frame.FLAG_END_STREAM: body_expected = False @@ -217,9 +219,9 @@ class HTTP2Protocol(object): for header, value in self.decoder.decode(header_block_fragment): headers[header] = value - return headers, body + return stream_id, headers, body - def create_response(self, code, headers=None, body=None): + def create_response(self, code, stream_id=None, headers=None, body=None): if headers is None: headers = [] @@ -227,7 +229,8 @@ class HTTP2Protocol(object): headers = [(b':status', bytes(str(code)))] + headers - stream_id = self.next_stream_id() + if not stream_id: + stream_id = self.next_stream_id() return list(itertools.chain( self._create_headers(headers, stream_id, end_stream=(body is None)), -- cgit v1.2.3 From abb37a3ef52ab9a0f68dc46e4a8ca165e365139b Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 17:31:08 +0200 Subject: http2: improve test suite --- netlib/http2/protocol.py | 16 +++++++-------- netlib/tcp.py | 9 +++++---- test/http2/test_http2_protocol.py | 13 +++++++------ test/test_tcp.py | 41 +++++++++++++++++++++++++++++++-------- 4 files changed, 53 insertions(+), 26 deletions(-) diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index a77edd9b..8191090c 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -55,7 +55,7 @@ class HTTP2Protocol(object): if isinstance(frm, frame.SettingsFrame): break - def _read_settings_ack(self, hide=False): + def _read_settings_ack(self, hide=False): # pragma no cover while True: frm = self.read_frame(hide) if isinstance(frm, frame.SettingsFrame): @@ -99,12 +99,12 @@ class HTTP2Protocol(object): raw_bytes = frm.to_bytes() self.tcp_handler.wfile.write(raw_bytes) self.tcp_handler.wfile.flush() - if not hide and self.dump_frames: + if not hide and self.dump_frames: # pragma no cover print(frm.human_readable(">>")) def read_frame(self, hide=False): frm = frame.Frame.from_file(self.tcp_handler.rfile, self) - if not hide and self.dump_frames: + if not hide and self.dump_frames: # pragma no cover print(frm.human_readable("<<")) if isinstance(frm, frame.SettingsFrame) and not frm.flags & frame.Frame.FLAG_ACK: self._apply_settings(frm.settings, hide) @@ -123,7 +123,9 @@ class HTTP2Protocol(object): state=self, flags=frame.Frame.FLAG_ACK), hide) - # self._read_settings_ack(hide) + + # be liberal in what we expect from the other end + # to be more strict use: self._read_settings_ack(hide) def _create_headers(self, headers, stream_id, end_stream=True): # TODO: implement max frame size checks and sending in chunks @@ -140,7 +142,7 @@ class HTTP2Protocol(object): stream_id=stream_id, header_block_fragment=header_block_fragment) - if self.dump_frames: + if self.dump_frames: # pragma no cover print(frm.human_readable(">>")) return [frm.to_bytes()] @@ -158,7 +160,7 @@ class HTTP2Protocol(object): stream_id=stream_id, payload=body) - if self.dump_frames: + if self.dump_frames: # pragma no cover print(frm.human_readable(">>")) return [frm.to_bytes()] @@ -225,8 +227,6 @@ class HTTP2Protocol(object): if headers is None: headers = [] - body='foobar' - headers = [(b':status', bytes(str(code)))] + headers if not stream_id: diff --git a/netlib/tcp.py b/netlib/tcp.py index 2e847d83..cafc3ed9 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -414,6 +414,9 @@ class _Connection(object): if cipher_list: try: context.set_cipher_list(cipher_list) + + # TODO: maybe change this to with newer pyOpenSSL APIs + context.set_tmp_ecdh(OpenSSL.crypto.get_elliptic_curve('prime256v1')) except SSL.Error as v: raise NetLibError("SSL cipher specification error: %s" % str(v)) @@ -421,8 +424,6 @@ class _Connection(object): if log_ssl_key: context.set_info_callback(log_ssl_key) - context.set_tmp_ecdh(OpenSSL.crypto.get_elliptic_curve('prime256v1')) - if OpenSSL._util.lib.Cryptography_HAS_ALPN: if alpn_protos is not None: # advertise application layer protocols @@ -526,7 +527,7 @@ class TCPClient(_Connection): if OpenSSL._util.lib.Cryptography_HAS_ALPN and self.ssl_established: return self.connection.get_alpn_proto_negotiated() else: - return None + return "" class BaseHandler(_Connection): @@ -636,7 +637,7 @@ class BaseHandler(_Connection): if OpenSSL._util.lib.Cryptography_HAS_ALPN and self.ssl_established: return self.connection.get_alpn_proto_negotiated() else: - return None + return "" class TCPServer(object): diff --git a/test/http2/test_http2_protocol.py b/test/http2/test_http2_protocol.py index 34c69fa9..231b35e0 100644 --- a/test/http2/test_http2_protocol.py +++ b/test/http2/test_http2_protocol.py @@ -300,8 +300,9 @@ class TestReadRequest(test.ServerTestBase): c.convert_to_ssl() protocol = http2.HTTP2Protocol(c, is_server=True) - headers, body = protocol.read_request() + stream_id, headers, body = protocol.read_request() + assert stream_id assert headers == {':method': 'GET', ':path': '/', ':scheme': 'https'} assert body == b'foobar' @@ -309,17 +310,17 @@ class TestReadRequest(test.ServerTestBase): class TestCreateResponse(): c = tcp.TCPClient(("127.0.0.1", 0)) - def test_create_request_simple(self): + def test_create_response_simple(self): bytes = http2.HTTP2Protocol(self.c, is_server=True).create_response(200) assert len(bytes) == 1 assert bytes[0] ==\ '00000101050000000288'.decode('hex') - def test_create_request_with_body(self): + def test_create_response_with_body(self): bytes = http2.HTTP2Protocol(self.c, is_server=True).create_response( - 200, [(b'foo', b'bar')], 'foobar') + 200, 1, [(b'foo', b'bar')], 'foobar') assert len(bytes) == 2 assert bytes[0] ==\ - '00000901040000000288408294e7838c767f'.decode('hex') + '00000901040000000188408294e7838c767f'.decode('hex') assert bytes[1] ==\ - '000006000100000002666f6f626172'.decode('hex') + '000006000100000001666f6f626172'.decode('hex') diff --git a/test/test_tcp.py b/test/test_tcp.py index 0cecaaa2..122c1f0f 100644 --- a/test/test_tcp.py +++ b/test/test_tcp.py @@ -41,6 +41,18 @@ class HangHandler(tcp.BaseHandler): time.sleep(1) +class ALPNHandler(tcp.BaseHandler): + sni = None + + def handle(self): + alp = self.get_alpn_proto_negotiated() + if alp: + self.wfile.write("%s" % alp) + else: + self.wfile.write("NONE") + self.wfile.flush() + + class TestServer(test.ServerTestBase): handler = EchoHandler @@ -416,30 +428,43 @@ class TestTimeOut(test.ServerTestBase): tutils.raises(tcp.NetLibTimeout, c.rfile.read, 10) -class TestALPN(test.ServerTestBase): - handler = EchoHandler +class TestALPNClient(test.ServerTestBase): + handler = ALPNHandler ssl = dict( - alpn_select="foobar" + alpn_select="bar" ) if OpenSSL._util.lib.Cryptography_HAS_ALPN: def test_alpn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) c.connect() - c.convert_to_ssl(alpn_protos=["foobar"]) - assert c.get_alpn_proto_negotiated() == "foobar" + c.convert_to_ssl(alpn_protos=["foo", "bar", "fasel"]) + assert c.get_alpn_proto_negotiated() == "bar" + assert c.rfile.readline().strip() == "bar" def test_no_alpn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) c.connect() - assert c.get_alpn_proto_negotiated() == None + c.convert_to_ssl() + assert c.get_alpn_proto_negotiated() == "" + assert c.rfile.readline().strip() == "NONE" else: def test_none_alpn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) c.connect() - c.convert_to_ssl(alpn_protos=["foobar"]) - assert c.get_alpn_proto_negotiated() == None + c.convert_to_ssl(alpn_protos=["foo", "bar", "fasel"]) + assert c.get_alpn_proto_negotiated() == "" + assert c.rfile.readline() == "NONE" + +class TestNoSSLNoALPNClient(test.ServerTestBase): + handler = ALPNHandler + + def test_no_ssl_no_alpn(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + assert c.get_alpn_proto_negotiated() == "" + assert c.rfile.readline().strip() == "NONE" class TestSSLTimeOut(test.ServerTestBase): -- cgit v1.2.3 From eb823a04a19de7fd9e15d225064ae4581f0b85bf Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 23:36:14 +0200 Subject: http2: improve :authority header --- netlib/http2/protocol.py | 3 +++ test/http2/test_http2_protocol.py | 4 ++-- 2 files changed, 5 insertions(+), 2 deletions(-) diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 8191090c..ac89bac4 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -171,6 +171,9 @@ class HTTP2Protocol(object): headers = [] authority = self.tcp_handler.sni if self.tcp_handler.sni else self.tcp_handler.address.host + if self.tcp_handler.address.port != 443: + authority += ":%d" % self.tcp_handler.address.port + headers = [ (b':method', bytes(method)), (b':path', bytes(path)), diff --git a/test/http2/test_http2_protocol.py b/test/http2/test_http2_protocol.py index 231b35e0..9b49acd3 100644 --- a/test/http2/test_http2_protocol.py +++ b/test/http2/test_http2_protocol.py @@ -222,14 +222,14 @@ class TestCreateRequest(): def test_create_request_simple(self): bytes = http2.HTTP2Protocol(self.c).create_request('GET', '/') assert len(bytes) == 1 - assert bytes[0] == '00000c0105000000018284874187089d5c0b8170ff'.decode('hex') + assert bytes[0] == '00000d0105000000018284874188089d5c0b8170dc07'.decode('hex') def test_create_request_with_body(self): bytes = http2.HTTP2Protocol(self.c).create_request( 'GET', '/', [(b'foo', b'bar')], 'foobar') assert len(bytes) == 2 assert bytes[0] ==\ - '0000140104000000018284874187089d5c0b8170ff408294e7838c767f'.decode('hex') + '0000150104000000018284874188089d5c0b8170dc07408294e7838c767f'.decode('hex') assert bytes[1] ==\ '000006000100000001666f6f626172'.decode('hex') -- cgit v1.2.3 From c9c93af453ec332b660f70402b78ae8f269280f0 Mon Sep 17 00:00:00 2001 From: Kyle Morton Date: Tue, 16 Jun 2015 11:11:10 -0700 Subject: Adding certifi as default CA bundle. --- netlib/tcp.py | 6 +++--- setup.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/netlib/tcp.py b/netlib/tcp.py index ca948514..b523bea4 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -7,6 +7,7 @@ import threading import time import traceback +import certifi import OpenSSL from OpenSSL import SSL @@ -373,7 +374,7 @@ class _Connection(object): method=SSLv23_METHOD, options=(OP_NO_SSLv2 | OP_NO_SSLv3), verify_options=VERIFY_NONE, - ca_path=None, + ca_path=certifi.where(), ca_pemfile=None, cipher_list=None, alpn_protos=None, @@ -403,8 +404,7 @@ class _Connection(object): (err_depth, errno)) context.set_verify(verify_options, verify_cert) - if ca_path is not None or ca_pemfile is not None: - context.load_verify_locations(ca_pemfile, ca_path) + context.load_verify_locations(ca_pemfile, ca_path) # Workaround for # https://github.com/pyca/pyopenssl/issues/190 diff --git a/setup.py b/setup.py index 0051ea77..aa27cd90 100644 --- a/setup.py +++ b/setup.py @@ -66,7 +66,8 @@ setup( "pyOpenSSL>=0.15.1", "cryptography>=0.9", "passlib>=1.6.2", - "hpack>=1.0.1"], + "hpack>=1.0.1", + "certifi"], setup_requires=[ "cffi", "pyOpenSSL>=0.15.1", -- cgit v1.2.3 From ff20e64537ad25aa988f212b0473bdb5e696611b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 16 Jun 2015 02:37:46 +0200 Subject: add landscape configuration --- .landscape.yml | 3 +++ 1 file changed, 3 insertions(+) create mode 100644 .landscape.yml diff --git a/.landscape.yml b/.landscape.yml new file mode 100644 index 00000000..680ee0e7 --- /dev/null +++ b/.landscape.yml @@ -0,0 +1,3 @@ +pylint: + disable: + - unpacking-non-sequence \ No newline at end of file -- cgit v1.2.3 From 836b1eab9700230991822102d411aed067308123 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 17 Jun 2015 13:10:27 +0200 Subject: fix warnings and code smells use prospector to find them --- .landscape.yml | 12 ++++++++++- netlib/http2/__init__.py | 1 - netlib/http2/frame.py | 55 ++++++++++++++++++++++++------------------------ netlib/http_cookies.py | 8 +++---- netlib/http_uastrings.py | 24 +++++++++++---------- netlib/tcp.py | 8 +++---- netlib/utils.py | 2 +- netlib/websockets.py | 16 +++++++------- setup.cfg | 8 ------- 9 files changed, 67 insertions(+), 67 deletions(-) delete mode 100644 setup.cfg diff --git a/.landscape.yml b/.landscape.yml index 680ee0e7..5926e7bf 100644 --- a/.landscape.yml +++ b/.landscape.yml @@ -1,3 +1,13 @@ +max-line-length: 120 pylint: disable: - - unpacking-non-sequence \ No newline at end of file + - missing-docstring + - protected-access + - too-few-public-methods + - too-many-arguments + - too-many-instance-attributes + - too-many-locals + - too-many-public-methods + - too-many-return-statements + - too-many-statements + - unpacking-non-sequence diff --git a/netlib/http2/__init__.py b/netlib/http2/__init__.py index 92897b5d..5acf7696 100644 --- a/netlib/http2/__init__.py +++ b/netlib/http2/__init__.py @@ -1,3 +1,2 @@ - from frame import * from protocol import * diff --git a/netlib/http2/frame.py b/netlib/http2/frame.py index 4a305d82..43676623 100644 --- a/netlib/http2/frame.py +++ b/netlib/http2/frame.py @@ -1,6 +1,5 @@ import sys import struct -from functools import reduce from hpack.hpack import Encoder, Decoder from .. import utils @@ -52,7 +51,7 @@ class Frame(object): self.stream_id = stream_id @classmethod - def _check_frame_size(self, length, state): + def _check_frame_size(cls, length, state): if state: settings = state.http2_settings else: @@ -67,7 +66,7 @@ class Frame(object): length, max_frame_size)) @classmethod - def from_file(self, fp, state=None): + def from_file(cls, fp, state=None): """ read a HTTP/2 frame sent by a server or client fp is a "file like" object that could be backed by a network @@ -83,7 +82,7 @@ class Frame(object): if raw_header[:4] == b'HTTP': # pragma no cover print >> sys.stderr, "WARNING: This looks like an HTTP/1 connection!" - self._check_frame_size(length, state) + cls._check_frame_size(length, state) payload = fp.safe_read(length) return FRAMES[fields[2]].from_bytes( @@ -146,10 +145,10 @@ class DataFrame(Frame): self.pad_length = pad_length @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) - if f.flags & self.FLAG_PADDED: + if f.flags & Frame.FLAG_PADDED: f.pad_length = struct.unpack('!B', payload[0])[0] f.payload = payload[1:-f.pad_length] else: @@ -204,16 +203,16 @@ class HeadersFrame(Frame): self.weight = weight @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) - if f.flags & self.FLAG_PADDED: + if f.flags & Frame.FLAG_PADDED: f.pad_length = struct.unpack('!B', payload[0])[0] f.header_block_fragment = payload[1:-f.pad_length] else: f.header_block_fragment = payload[0:] - if f.flags & self.FLAG_PRIORITY: + if f.flags & Frame.FLAG_PRIORITY: f.stream_dependency, f.weight = struct.unpack( '!LB', f.header_block_fragment[:5]) f.exclusive = bool(f.stream_dependency >> 31) @@ -279,8 +278,8 @@ class PriorityFrame(Frame): self.weight = weight @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) f.stream_dependency, f.weight = struct.unpack('!LB', payload) f.exclusive = bool(f.stream_dependency >> 31) @@ -325,8 +324,8 @@ class RstStreamFrame(Frame): self.error_code = error_code @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) f.error_code = struct.unpack('!L', payload)[0] return f @@ -369,8 +368,8 @@ class SettingsFrame(Frame): self.settings = settings @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) for i in xrange(0, len(payload), 6): identifier, value = struct.unpack("!HL", payload[i:i + 6]) @@ -420,10 +419,10 @@ class PushPromiseFrame(Frame): self.header_block_fragment = header_block_fragment @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) - if f.flags & self.FLAG_PADDED: + if f.flags & Frame.FLAG_PADDED: f.pad_length, f.promised_stream = struct.unpack('!BL', payload[:5]) f.header_block_fragment = payload[5:-f.pad_length] else: @@ -480,8 +479,8 @@ class PingFrame(Frame): self.payload = payload @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) f.payload = payload return f @@ -517,8 +516,8 @@ class GoAwayFrame(Frame): self.data = data @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) f.last_stream, f.error_code = struct.unpack("!LL", payload[:8]) f.last_stream &= 0x7FFFFFFF @@ -558,8 +557,8 @@ class WindowUpdateFrame(Frame): self.window_size_increment = window_size_increment @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) f.window_size_increment = struct.unpack("!L", payload)[0] f.window_size_increment &= 0x7FFFFFFF @@ -592,8 +591,8 @@ class ContinuationFrame(Frame): self.header_block_fragment = header_block_fragment @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) f.header_block_fragment = payload return f diff --git a/netlib/http_cookies.py b/netlib/http_cookies.py index 5cb39e5c..b7311714 100644 --- a/netlib/http_cookies.py +++ b/netlib/http_cookies.py @@ -158,7 +158,7 @@ def _parse_set_cookie_pairs(s): return pairs -def parse_set_cookie_header(str): +def parse_set_cookie_header(line): """ Parse a Set-Cookie header value @@ -166,7 +166,7 @@ def parse_set_cookie_header(str): ODictCaseless set of attributes. No attempt is made to parse attribute values - they are treated purely as strings. """ - pairs = _parse_set_cookie_pairs(str) + pairs = _parse_set_cookie_pairs(line) if pairs: return pairs[0][0], pairs[0][1], odict.ODictCaseless(pairs[1:]) @@ -180,12 +180,12 @@ def format_set_cookie_header(name, value, attrs): return _format_set_cookie_pairs(pairs) -def parse_cookie_header(str): +def parse_cookie_header(line): """ Parse a Cookie header value. Returns a (possibly empty) ODict object. """ - pairs, off = _read_pairs(str) + pairs, off = _read_pairs(line) return odict.ODict(pairs) diff --git a/netlib/http_uastrings.py b/netlib/http_uastrings.py index d9869531..c1ef557c 100644 --- a/netlib/http_uastrings.py +++ b/netlib/http_uastrings.py @@ -5,40 +5,42 @@ from __future__ import (absolute_import, print_function, division) kept reasonably current to reflect common usage. """ +# pylint: line-too-long + # A collection of (name, shortcut, string) tuples. UASTRINGS = [ ("android", "a", - "Mozilla/5.0 (Linux; U; Android 4.1.1; en-gb; Nexus 7 Build/JRO03D) AFL/01.04.02"), + "Mozilla/5.0 (Linux; U; Android 4.1.1; en-gb; Nexus 7 Build/JRO03D) AFL/01.04.02"), # noqa ("blackberry", "l", - "Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.346 Mobile Safari/534.11+"), + "Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.346 Mobile Safari/534.11+"), # noqa ("bingbot", "b", - "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"), + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"), # noqa ("chrome", "c", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"), + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"), # noqa ("firefox", "f", - "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:14.0) Gecko/20120405 Firefox/14.0a1"), + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:14.0) Gecko/20120405 Firefox/14.0a1"), # noqa ("googlebot", "g", - "Googlebot/2.1 (+http://www.googlebot.com/bot.html)"), + "Googlebot/2.1 (+http://www.googlebot.com/bot.html)"), # noqa ("ie9", "i", - "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))"), + "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))"), # noqa ("ipad", "p", - "Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3"), + "Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3"), # noqa ("iphone", "h", - "Mozilla/5.0 (iPhone; CPU iPhone OS 4_2_1 like Mac OS X) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148a Safari/6533.18.5", - ), + "Mozilla/5.0 (iPhone; CPU iPhone OS 4_2_1 like Mac OS X) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148a Safari/6533.18.5"), # noqa ("safari", "s", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10")] + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10"), # noqa +] def get_by_shortcut(s): diff --git a/netlib/tcp.py b/netlib/tcp.py index 953cef6e..807015c8 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -297,7 +297,7 @@ def close_socket(sock): """ try: # We already indicate that we close our end. - # may raise "Transport endpoint is not connected" on Linux + # may raise "Transport endpoint is not connected" on Linux sock.shutdown(socket.SHUT_WR) # Section 4.2.2.13 of RFC 1122 tells us that a close() with any pending @@ -368,10 +368,6 @@ class _Connection(object): except SSL.Error: pass - """ - Creates an SSL Context. - """ - def _create_ssl_context(self, method=SSLv23_METHOD, options=(OP_NO_SSLv2 | OP_NO_SSLv3), @@ -383,6 +379,8 @@ class _Connection(object): alpn_select=None, ): """ + Creates an SSL Context. + :param method: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, TLSv1_METHOD, TLSv1_1_METHOD, or TLSv1_2_METHOD :param options: A bit field consisting of OpenSSL.SSL.OP_* values :param verify_options: A bit field consisting of OpenSSL.SSL.VERIFY_* values diff --git a/netlib/utils.py b/netlib/utils.py index 9c5404e6..ac42bd53 100644 --- a/netlib/utils.py +++ b/netlib/utils.py @@ -67,7 +67,7 @@ def getbit(byte, offset): return True -class BiDi: +class BiDi(object): """ A wee utility class for keeping bi-directional mappings, like field diff --git a/netlib/websockets.py b/netlib/websockets.py index 346adf1b..c45db4df 100644 --- a/netlib/websockets.py +++ b/netlib/websockets.py @@ -35,7 +35,7 @@ OPCODE = utils.BiDi( ) -class Masker: +class Masker(object): """ Data sent from the server must be masked to prevent malicious clients @@ -94,15 +94,15 @@ def server_handshake_headers(key): ) -def make_length_code(len): +def make_length_code(length): """ A websockets frame contains an initial length_code, and an optional extended length code to represent the actual length if length code is larger than 125 """ - if len <= 125: - return len - elif len >= 126 and len <= 65535: + if length <= 125: + return length + elif length >= 126 and length <= 65535: return 126 else: return 127 @@ -129,7 +129,7 @@ def create_server_nonce(client_nonce): DEFAULT = object() -class FrameHeader: +class FrameHeader(object): def __init__( self, @@ -216,7 +216,7 @@ class FrameHeader: return b @classmethod - def from_file(klass, fp): + def from_file(cls, fp): """ read a websockets frame header """ @@ -248,7 +248,7 @@ class FrameHeader: else: masking_key = None - return klass( + return cls( fin=fin, rsv1=rsv1, rsv2=rsv2, diff --git a/setup.cfg b/setup.cfg deleted file mode 100644 index 4207020e..00000000 --- a/setup.cfg +++ /dev/null @@ -1,8 +0,0 @@ -[flake8] -max-line-length = 80 -max-complexity = 15 - -[pep8] -max-line-length = 80 -exclude = */contrib/* -ignore = E251,E309 -- cgit v1.2.3 From a652e050b759ca27aa3b794b8e11009853edef34 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 17 Jun 2015 13:19:44 +0200 Subject: add landscape.io badge --- README.mkd | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/README.mkd b/README.mkd index 7039e203..2f87d111 100644 --- a/README.mkd +++ b/README.mkd @@ -1,4 +1,5 @@ -[![Build Status](https://img.shields.io/travis/mitmproxy/netlib/master.svg)](https://travis-ci.org/mitmproxy/netlib) +[![Build Status](https://img.shields.io/travis/mitmproxy/netlib/master.svg)](https://travis-ci.org/mitmproxy/netlib) +[![Code Health](https://landscape.io/github/mitmproxy/netlib/master/landscape.svg?style=flat)](https://landscape.io/github/mitmproxy/netlib/master) [![Coverage Status](https://img.shields.io/coveralls/mitmproxy/netlib/master.svg)](https://coveralls.io/r/mitmproxy/netlib) [![Downloads](https://img.shields.io/pypi/dm/netlib.svg?color=orange)](https://pypi.python.org/pypi/netlib) [![Latest Version](https://img.shields.io/pypi/v/netlib.svg)](https://pypi.python.org/pypi/netlib) -- cgit v1.2.3 From 6e301f37d0597d86008c440f62526f906f0ae9f4 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 18 Jun 2015 12:18:22 +1200 Subject: Only set OP_NO_COMPRESSION by default if it exists in our version of OpenSSL We'll need to start testing under both new and old versions of OpenSSL somehow to catch these... --- netlib/tcp.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) diff --git a/netlib/tcp.py b/netlib/tcp.py index a1d1fe62..52ebc3c0 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -22,6 +22,17 @@ TLSv1_METHOD = SSL.TLSv1_METHOD TLSv1_1_METHOD = SSL.TLSv1_1_METHOD TLSv1_2_METHOD = SSL.TLSv1_2_METHOD + +SSL_DEFAULT_OPTIONS = ( + SSL.OP_NO_SSLv2 | + SSL.OP_NO_SSLv3 | + SSL.OP_CIPHER_SERVER_PREFERENCE +) + +if hasattr(SSL, "OP_NO_COMPRESSION"): + SSL_DEFAULT_OPTIONS |= SSL.OP_NO_COMPRESSION + + class NetLibError(Exception): pass @@ -365,7 +376,7 @@ class _Connection(object): def _create_ssl_context(self, method=SSLv23_METHOD, - options=(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3 | SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_COMPRESSION), + options=SSL_DEFAULT_OPTIONS, verify_options=SSL.VERIFY_NONE, ca_path=None, ca_pemfile=None, -- cgit v1.2.3 From 61cbe36e4016d77b93386e3df9b17b36b1633d7e Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 18 Jun 2015 10:38:26 +0200 Subject: http2: rename test file --- test/http2/test_http2_protocol.py | 326 -------------------------------------- test/http2/test_protocol.py | 326 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 326 insertions(+), 326 deletions(-) delete mode 100644 test/http2/test_http2_protocol.py create mode 100644 test/http2/test_protocol.py diff --git a/test/http2/test_http2_protocol.py b/test/http2/test_http2_protocol.py deleted file mode 100644 index 9b49acd3..00000000 --- a/test/http2/test_http2_protocol.py +++ /dev/null @@ -1,326 +0,0 @@ -import OpenSSL - -from netlib import http2 -from netlib import tcp -from netlib import test -from netlib.http2.frame import * -from test import tutils - - -class EchoHandler(tcp.BaseHandler): - sni = None - - def handle(self): - while True: - v = self.rfile.safe_read(1) - self.wfile.write(v) - self.wfile.flush() - - -class TestCheckALPNMatch(test.ServerTestBase): - handler = EchoHandler - ssl = dict( - alpn_select=http2.HTTP2Protocol.ALPN_PROTO_H2, - ) - - if OpenSSL._util.lib.Cryptography_HAS_ALPN: - - def test_check_alpn(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - c.connect() - c.convert_to_ssl(alpn_protos=[http2.HTTP2Protocol.ALPN_PROTO_H2]) - protocol = http2.HTTP2Protocol(c) - assert protocol.check_alpn() - - -class TestCheckALPNMismatch(test.ServerTestBase): - handler = EchoHandler - ssl = dict( - alpn_select=None, - ) - - if OpenSSL._util.lib.Cryptography_HAS_ALPN: - - def test_check_alpn(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - c.connect() - c.convert_to_ssl(alpn_protos=[http2.HTTP2Protocol.ALPN_PROTO_H2]) - protocol = http2.HTTP2Protocol(c) - tutils.raises(NotImplementedError, protocol.check_alpn) - - -class TestPerformServerConnectionPreface(test.ServerTestBase): - class handler(tcp.BaseHandler): - - def handle(self): - # send magic - self.wfile.write( - '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex')) - self.wfile.flush() - - # send empty settings frame - self.wfile.write('000000040000000000'.decode('hex')) - self.wfile.flush() - - # check empty settings frame - assert self.rfile.read(9) ==\ - '000000040000000000'.decode('hex') - - # check settings acknowledgement - assert self.rfile.read(9) == \ - '000000040100000000'.decode('hex') - - # send settings acknowledgement - self.wfile.write('000000040100000000'.decode('hex')) - self.wfile.flush() - - def test_perform_server_connection_preface(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - c.connect() - protocol = http2.HTTP2Protocol(c) - protocol.perform_server_connection_preface() - - -class TestPerformClientConnectionPreface(test.ServerTestBase): - class handler(tcp.BaseHandler): - - def handle(self): - # check magic - assert self.rfile.read(24) ==\ - '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex') - - # check empty settings frame - assert self.rfile.read(9) ==\ - '000000040000000000'.decode('hex') - - # send empty settings frame - self.wfile.write('000000040000000000'.decode('hex')) - self.wfile.flush() - - # check settings acknowledgement - assert self.rfile.read(9) == \ - '000000040100000000'.decode('hex') - - # send settings acknowledgement - self.wfile.write('000000040100000000'.decode('hex')) - self.wfile.flush() - - def test_perform_client_connection_preface(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - c.connect() - protocol = http2.HTTP2Protocol(c) - protocol.perform_client_connection_preface() - - -class TestClientStreamIds(): - c = tcp.TCPClient(("127.0.0.1", 0)) - protocol = http2.HTTP2Protocol(c) - - def test_client_stream_ids(self): - assert self.protocol.current_stream_id is None - assert self.protocol.next_stream_id() == 1 - assert self.protocol.current_stream_id == 1 - assert self.protocol.next_stream_id() == 3 - assert self.protocol.current_stream_id == 3 - assert self.protocol.next_stream_id() == 5 - assert self.protocol.current_stream_id == 5 - - -class TestServerStreamIds(): - c = tcp.TCPClient(("127.0.0.1", 0)) - protocol = http2.HTTP2Protocol(c, is_server=True) - - def test_server_stream_ids(self): - assert self.protocol.current_stream_id is None - assert self.protocol.next_stream_id() == 2 - assert self.protocol.current_stream_id == 2 - assert self.protocol.next_stream_id() == 4 - assert self.protocol.current_stream_id == 4 - assert self.protocol.next_stream_id() == 6 - assert self.protocol.current_stream_id == 6 - - -class TestApplySettings(test.ServerTestBase): - class handler(tcp.BaseHandler): - - def handle(self): - # check settings acknowledgement - assert self.rfile.read(9) == '000000040100000000'.decode('hex') - self.wfile.write("OK") - self.wfile.flush() - - ssl = True - - def test_apply_settings(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - c.connect() - c.convert_to_ssl() - protocol = http2.HTTP2Protocol(c) - - protocol._apply_settings({ - SettingsFrame.SETTINGS.SETTINGS_ENABLE_PUSH: 'foo', - SettingsFrame.SETTINGS.SETTINGS_MAX_CONCURRENT_STREAMS: 'bar', - SettingsFrame.SETTINGS.SETTINGS_INITIAL_WINDOW_SIZE: 'deadbeef', - }) - - assert c.rfile.safe_read(2) == "OK" - - assert protocol.http2_settings[ - SettingsFrame.SETTINGS.SETTINGS_ENABLE_PUSH] == 'foo' - assert protocol.http2_settings[ - SettingsFrame.SETTINGS.SETTINGS_MAX_CONCURRENT_STREAMS] == 'bar' - assert protocol.http2_settings[ - SettingsFrame.SETTINGS.SETTINGS_INITIAL_WINDOW_SIZE] == 'deadbeef' - - -class TestCreateHeaders(): - c = tcp.TCPClient(("127.0.0.1", 0)) - - def test_create_headers(self): - headers = [ - (b':method', b'GET'), - (b':path', b'index.html'), - (b':scheme', b'https'), - (b'foo', b'bar')] - - bytes = http2.HTTP2Protocol(self.c)._create_headers( - headers, 1, end_stream=True) - assert b''.join(bytes) ==\ - '000014010500000001824488355217caf3a69a3f87408294e7838c767f'\ - .decode('hex') - - bytes = http2.HTTP2Protocol(self.c)._create_headers( - headers, 1, end_stream=False) - assert b''.join(bytes) ==\ - '000014010400000001824488355217caf3a69a3f87408294e7838c767f'\ - .decode('hex') - - # TODO: add test for too large header_block_fragments - - -class TestCreateBody(): - c = tcp.TCPClient(("127.0.0.1", 0)) - protocol = http2.HTTP2Protocol(c) - - def test_create_body_empty(self): - bytes = self.protocol._create_body(b'', 1) - assert b''.join(bytes) == ''.decode('hex') - - def test_create_body_single_frame(self): - bytes = self.protocol._create_body('foobar', 1) - assert b''.join(bytes) == '000006000100000001666f6f626172'.decode('hex') - - def test_create_body_multiple_frames(self): - pass - # bytes = self.protocol._create_body('foobar' * 3000, 1) - # TODO: add test for too large frames - - -class TestCreateRequest(): - c = tcp.TCPClient(("127.0.0.1", 0)) - - def test_create_request_simple(self): - bytes = http2.HTTP2Protocol(self.c).create_request('GET', '/') - assert len(bytes) == 1 - assert bytes[0] == '00000d0105000000018284874188089d5c0b8170dc07'.decode('hex') - - def test_create_request_with_body(self): - bytes = http2.HTTP2Protocol(self.c).create_request( - 'GET', '/', [(b'foo', b'bar')], 'foobar') - assert len(bytes) == 2 - assert bytes[0] ==\ - '0000150104000000018284874188089d5c0b8170dc07408294e7838c767f'.decode('hex') - assert bytes[1] ==\ - '000006000100000001666f6f626172'.decode('hex') - - -class TestReadResponse(test.ServerTestBase): - class handler(tcp.BaseHandler): - - def handle(self): - self.wfile.write( - b'00000801040000000188628594e78c767f'.decode('hex')) - self.wfile.write( - b'000006000100000001666f6f626172'.decode('hex')) - self.wfile.flush() - - ssl = True - - def test_read_response(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - c.connect() - c.convert_to_ssl() - protocol = http2.HTTP2Protocol(c) - - status, headers, body = protocol.read_response() - - assert headers == {':status': '200', 'etag': 'foobar'} - assert status == "200" - assert body == b'foobar' - - -class TestReadEmptyResponse(test.ServerTestBase): - class handler(tcp.BaseHandler): - - def handle(self): - self.wfile.write( - b'00000801050000000188628594e78c767f'.decode('hex')) - self.wfile.flush() - - ssl = True - - def test_read_empty_response(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - c.connect() - c.convert_to_ssl() - protocol = http2.HTTP2Protocol(c) - - status, headers, body = protocol.read_response() - - assert headers == {':status': '200', 'etag': 'foobar'} - assert status == "200" - assert body == b'' - - -class TestReadRequest(test.ServerTestBase): - class handler(tcp.BaseHandler): - - def handle(self): - self.wfile.write( - b'000003010400000001828487'.decode('hex')) - self.wfile.write( - b'000006000100000001666f6f626172'.decode('hex')) - self.wfile.flush() - - ssl = True - - def test_read_request(self): - c = tcp.TCPClient(("127.0.0.1", self.port)) - c.connect() - c.convert_to_ssl() - protocol = http2.HTTP2Protocol(c, is_server=True) - - stream_id, headers, body = protocol.read_request() - - assert stream_id - assert headers == {':method': 'GET', ':path': '/', ':scheme': 'https'} - assert body == b'foobar' - - -class TestCreateResponse(): - c = tcp.TCPClient(("127.0.0.1", 0)) - - def test_create_response_simple(self): - bytes = http2.HTTP2Protocol(self.c, is_server=True).create_response(200) - assert len(bytes) == 1 - assert bytes[0] ==\ - '00000101050000000288'.decode('hex') - - def test_create_response_with_body(self): - bytes = http2.HTTP2Protocol(self.c, is_server=True).create_response( - 200, 1, [(b'foo', b'bar')], 'foobar') - assert len(bytes) == 2 - assert bytes[0] ==\ - '00000901040000000188408294e7838c767f'.decode('hex') - assert bytes[1] ==\ - '000006000100000001666f6f626172'.decode('hex') diff --git a/test/http2/test_protocol.py b/test/http2/test_protocol.py new file mode 100644 index 00000000..9b49acd3 --- /dev/null +++ b/test/http2/test_protocol.py @@ -0,0 +1,326 @@ +import OpenSSL + +from netlib import http2 +from netlib import tcp +from netlib import test +from netlib.http2.frame import * +from test import tutils + + +class EchoHandler(tcp.BaseHandler): + sni = None + + def handle(self): + while True: + v = self.rfile.safe_read(1) + self.wfile.write(v) + self.wfile.flush() + + +class TestCheckALPNMatch(test.ServerTestBase): + handler = EchoHandler + ssl = dict( + alpn_select=http2.HTTP2Protocol.ALPN_PROTO_H2, + ) + + if OpenSSL._util.lib.Cryptography_HAS_ALPN: + + def test_check_alpn(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + c.convert_to_ssl(alpn_protos=[http2.HTTP2Protocol.ALPN_PROTO_H2]) + protocol = http2.HTTP2Protocol(c) + assert protocol.check_alpn() + + +class TestCheckALPNMismatch(test.ServerTestBase): + handler = EchoHandler + ssl = dict( + alpn_select=None, + ) + + if OpenSSL._util.lib.Cryptography_HAS_ALPN: + + def test_check_alpn(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + c.convert_to_ssl(alpn_protos=[http2.HTTP2Protocol.ALPN_PROTO_H2]) + protocol = http2.HTTP2Protocol(c) + tutils.raises(NotImplementedError, protocol.check_alpn) + + +class TestPerformServerConnectionPreface(test.ServerTestBase): + class handler(tcp.BaseHandler): + + def handle(self): + # send magic + self.wfile.write( + '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex')) + self.wfile.flush() + + # send empty settings frame + self.wfile.write('000000040000000000'.decode('hex')) + self.wfile.flush() + + # check empty settings frame + assert self.rfile.read(9) ==\ + '000000040000000000'.decode('hex') + + # check settings acknowledgement + assert self.rfile.read(9) == \ + '000000040100000000'.decode('hex') + + # send settings acknowledgement + self.wfile.write('000000040100000000'.decode('hex')) + self.wfile.flush() + + def test_perform_server_connection_preface(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + protocol = http2.HTTP2Protocol(c) + protocol.perform_server_connection_preface() + + +class TestPerformClientConnectionPreface(test.ServerTestBase): + class handler(tcp.BaseHandler): + + def handle(self): + # check magic + assert self.rfile.read(24) ==\ + '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex') + + # check empty settings frame + assert self.rfile.read(9) ==\ + '000000040000000000'.decode('hex') + + # send empty settings frame + self.wfile.write('000000040000000000'.decode('hex')) + self.wfile.flush() + + # check settings acknowledgement + assert self.rfile.read(9) == \ + '000000040100000000'.decode('hex') + + # send settings acknowledgement + self.wfile.write('000000040100000000'.decode('hex')) + self.wfile.flush() + + def test_perform_client_connection_preface(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + protocol = http2.HTTP2Protocol(c) + protocol.perform_client_connection_preface() + + +class TestClientStreamIds(): + c = tcp.TCPClient(("127.0.0.1", 0)) + protocol = http2.HTTP2Protocol(c) + + def test_client_stream_ids(self): + assert self.protocol.current_stream_id is None + assert self.protocol.next_stream_id() == 1 + assert self.protocol.current_stream_id == 1 + assert self.protocol.next_stream_id() == 3 + assert self.protocol.current_stream_id == 3 + assert self.protocol.next_stream_id() == 5 + assert self.protocol.current_stream_id == 5 + + +class TestServerStreamIds(): + c = tcp.TCPClient(("127.0.0.1", 0)) + protocol = http2.HTTP2Protocol(c, is_server=True) + + def test_server_stream_ids(self): + assert self.protocol.current_stream_id is None + assert self.protocol.next_stream_id() == 2 + assert self.protocol.current_stream_id == 2 + assert self.protocol.next_stream_id() == 4 + assert self.protocol.current_stream_id == 4 + assert self.protocol.next_stream_id() == 6 + assert self.protocol.current_stream_id == 6 + + +class TestApplySettings(test.ServerTestBase): + class handler(tcp.BaseHandler): + + def handle(self): + # check settings acknowledgement + assert self.rfile.read(9) == '000000040100000000'.decode('hex') + self.wfile.write("OK") + self.wfile.flush() + + ssl = True + + def test_apply_settings(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + c.convert_to_ssl() + protocol = http2.HTTP2Protocol(c) + + protocol._apply_settings({ + SettingsFrame.SETTINGS.SETTINGS_ENABLE_PUSH: 'foo', + SettingsFrame.SETTINGS.SETTINGS_MAX_CONCURRENT_STREAMS: 'bar', + SettingsFrame.SETTINGS.SETTINGS_INITIAL_WINDOW_SIZE: 'deadbeef', + }) + + assert c.rfile.safe_read(2) == "OK" + + assert protocol.http2_settings[ + SettingsFrame.SETTINGS.SETTINGS_ENABLE_PUSH] == 'foo' + assert protocol.http2_settings[ + SettingsFrame.SETTINGS.SETTINGS_MAX_CONCURRENT_STREAMS] == 'bar' + assert protocol.http2_settings[ + SettingsFrame.SETTINGS.SETTINGS_INITIAL_WINDOW_SIZE] == 'deadbeef' + + +class TestCreateHeaders(): + c = tcp.TCPClient(("127.0.0.1", 0)) + + def test_create_headers(self): + headers = [ + (b':method', b'GET'), + (b':path', b'index.html'), + (b':scheme', b'https'), + (b'foo', b'bar')] + + bytes = http2.HTTP2Protocol(self.c)._create_headers( + headers, 1, end_stream=True) + assert b''.join(bytes) ==\ + '000014010500000001824488355217caf3a69a3f87408294e7838c767f'\ + .decode('hex') + + bytes = http2.HTTP2Protocol(self.c)._create_headers( + headers, 1, end_stream=False) + assert b''.join(bytes) ==\ + '000014010400000001824488355217caf3a69a3f87408294e7838c767f'\ + .decode('hex') + + # TODO: add test for too large header_block_fragments + + +class TestCreateBody(): + c = tcp.TCPClient(("127.0.0.1", 0)) + protocol = http2.HTTP2Protocol(c) + + def test_create_body_empty(self): + bytes = self.protocol._create_body(b'', 1) + assert b''.join(bytes) == ''.decode('hex') + + def test_create_body_single_frame(self): + bytes = self.protocol._create_body('foobar', 1) + assert b''.join(bytes) == '000006000100000001666f6f626172'.decode('hex') + + def test_create_body_multiple_frames(self): + pass + # bytes = self.protocol._create_body('foobar' * 3000, 1) + # TODO: add test for too large frames + + +class TestCreateRequest(): + c = tcp.TCPClient(("127.0.0.1", 0)) + + def test_create_request_simple(self): + bytes = http2.HTTP2Protocol(self.c).create_request('GET', '/') + assert len(bytes) == 1 + assert bytes[0] == '00000d0105000000018284874188089d5c0b8170dc07'.decode('hex') + + def test_create_request_with_body(self): + bytes = http2.HTTP2Protocol(self.c).create_request( + 'GET', '/', [(b'foo', b'bar')], 'foobar') + assert len(bytes) == 2 + assert bytes[0] ==\ + '0000150104000000018284874188089d5c0b8170dc07408294e7838c767f'.decode('hex') + assert bytes[1] ==\ + '000006000100000001666f6f626172'.decode('hex') + + +class TestReadResponse(test.ServerTestBase): + class handler(tcp.BaseHandler): + + def handle(self): + self.wfile.write( + b'00000801040000000188628594e78c767f'.decode('hex')) + self.wfile.write( + b'000006000100000001666f6f626172'.decode('hex')) + self.wfile.flush() + + ssl = True + + def test_read_response(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + c.convert_to_ssl() + protocol = http2.HTTP2Protocol(c) + + status, headers, body = protocol.read_response() + + assert headers == {':status': '200', 'etag': 'foobar'} + assert status == "200" + assert body == b'foobar' + + +class TestReadEmptyResponse(test.ServerTestBase): + class handler(tcp.BaseHandler): + + def handle(self): + self.wfile.write( + b'00000801050000000188628594e78c767f'.decode('hex')) + self.wfile.flush() + + ssl = True + + def test_read_empty_response(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + c.convert_to_ssl() + protocol = http2.HTTP2Protocol(c) + + status, headers, body = protocol.read_response() + + assert headers == {':status': '200', 'etag': 'foobar'} + assert status == "200" + assert body == b'' + + +class TestReadRequest(test.ServerTestBase): + class handler(tcp.BaseHandler): + + def handle(self): + self.wfile.write( + b'000003010400000001828487'.decode('hex')) + self.wfile.write( + b'000006000100000001666f6f626172'.decode('hex')) + self.wfile.flush() + + ssl = True + + def test_read_request(self): + c = tcp.TCPClient(("127.0.0.1", self.port)) + c.connect() + c.convert_to_ssl() + protocol = http2.HTTP2Protocol(c, is_server=True) + + stream_id, headers, body = protocol.read_request() + + assert stream_id + assert headers == {':method': 'GET', ':path': '/', ':scheme': 'https'} + assert body == b'foobar' + + +class TestCreateResponse(): + c = tcp.TCPClient(("127.0.0.1", 0)) + + def test_create_response_simple(self): + bytes = http2.HTTP2Protocol(self.c, is_server=True).create_response(200) + assert len(bytes) == 1 + assert bytes[0] ==\ + '00000101050000000288'.decode('hex') + + def test_create_response_with_body(self): + bytes = http2.HTTP2Protocol(self.c, is_server=True).create_response( + 200, 1, [(b'foo', b'bar')], 'foobar') + assert len(bytes) == 2 + assert bytes[0] ==\ + '00000901040000000188408294e7838c767f'.decode('hex') + assert bytes[1] ==\ + '000006000100000001666f6f626172'.decode('hex') -- cgit v1.2.3 From 6a4dcaf3561cf279937114a8a80ebad8adcc1eec Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 18 Jun 2015 11:33:43 +0200 Subject: remove implementation badge line too short :-/ --- README.mkd | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/README.mkd b/README.mkd index 2f87d111..f5e66d99 100644 --- a/README.mkd +++ b/README.mkd @@ -4,7 +4,6 @@ [![Downloads](https://img.shields.io/pypi/dm/netlib.svg?color=orange)](https://pypi.python.org/pypi/netlib) [![Latest Version](https://img.shields.io/pypi/v/netlib.svg)](https://pypi.python.org/pypi/netlib) [![Supported Python versions](https://img.shields.io/pypi/pyversions/netlib.svg)](https://pypi.python.org/pypi/netlib) -[![Supported Python implementations](https://img.shields.io/pypi/implementation/netlib.svg)](https://pypi.python.org/pypi/netlib) Netlib is a collection of network utility classes, used by the pathod and mitmproxy projects. It differs from other projects in some fundamental @@ -16,7 +15,7 @@ functions, and are designed to allow misbehaviour when needed. Requirements ------------ -* [Python](http://www.python.org) 2.7.x. +* [Python](http://www.python.org) 2.7.x or a compatible version of pypy. * Third-party packages listed in [setup.py](https://github.com/mitmproxy/netlib/blob/master/setup.py) Hacking -- cgit v1.2.3