From eeaed93a83fbe14762e263e9f25b5361088daa15 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 11 Jun 2015 15:37:17 +0200 Subject: improve ALPN integration --- netlib/tcp.py | 23 +++++++++++++++-------- 1 file changed, 15 insertions(+), 8 deletions(-) (limited to 'netlib') diff --git a/netlib/tcp.py b/netlib/tcp.py index 9a980035..98b17c50 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -404,16 +404,17 @@ class _Connection(object): context.set_info_callback(log_ssl_key) if OpenSSL._util.lib.Cryptography_HAS_ALPN: - # advertise application layer protocols if alpn_protos is not None: + # advertise application layer protocols context.set_alpn_protos(alpn_protos) - - # select application layer protocol - if alpn_select is not None: - def alpn_select_f(conn, options): - return bytes(alpn_select) - - context.set_alpn_select_callback(alpn_select_f) + elif alpn_select is not None: + # select application layer protocol + def alpn_select_callback(conn, options): + if alpn_select in options: + return bytes(alpn_select) + else: + return options[0] + context.set_alpn_select_callback(alpn_select_callback) return context @@ -612,6 +613,12 @@ class BaseHandler(_Connection): def settimeout(self, n): self.connection.settimeout(n) + def get_alpn_proto_negotiated(self): + if OpenSSL._util.lib.Cryptography_HAS_ALPN and self.ssl_established: + return self.connection.get_alpn_proto_negotiated() + else: # pragma no cover + return None + class TCPServer(object): request_queue_size = 20 -- cgit v1.2.3 From 8ea157775debeccfa0f2fab3aa7e009d13ce4391 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 11 Jun 2015 15:38:32 +0200 Subject: http2: general improvements --- netlib/http2/protocol.py | 63 +++++++++++++++++++++++++++++++++--------------- 1 file changed, 43 insertions(+), 20 deletions(-) (limited to 'netlib') diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index feac220c..4b69764f 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -26,12 +26,13 @@ class HTTP2Protocol(object): ) # "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" - CLIENT_CONNECTION_PREFACE = '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a' + CLIENT_CONNECTION_PREFACE = '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex') ALPN_PROTO_H2 = 'h2' - def __init__(self, tcp_client): - self.tcp_client = tcp_client + def __init__(self, tcp_handler, is_server=False): + self.tcp_handler = tcp_handler + self.is_server = is_server self.http2_settings = frame.HTTP2_DEFAULT_SETTINGS.copy() self.current_stream_id = None @@ -39,28 +40,39 @@ class HTTP2Protocol(object): self.decoder = Decoder() def check_alpn(self): - alp = self.tcp_client.get_alpn_proto_negotiated() + alp = self.tcp_handler.get_alpn_proto_negotiated() if alp != self.ALPN_PROTO_H2: raise NotImplementedError( "HTTP2Protocol can not handle unknown ALP: %s" % alp) return True - def perform_connection_preface(self): - self.tcp_client.wfile.write( - bytes(self.CLIENT_CONNECTION_PREFACE.decode('hex'))) - self.send_frame(frame.SettingsFrame(state=self)) - - # read server settings frame - frm = frame.Frame.from_file(self.tcp_client.rfile, self) + def _receive_settings(self): + frm = frame.Frame.from_file(self.tcp_handler.rfile, self) assert isinstance(frm, frame.SettingsFrame) self._apply_settings(frm.settings) - # read setting ACK frame + def _read_settings_ack(self): settings_ack_frame = self.read_frame() assert isinstance(settings_ack_frame, frame.SettingsFrame) assert settings_ack_frame.flags & frame.Frame.FLAG_ACK assert len(settings_ack_frame.settings) == 0 + def perform_server_connection_preface(self): + magic_length = len(self.CLIENT_CONNECTION_PREFACE) + magic = self.tcp_handler.rfile.safe_read(magic_length) + assert magic == self.CLIENT_CONNECTION_PREFACE + + self.send_frame(frame.SettingsFrame(state=self)) + self._receive_settings() + self._read_settings_ack() + + def perform_client_connection_preface(self): + self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE) + + self.send_frame(frame.SettingsFrame(state=self)) + self._receive_settings() + self._read_settings_ack() + def next_stream_id(self): if self.current_stream_id is None: self.current_stream_id = 1 @@ -70,11 +82,11 @@ class HTTP2Protocol(object): def send_frame(self, frame): raw_bytes = frame.to_bytes() - self.tcp_client.wfile.write(raw_bytes) - self.tcp_client.wfile.flush() + self.tcp_handler.wfile.write(raw_bytes) + self.tcp_handler.wfile.flush() def read_frame(self): - frm = frame.Frame.from_file(self.tcp_client.rfile, self) + frm = frame.Frame.from_file(self.tcp_handler.rfile, self) if isinstance(frm, frame.SettingsFrame): self._apply_settings(frm.settings) @@ -139,25 +151,36 @@ class HTTP2Protocol(object): self._create_body(body, stream_id))) def read_response(self): + headers, body = self._receive_transmission() + return headers[':status'], headers, body + + def read_request(self): + return self._receive_transmission() + + def _receive_transmission(self): + body_expected = True + header_block_fragment = b'' body = b'' while True: frm = self.read_frame() - if isinstance(frm, frame.HeadersFrame): + if isinstance(frm, frame.HeadersFrame) or isinstance(frm, frame.ContinuationFrame): header_block_fragment += frm.header_block_fragment - if frm.flags | frame.Frame.FLAG_END_HEADERS: + if frm.flags & frame.Frame.FLAG_END_HEADERS: + if frm.flags & frame.Frame.FLAG_END_STREAM: + body_expected = False break - while True: + while body_expected: frm = self.read_frame() if isinstance(frm, frame.DataFrame): body += frm.payload - if frm.flags | frame.Frame.FLAG_END_STREAM: + if frm.flags & frame.Frame.FLAG_END_STREAM: break headers = {} for header, value in self.decoder.decode(header_block_fragment): headers[header] = value - return headers[':status'], headers, body + return headers, body -- cgit v1.2.3 From a901bc3032747faf00adf82c3187d38213c070ca Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 12 Jun 2015 14:41:54 +0200 Subject: http2: add response creation --- netlib/http2/protocol.py | 56 +++++++++++++++++++++++++++++++++++------------- 1 file changed, 41 insertions(+), 15 deletions(-) (limited to 'netlib') diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 4b69764f..56aee490 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -26,7 +26,8 @@ class HTTP2Protocol(object): ) # "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" - CLIENT_CONNECTION_PREFACE = '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex') + CLIENT_CONNECTION_PREFACE =\ + '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'.decode('hex') ALPN_PROTO_H2 = 'h2' @@ -38,6 +39,7 @@ class HTTP2Protocol(object): self.current_stream_id = None self.encoder = Encoder() self.decoder = Decoder() + self.connection_preface_performed = False def check_alpn(self): alp = self.tcp_handler.get_alpn_proto_negotiated() @@ -57,25 +59,36 @@ class HTTP2Protocol(object): assert settings_ack_frame.flags & frame.Frame.FLAG_ACK assert len(settings_ack_frame.settings) == 0 - def perform_server_connection_preface(self): - magic_length = len(self.CLIENT_CONNECTION_PREFACE) - magic = self.tcp_handler.rfile.safe_read(magic_length) - assert magic == self.CLIENT_CONNECTION_PREFACE + def perform_server_connection_preface(self, force=False): + if force or not self.connection_preface_performed: + self.connection_preface_performed = True - self.send_frame(frame.SettingsFrame(state=self)) - self._receive_settings() - self._read_settings_ack() + magic_length = len(self.CLIENT_CONNECTION_PREFACE) + magic = self.tcp_handler.rfile.safe_read(magic_length) + assert magic == self.CLIENT_CONNECTION_PREFACE - def perform_client_connection_preface(self): - self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE) + self.send_frame(frame.SettingsFrame(state=self)) + self._receive_settings() + self._read_settings_ack() - self.send_frame(frame.SettingsFrame(state=self)) - self._receive_settings() - self._read_settings_ack() + def perform_client_connection_preface(self, force=False): + if force or not self.connection_preface_performed: + self.connection_preface_performed = True + + self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE) + + self.send_frame(frame.SettingsFrame(state=self)) + self._receive_settings() + self._read_settings_ack() def next_stream_id(self): if self.current_stream_id is None: - self.current_stream_id = 1 + if self.is_server: + # servers must use even stream ids + self.current_stream_id = 2 + else: + # clients must use odd stream ids + self.current_stream_id = 1 else: self.current_stream_id += 2 return self.current_stream_id @@ -165,7 +178,8 @@ class HTTP2Protocol(object): while True: frm = self.read_frame() - if isinstance(frm, frame.HeadersFrame) or isinstance(frm, frame.ContinuationFrame): + if isinstance(frm, frame.HeadersFrame)\ + or isinstance(frm, frame.ContinuationFrame): header_block_fragment += frm.header_block_fragment if frm.flags & frame.Frame.FLAG_END_HEADERS: if frm.flags & frame.Frame.FLAG_END_STREAM: @@ -184,3 +198,15 @@ class HTTP2Protocol(object): headers[header] = value return headers, body + + def create_response(self, code, headers=None, body=None): + if headers is None: + headers = [] + + headers = [(b':status', bytes(str(code)))] + headers + + stream_id = self.next_stream_id() + + return list(itertools.chain( + self._create_headers(headers, stream_id, end_stream=(body is None)), + self._create_body(body, stream_id))) -- cgit v1.2.3 From 5fab755a05f2ddd1b3e8e446e10fdcbded894e70 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 12 Jun 2015 15:21:23 +0200 Subject: add more tests --- netlib/tcp.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'netlib') diff --git a/netlib/tcp.py b/netlib/tcp.py index 98b17c50..eb8a523f 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -412,7 +412,7 @@ class _Connection(object): def alpn_select_callback(conn, options): if alpn_select in options: return bytes(alpn_select) - else: + else: # pragma no cover return options[0] context.set_alpn_select_callback(alpn_select_callback) @@ -500,9 +500,9 @@ class TCPClient(_Connection): return self.connection.gettimeout() def get_alpn_proto_negotiated(self): - if OpenSSL._util.lib.Cryptography_HAS_ALPN: + if OpenSSL._util.lib.Cryptography_HAS_ALPN and self.ssl_established: return self.connection.get_alpn_proto_negotiated() - else: # pragma no cover + else: return None @@ -616,7 +616,7 @@ class BaseHandler(_Connection): def get_alpn_proto_negotiated(self): if OpenSSL._util.lib.Cryptography_HAS_ALPN and self.ssl_established: return self.connection.get_alpn_proto_negotiated() - else: # pragma no cover + else: return None -- cgit v1.2.3 From 9c6d237d02290c2388f19ec8f215827d4f921e4b Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 12 Jun 2015 16:03:01 +0200 Subject: add new TLS methods --- netlib/tcp.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'netlib') diff --git a/netlib/tcp.py b/netlib/tcp.py index eb8a523f..74fe70d4 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -19,6 +19,9 @@ SSLv2_METHOD = SSL.SSLv2_METHOD SSLv3_METHOD = SSL.SSLv3_METHOD SSLv23_METHOD = SSL.SSLv23_METHOD TLSv1_METHOD = SSL.TLSv1_METHOD +TLSv1_1_METHOD = SSL.TLSv1_1_METHOD +TLSv1_2_METHOD = SSL.TLSv1_2_METHOD + OP_NO_SSLv2 = SSL.OP_NO_SSLv2 OP_NO_SSLv3 = SSL.OP_NO_SSLv3 @@ -376,7 +379,7 @@ class _Connection(object): alpn_select=None, ): """ - :param method: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, TLSv1_METHOD or TLSv1_1_METHOD + :param method: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, TLSv1_METHOD, TLSv1_1_METHOD, or TLSv1_2_METHOD :param options: A bit field consisting of OpenSSL.SSL.OP_* values :param cipher_list: A textual OpenSSL cipher list, see https://www.openssl.org/docs/apps/ciphers.html :rtype : SSL.Context -- cgit v1.2.3 From 8d71a5b4aba8248b97918b11b12275bbf5197337 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 14 Jun 2015 19:17:34 +0200 Subject: http2: add authority header --- netlib/http2/protocol.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'netlib') diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 56aee490..1e722dfb 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -152,10 +152,13 @@ class HTTP2Protocol(object): if headers is None: headers = [] + authority = self.tcp_handler.sni if self.tcp_handler.sni else self.tcp_handler.address.host headers = [ (b':method', bytes(method)), (b':path', bytes(path)), - (b':scheme', b'https')] + headers + (b':scheme', b'https'), + (b':authority', authority), + ] + headers stream_id = self.next_stream_id() @@ -192,6 +195,7 @@ class HTTP2Protocol(object): body += frm.payload if frm.flags & frame.Frame.FLAG_END_STREAM: break + # TODO: implement window update & flow headers = {} for header, value in self.decoder.decode(header_block_fragment): -- cgit v1.2.3 From 0d137eac6f4c00a72d3aa4d11fce7d1ea15f0f21 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 14 Jun 2015 19:50:35 +0200 Subject: simplify ALPN --- netlib/tcp.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'netlib') diff --git a/netlib/tcp.py b/netlib/tcp.py index 74fe70d4..897e3e65 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -535,7 +535,6 @@ class BaseHandler(_Connection): request_client_cert=None, chain_file=None, dhparams=None, - alpn_select=None, **sslctx_kwargs): """ cert: A certutils.SSLCert object. @@ -562,9 +561,7 @@ class BaseHandler(_Connection): until then we're conservative. """ - context = self._create_ssl_context( - alpn_select=alpn_select, - **sslctx_kwargs) + context = self._create_ssl_context(**sslctx_kwargs) context.use_privatekey(key) context.use_certificate(cert.x509) @@ -589,7 +586,7 @@ class BaseHandler(_Connection): return context - def convert_to_ssl(self, cert, key, alpn_select=None, **sslctx_kwargs): + def convert_to_ssl(self, cert, key, **sslctx_kwargs): """ Convert connection to SSL. For a list of parameters, see BaseHandler._create_ssl_context(...) @@ -598,7 +595,6 @@ class BaseHandler(_Connection): context = self.create_ssl_context( cert, key, - alpn_select=alpn_select, **sslctx_kwargs) self.connection = SSL.Connection(context, self.connection) self.connection.set_accept_state() -- cgit v1.2.3 From fe764cde5229046b8447062971c61fac745d2d58 Mon Sep 17 00:00:00 2001 From: Kyle Morton Date: Mon, 15 Jun 2015 10:16:44 -0700 Subject: Adding support for upstream certificate validation when using SSL/TLS with an instance of TCPClient. --- netlib/tcp.py | 23 +++++++++++++++++++++++ 1 file changed, 23 insertions(+) (limited to 'netlib') diff --git a/netlib/tcp.py b/netlib/tcp.py index 9a980035..ca948514 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -21,6 +21,7 @@ SSLv23_METHOD = SSL.SSLv23_METHOD TLSv1_METHOD = SSL.TLSv1_METHOD OP_NO_SSLv2 = SSL.OP_NO_SSLv2 OP_NO_SSLv3 = SSL.OP_NO_SSLv3 +VERIFY_NONE = SSL.VERIFY_NONE class NetLibError(Exception): @@ -371,6 +372,9 @@ class _Connection(object): def _create_ssl_context(self, method=SSLv23_METHOD, options=(OP_NO_SSLv2 | OP_NO_SSLv3), + verify_options=VERIFY_NONE, + ca_path=None, + ca_pemfile=None, cipher_list=None, alpn_protos=None, alpn_select=None, @@ -378,6 +382,9 @@ class _Connection(object): """ :param method: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, TLSv1_METHOD or TLSv1_1_METHOD :param options: A bit field consisting of OpenSSL.SSL.OP_* values + :param verify_options: A bit field consisting of OpenSSL.SSL.VERIFY_* values + :param ca_path: Path to a directory of trusted CA certificates prepared using the c_rehash tool + :param ca_pemfile: Path to a PEM formatted trusted CA certificate :param cipher_list: A textual OpenSSL cipher list, see https://www.openssl.org/docs/apps/ciphers.html :rtype : SSL.Context """ @@ -386,6 +393,19 @@ class _Connection(object): if options is not None: context.set_options(options) + # Verify Options (NONE/PEER/PEER|FAIL_IF_... and trusted CAs) + if verify_options is not None and verify_options is not VERIFY_NONE: + def verify_cert(conn, cert, errno, err_depth, is_cert_verified): + if is_cert_verified: + return True + raise NetLibError( + "Upstream certificate validation failed at depth: %s with error number: %s" % + (err_depth, errno)) + + context.set_verify(verify_options, verify_cert) + if ca_path is not None or ca_pemfile is not None: + context.load_verify_locations(ca_pemfile, ca_path) + # Workaround for # https://github.com/pyca/pyopenssl/issues/190 # https://github.com/mitmproxy/mitmproxy/issues/472 @@ -458,6 +478,9 @@ class TCPClient(_Connection): cert: Path to a file containing both client cert and private key. options: A bit field consisting of OpenSSL.SSL.OP_* values + verify_options: A bit field consisting of OpenSSL.SSL.VERIFY_* values + ca_path: Path to a directory of trusted CA certificates prepared using the c_rehash tool + ca_pemfile: Path to a PEM formatted trusted CA certificate """ context = self.create_ssl_context( alpn_protos=alpn_protos, -- cgit v1.2.3 From 12702b9a01fb6baf4d675d6f974c140581982843 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 13:15:06 +0200 Subject: http2: improve frame output --- netlib/http2/frame.py | 11 +++------ netlib/http2/protocol.py | 61 ++++++++++++++++++++++++++++-------------------- 2 files changed, 39 insertions(+), 33 deletions(-) (limited to 'netlib') diff --git a/netlib/http2/frame.py b/netlib/http2/frame.py index 4a305d82..3e285cba 100644 --- a/netlib/http2/frame.py +++ b/netlib/http2/frame.py @@ -113,16 +113,11 @@ class Frame(object): def payload_human_readable(self): # pragma: no cover raise NotImplementedError() - def human_readable(self): + def human_readable(self, direction="-"): return "\n".join([ - "============================================================", - "length: %d bytes" % self.length, - "type: %s (%#x)" % (self.__class__.__name__, self.TYPE), - "flags: %#x" % self.flags, - "stream_id: %#x" % self.stream_id, - "------------------------------------------------------------", + "%s: %s | length: %d | flags: %#x | stream_id: %d" % (direction, self.__class__.__name__, self.length, self.flags, self.stream_id), self.payload_human_readable(), - "============================================================", + "===============================================================", ]) def __eq__(self, other): diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 1e722dfb..7bf68602 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -48,13 +48,12 @@ class HTTP2Protocol(object): "HTTP2Protocol can not handle unknown ALP: %s" % alp) return True - def _receive_settings(self): - frm = frame.Frame.from_file(self.tcp_handler.rfile, self) + def _receive_settings(self, hide=False): + frm = self.read_frame(hide) assert isinstance(frm, frame.SettingsFrame) - self._apply_settings(frm.settings) - def _read_settings_ack(self): - settings_ack_frame = self.read_frame() + def _read_settings_ack(self, hide=False): + settings_ack_frame = self.read_frame(hide) assert isinstance(settings_ack_frame, frame.SettingsFrame) assert settings_ack_frame.flags & frame.Frame.FLAG_ACK assert len(settings_ack_frame.settings) == 0 @@ -67,9 +66,8 @@ class HTTP2Protocol(object): magic = self.tcp_handler.rfile.safe_read(magic_length) assert magic == self.CLIENT_CONNECTION_PREFACE - self.send_frame(frame.SettingsFrame(state=self)) - self._receive_settings() - self._read_settings_ack() + self.send_frame(frame.SettingsFrame(state=self), hide=True) + self._receive_settings(hide=True) def perform_client_connection_preface(self, force=False): if force or not self.connection_preface_performed: @@ -77,9 +75,8 @@ class HTTP2Protocol(object): self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE) - self.send_frame(frame.SettingsFrame(state=self)) - self._receive_settings() - self._read_settings_ack() + self.send_frame(frame.SettingsFrame(state=self), hide=True) + self._receive_settings(hide=True) def next_stream_id(self): if self.current_stream_id is None: @@ -93,30 +90,35 @@ class HTTP2Protocol(object): self.current_stream_id += 2 return self.current_stream_id - def send_frame(self, frame): - raw_bytes = frame.to_bytes() + def send_frame(self, frm, hide=False): + raw_bytes = frm.to_bytes() self.tcp_handler.wfile.write(raw_bytes) self.tcp_handler.wfile.flush() + if not hide and self.tcp_handler.http2_framedump: + print(frm.human_readable(">>")) - def read_frame(self): + def read_frame(self, hide=False): frm = frame.Frame.from_file(self.tcp_handler.rfile, self) - if isinstance(frm, frame.SettingsFrame): - self._apply_settings(frm.settings) + if not hide and self.tcp_handler.http2_framedump: + print(frm.human_readable("<<")) + if isinstance(frm, frame.SettingsFrame) and not frm.flags & frame.Frame.FLAG_ACK: + self._apply_settings(frm.settings, hide) return frm - def _apply_settings(self, settings): + def _apply_settings(self, settings, hide=False): for setting, value in settings.items(): old_value = self.http2_settings[setting] if not old_value: old_value = '-' - self.http2_settings[setting] = value self.send_frame( frame.SettingsFrame( state=self, - flags=frame.Frame.FLAG_ACK)) + flags=frame.Frame.FLAG_ACK), + hide) + self._read_settings_ack(hide) def _create_headers(self, headers, stream_id, end_stream=True): # TODO: implement max frame size checks and sending in chunks @@ -127,12 +129,16 @@ class HTTP2Protocol(object): header_block_fragment = self.encoder.encode(headers) - bytes = frame.HeadersFrame( + frm = frame.HeadersFrame( state=self, flags=flags, stream_id=stream_id, - header_block_fragment=header_block_fragment).to_bytes() - return [bytes] + header_block_fragment=header_block_fragment) + + if self.tcp_handler.http2_framedump: + print(frm.human_readable(">>")) + + return [frm.to_bytes()] def _create_body(self, body, stream_id): if body is None or len(body) == 0: @@ -141,12 +147,17 @@ class HTTP2Protocol(object): # TODO: implement max frame size checks and sending in chunks # TODO: implement flow-control window - bytes = frame.DataFrame( + frm = frame.DataFrame( state=self, flags=frame.Frame.FLAG_END_STREAM, stream_id=stream_id, - payload=body).to_bytes() - return [bytes] + payload=body) + + if self.tcp_handler.http2_framedump: + print(frm.human_readable(">>")) + + return [frm.to_bytes()] + def create_request(self, method, path, headers=None, body=None): if headers is None: -- cgit v1.2.3 From 79ff43993018209a76a2a7cff995e912eb20d4c3 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 09:47:43 +0200 Subject: add elliptic curve during TLS handshake --- netlib/tcp.py | 13 +++++-------- 1 file changed, 5 insertions(+), 8 deletions(-) (limited to 'netlib') diff --git a/netlib/tcp.py b/netlib/tcp.py index 953cef6e..2e847d83 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -22,11 +22,6 @@ TLSv1_METHOD = SSL.TLSv1_METHOD TLSv1_1_METHOD = SSL.TLSv1_1_METHOD TLSv1_2_METHOD = SSL.TLSv1_2_METHOD -OP_NO_SSLv2 = SSL.OP_NO_SSLv2 -OP_NO_SSLv3 = SSL.OP_NO_SSLv3 -VERIFY_NONE = SSL.VERIFY_NONE - - class NetLibError(Exception): pass @@ -374,8 +369,8 @@ class _Connection(object): def _create_ssl_context(self, method=SSLv23_METHOD, - options=(OP_NO_SSLv2 | OP_NO_SSLv3), - verify_options=VERIFY_NONE, + options=(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3 | SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_COMPRESSION), + verify_options=SSL.VERIFY_NONE, ca_path=None, ca_pemfile=None, cipher_list=None, @@ -397,7 +392,7 @@ class _Connection(object): context.set_options(options) # Verify Options (NONE/PEER/PEER|FAIL_IF_... and trusted CAs) - if verify_options is not None and verify_options is not VERIFY_NONE: + if verify_options is not None and verify_options is not SSL.VERIFY_NONE: def verify_cert(conn, cert, errno, err_depth, is_cert_verified): if is_cert_verified: return True @@ -426,6 +421,8 @@ class _Connection(object): if log_ssl_key: context.set_info_callback(log_ssl_key) + context.set_tmp_ecdh(OpenSSL.crypto.get_elliptic_curve('prime256v1')) + if OpenSSL._util.lib.Cryptography_HAS_ALPN: if alpn_protos is not None: # advertise application layer protocols -- cgit v1.2.3 From e3db241a2fa47a38fcb85532ed52eeecf1a7b965 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 13:43:23 +0200 Subject: http2: improve frame output --- netlib/http2/protocol.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'netlib') diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 7bf68602..24fcb712 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -31,7 +31,7 @@ class HTTP2Protocol(object): ALPN_PROTO_H2 = 'h2' - def __init__(self, tcp_handler, is_server=False): + def __init__(self, tcp_handler, is_server=False, dump_frames=False): self.tcp_handler = tcp_handler self.is_server = is_server @@ -40,6 +40,7 @@ class HTTP2Protocol(object): self.encoder = Encoder() self.decoder = Decoder() self.connection_preface_performed = False + self.dump_frames = dump_frames def check_alpn(self): alp = self.tcp_handler.get_alpn_proto_negotiated() @@ -94,12 +95,12 @@ class HTTP2Protocol(object): raw_bytes = frm.to_bytes() self.tcp_handler.wfile.write(raw_bytes) self.tcp_handler.wfile.flush() - if not hide and self.tcp_handler.http2_framedump: + if not hide and self.dump_frames: print(frm.human_readable(">>")) def read_frame(self, hide=False): frm = frame.Frame.from_file(self.tcp_handler.rfile, self) - if not hide and self.tcp_handler.http2_framedump: + if not hide and self.dump_frames: print(frm.human_readable("<<")) if isinstance(frm, frame.SettingsFrame) and not frm.flags & frame.Frame.FLAG_ACK: self._apply_settings(frm.settings, hide) @@ -135,7 +136,7 @@ class HTTP2Protocol(object): stream_id=stream_id, header_block_fragment=header_block_fragment) - if self.tcp_handler.http2_framedump: + if self.dump_frames: print(frm.human_readable(">>")) return [frm.to_bytes()] @@ -153,7 +154,7 @@ class HTTP2Protocol(object): stream_id=stream_id, payload=body) - if self.tcp_handler.http2_framedump: + if self.dump_frames: print(frm.human_readable(">>")) return [frm.to_bytes()] -- cgit v1.2.3 From d0a9d3cdda6d1f784a23ea4bd9efd3134e292628 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 14:21:34 +0200 Subject: http2: only first headers frame as END_STREAM flag --- netlib/http2/protocol.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'netlib') diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 24fcb712..682b7863 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -196,9 +196,9 @@ class HTTP2Protocol(object): if isinstance(frm, frame.HeadersFrame)\ or isinstance(frm, frame.ContinuationFrame): header_block_fragment += frm.header_block_fragment + if frm.flags & frame.Frame.FLAG_END_STREAM: + body_expected = False if frm.flags & frame.Frame.FLAG_END_HEADERS: - if frm.flags & frame.Frame.FLAG_END_STREAM: - body_expected = False break while body_expected: -- cgit v1.2.3 From 1c124421e34d310c6e0577f20b595413d639a5c3 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 15:31:58 +0200 Subject: http2: fix header_block_fragments and length --- netlib/http2/frame.py | 13 +++++++++++-- netlib/http2/protocol.py | 23 +++++++++++++++-------- 2 files changed, 26 insertions(+), 10 deletions(-) (limited to 'netlib') diff --git a/netlib/http2/frame.py b/netlib/http2/frame.py index 3e285cba..98ced904 100644 --- a/netlib/http2/frame.py +++ b/netlib/http2/frame.py @@ -114,6 +114,8 @@ class Frame(object): raise NotImplementedError() def human_readable(self, direction="-"): + self.length = len(self.payload_bytes()) + return "\n".join([ "%s: %s | length: %d | flags: %#x | stream_id: %d" % (direction, self.__class__.__name__, self.length, self.flags, self.stream_id), self.payload_human_readable(), @@ -456,7 +458,10 @@ class PushPromiseFrame(Frame): s.append("padding: %d" % self.pad_length) s.append("promised stream: %#x" % self.promised_stream) - s.append("header_block_fragment: %s" % str(self.header_block_fragment)) + s.append( + "header_block_fragment: %s" % + self.header_block_fragment.encode('hex')) + return "\n".join(s) @@ -600,7 +605,11 @@ class ContinuationFrame(Frame): return self.header_block_fragment def payload_human_readable(self): - return "header_block_fragment: %s" % str(self.header_block_fragment) + s = [] + s.append( + "header_block_fragment: %s" % + self.header_block_fragment.encode('hex')) + return "\n".join(s) _FRAME_CLASSES = [ DataFrame, diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 682b7863..f17f998f 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -50,14 +50,18 @@ class HTTP2Protocol(object): return True def _receive_settings(self, hide=False): - frm = self.read_frame(hide) - assert isinstance(frm, frame.SettingsFrame) + while True: + frm = self.read_frame(hide) + if isinstance(frm, frame.SettingsFrame): + break def _read_settings_ack(self, hide=False): - settings_ack_frame = self.read_frame(hide) - assert isinstance(settings_ack_frame, frame.SettingsFrame) - assert settings_ack_frame.flags & frame.Frame.FLAG_ACK - assert len(settings_ack_frame.settings) == 0 + while True: + frm = self.read_frame(hide) + if isinstance(frm, frame.SettingsFrame): + assert settings_ack_frame.flags & frame.Frame.FLAG_ACK + assert len(settings_ack_frame.settings) == 0 + break def perform_server_connection_preface(self, force=False): if force or not self.connection_preface_performed: @@ -119,7 +123,7 @@ class HTTP2Protocol(object): state=self, flags=frame.Frame.FLAG_ACK), hide) - self._read_settings_ack(hide) + # self._read_settings_ack(hide) def _create_headers(self, headers, stream_id, end_stream=True): # TODO: implement max frame size checks and sending in chunks @@ -219,10 +223,13 @@ class HTTP2Protocol(object): if headers is None: headers = [] + body='foobar' + headers = [(b':status', bytes(str(code)))] + headers stream_id = self.next_stream_id() return list(itertools.chain( self._create_headers(headers, stream_id, end_stream=(body is None)), - self._create_body(body, stream_id))) + self._create_body(body, stream_id), + )) -- cgit v1.2.3 From 20c136e070cee0e93e870bf32199cb36b1b85275 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 15:51:40 +0200 Subject: http2: return stream_id from request for response --- netlib/http2/protocol.py | 11 +++++++---- 1 file changed, 7 insertions(+), 4 deletions(-) (limited to 'netlib') diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index f17f998f..a77edd9b 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -183,7 +183,7 @@ class HTTP2Protocol(object): self._create_body(body, stream_id))) def read_response(self): - headers, body = self._receive_transmission() + stream_id, headers, body = self._receive_transmission() return headers[':status'], headers, body def read_request(self): @@ -192,6 +192,7 @@ class HTTP2Protocol(object): def _receive_transmission(self): body_expected = True + stream_id = 0 header_block_fragment = b'' body = b'' @@ -199,6 +200,7 @@ class HTTP2Protocol(object): frm = self.read_frame() if isinstance(frm, frame.HeadersFrame)\ or isinstance(frm, frame.ContinuationFrame): + stream_id = frm.stream_id header_block_fragment += frm.header_block_fragment if frm.flags & frame.Frame.FLAG_END_STREAM: body_expected = False @@ -217,9 +219,9 @@ class HTTP2Protocol(object): for header, value in self.decoder.decode(header_block_fragment): headers[header] = value - return headers, body + return stream_id, headers, body - def create_response(self, code, headers=None, body=None): + def create_response(self, code, stream_id=None, headers=None, body=None): if headers is None: headers = [] @@ -227,7 +229,8 @@ class HTTP2Protocol(object): headers = [(b':status', bytes(str(code)))] + headers - stream_id = self.next_stream_id() + if not stream_id: + stream_id = self.next_stream_id() return list(itertools.chain( self._create_headers(headers, stream_id, end_stream=(body is None)), -- cgit v1.2.3 From abb37a3ef52ab9a0f68dc46e4a8ca165e365139b Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 17:31:08 +0200 Subject: http2: improve test suite --- netlib/http2/protocol.py | 16 ++++++++-------- netlib/tcp.py | 9 +++++---- 2 files changed, 13 insertions(+), 12 deletions(-) (limited to 'netlib') diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index a77edd9b..8191090c 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -55,7 +55,7 @@ class HTTP2Protocol(object): if isinstance(frm, frame.SettingsFrame): break - def _read_settings_ack(self, hide=False): + def _read_settings_ack(self, hide=False): # pragma no cover while True: frm = self.read_frame(hide) if isinstance(frm, frame.SettingsFrame): @@ -99,12 +99,12 @@ class HTTP2Protocol(object): raw_bytes = frm.to_bytes() self.tcp_handler.wfile.write(raw_bytes) self.tcp_handler.wfile.flush() - if not hide and self.dump_frames: + if not hide and self.dump_frames: # pragma no cover print(frm.human_readable(">>")) def read_frame(self, hide=False): frm = frame.Frame.from_file(self.tcp_handler.rfile, self) - if not hide and self.dump_frames: + if not hide and self.dump_frames: # pragma no cover print(frm.human_readable("<<")) if isinstance(frm, frame.SettingsFrame) and not frm.flags & frame.Frame.FLAG_ACK: self._apply_settings(frm.settings, hide) @@ -123,7 +123,9 @@ class HTTP2Protocol(object): state=self, flags=frame.Frame.FLAG_ACK), hide) - # self._read_settings_ack(hide) + + # be liberal in what we expect from the other end + # to be more strict use: self._read_settings_ack(hide) def _create_headers(self, headers, stream_id, end_stream=True): # TODO: implement max frame size checks and sending in chunks @@ -140,7 +142,7 @@ class HTTP2Protocol(object): stream_id=stream_id, header_block_fragment=header_block_fragment) - if self.dump_frames: + if self.dump_frames: # pragma no cover print(frm.human_readable(">>")) return [frm.to_bytes()] @@ -158,7 +160,7 @@ class HTTP2Protocol(object): stream_id=stream_id, payload=body) - if self.dump_frames: + if self.dump_frames: # pragma no cover print(frm.human_readable(">>")) return [frm.to_bytes()] @@ -225,8 +227,6 @@ class HTTP2Protocol(object): if headers is None: headers = [] - body='foobar' - headers = [(b':status', bytes(str(code)))] + headers if not stream_id: diff --git a/netlib/tcp.py b/netlib/tcp.py index 2e847d83..cafc3ed9 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -414,6 +414,9 @@ class _Connection(object): if cipher_list: try: context.set_cipher_list(cipher_list) + + # TODO: maybe change this to with newer pyOpenSSL APIs + context.set_tmp_ecdh(OpenSSL.crypto.get_elliptic_curve('prime256v1')) except SSL.Error as v: raise NetLibError("SSL cipher specification error: %s" % str(v)) @@ -421,8 +424,6 @@ class _Connection(object): if log_ssl_key: context.set_info_callback(log_ssl_key) - context.set_tmp_ecdh(OpenSSL.crypto.get_elliptic_curve('prime256v1')) - if OpenSSL._util.lib.Cryptography_HAS_ALPN: if alpn_protos is not None: # advertise application layer protocols @@ -526,7 +527,7 @@ class TCPClient(_Connection): if OpenSSL._util.lib.Cryptography_HAS_ALPN and self.ssl_established: return self.connection.get_alpn_proto_negotiated() else: - return None + return "" class BaseHandler(_Connection): @@ -636,7 +637,7 @@ class BaseHandler(_Connection): if OpenSSL._util.lib.Cryptography_HAS_ALPN and self.ssl_established: return self.connection.get_alpn_proto_negotiated() else: - return None + return "" class TCPServer(object): -- cgit v1.2.3 From eb823a04a19de7fd9e15d225064ae4581f0b85bf Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 15 Jun 2015 23:36:14 +0200 Subject: http2: improve :authority header --- netlib/http2/protocol.py | 3 +++ 1 file changed, 3 insertions(+) (limited to 'netlib') diff --git a/netlib/http2/protocol.py b/netlib/http2/protocol.py index 8191090c..ac89bac4 100644 --- a/netlib/http2/protocol.py +++ b/netlib/http2/protocol.py @@ -171,6 +171,9 @@ class HTTP2Protocol(object): headers = [] authority = self.tcp_handler.sni if self.tcp_handler.sni else self.tcp_handler.address.host + if self.tcp_handler.address.port != 443: + authority += ":%d" % self.tcp_handler.address.port + headers = [ (b':method', bytes(method)), (b':path', bytes(path)), -- cgit v1.2.3 From c9c93af453ec332b660f70402b78ae8f269280f0 Mon Sep 17 00:00:00 2001 From: Kyle Morton Date: Tue, 16 Jun 2015 11:11:10 -0700 Subject: Adding certifi as default CA bundle. --- netlib/tcp.py | 6 +++--- 1 file changed, 3 insertions(+), 3 deletions(-) (limited to 'netlib') diff --git a/netlib/tcp.py b/netlib/tcp.py index ca948514..b523bea4 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -7,6 +7,7 @@ import threading import time import traceback +import certifi import OpenSSL from OpenSSL import SSL @@ -373,7 +374,7 @@ class _Connection(object): method=SSLv23_METHOD, options=(OP_NO_SSLv2 | OP_NO_SSLv3), verify_options=VERIFY_NONE, - ca_path=None, + ca_path=certifi.where(), ca_pemfile=None, cipher_list=None, alpn_protos=None, @@ -403,8 +404,7 @@ class _Connection(object): (err_depth, errno)) context.set_verify(verify_options, verify_cert) - if ca_path is not None or ca_pemfile is not None: - context.load_verify_locations(ca_pemfile, ca_path) + context.load_verify_locations(ca_pemfile, ca_path) # Workaround for # https://github.com/pyca/pyopenssl/issues/190 -- cgit v1.2.3 From 836b1eab9700230991822102d411aed067308123 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 17 Jun 2015 13:10:27 +0200 Subject: fix warnings and code smells use prospector to find them --- netlib/http2/__init__.py | 1 - netlib/http2/frame.py | 55 ++++++++++++++++++++++++------------------------ netlib/http_cookies.py | 8 +++---- netlib/http_uastrings.py | 24 +++++++++++---------- netlib/tcp.py | 8 +++---- netlib/utils.py | 2 +- netlib/websockets.py | 16 +++++++------- 7 files changed, 56 insertions(+), 58 deletions(-) (limited to 'netlib') diff --git a/netlib/http2/__init__.py b/netlib/http2/__init__.py index 92897b5d..5acf7696 100644 --- a/netlib/http2/__init__.py +++ b/netlib/http2/__init__.py @@ -1,3 +1,2 @@ - from frame import * from protocol import * diff --git a/netlib/http2/frame.py b/netlib/http2/frame.py index 4a305d82..43676623 100644 --- a/netlib/http2/frame.py +++ b/netlib/http2/frame.py @@ -1,6 +1,5 @@ import sys import struct -from functools import reduce from hpack.hpack import Encoder, Decoder from .. import utils @@ -52,7 +51,7 @@ class Frame(object): self.stream_id = stream_id @classmethod - def _check_frame_size(self, length, state): + def _check_frame_size(cls, length, state): if state: settings = state.http2_settings else: @@ -67,7 +66,7 @@ class Frame(object): length, max_frame_size)) @classmethod - def from_file(self, fp, state=None): + def from_file(cls, fp, state=None): """ read a HTTP/2 frame sent by a server or client fp is a "file like" object that could be backed by a network @@ -83,7 +82,7 @@ class Frame(object): if raw_header[:4] == b'HTTP': # pragma no cover print >> sys.stderr, "WARNING: This looks like an HTTP/1 connection!" - self._check_frame_size(length, state) + cls._check_frame_size(length, state) payload = fp.safe_read(length) return FRAMES[fields[2]].from_bytes( @@ -146,10 +145,10 @@ class DataFrame(Frame): self.pad_length = pad_length @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) - if f.flags & self.FLAG_PADDED: + if f.flags & Frame.FLAG_PADDED: f.pad_length = struct.unpack('!B', payload[0])[0] f.payload = payload[1:-f.pad_length] else: @@ -204,16 +203,16 @@ class HeadersFrame(Frame): self.weight = weight @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) - if f.flags & self.FLAG_PADDED: + if f.flags & Frame.FLAG_PADDED: f.pad_length = struct.unpack('!B', payload[0])[0] f.header_block_fragment = payload[1:-f.pad_length] else: f.header_block_fragment = payload[0:] - if f.flags & self.FLAG_PRIORITY: + if f.flags & Frame.FLAG_PRIORITY: f.stream_dependency, f.weight = struct.unpack( '!LB', f.header_block_fragment[:5]) f.exclusive = bool(f.stream_dependency >> 31) @@ -279,8 +278,8 @@ class PriorityFrame(Frame): self.weight = weight @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) f.stream_dependency, f.weight = struct.unpack('!LB', payload) f.exclusive = bool(f.stream_dependency >> 31) @@ -325,8 +324,8 @@ class RstStreamFrame(Frame): self.error_code = error_code @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) f.error_code = struct.unpack('!L', payload)[0] return f @@ -369,8 +368,8 @@ class SettingsFrame(Frame): self.settings = settings @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) for i in xrange(0, len(payload), 6): identifier, value = struct.unpack("!HL", payload[i:i + 6]) @@ -420,10 +419,10 @@ class PushPromiseFrame(Frame): self.header_block_fragment = header_block_fragment @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) - if f.flags & self.FLAG_PADDED: + if f.flags & Frame.FLAG_PADDED: f.pad_length, f.promised_stream = struct.unpack('!BL', payload[:5]) f.header_block_fragment = payload[5:-f.pad_length] else: @@ -480,8 +479,8 @@ class PingFrame(Frame): self.payload = payload @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) f.payload = payload return f @@ -517,8 +516,8 @@ class GoAwayFrame(Frame): self.data = data @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) f.last_stream, f.error_code = struct.unpack("!LL", payload[:8]) f.last_stream &= 0x7FFFFFFF @@ -558,8 +557,8 @@ class WindowUpdateFrame(Frame): self.window_size_increment = window_size_increment @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) f.window_size_increment = struct.unpack("!L", payload)[0] f.window_size_increment &= 0x7FFFFFFF @@ -592,8 +591,8 @@ class ContinuationFrame(Frame): self.header_block_fragment = header_block_fragment @classmethod - def from_bytes(self, state, length, flags, stream_id, payload): - f = self(state=state, length=length, flags=flags, stream_id=stream_id) + def from_bytes(cls, state, length, flags, stream_id, payload): + f = cls(state=state, length=length, flags=flags, stream_id=stream_id) f.header_block_fragment = payload return f diff --git a/netlib/http_cookies.py b/netlib/http_cookies.py index 5cb39e5c..b7311714 100644 --- a/netlib/http_cookies.py +++ b/netlib/http_cookies.py @@ -158,7 +158,7 @@ def _parse_set_cookie_pairs(s): return pairs -def parse_set_cookie_header(str): +def parse_set_cookie_header(line): """ Parse a Set-Cookie header value @@ -166,7 +166,7 @@ def parse_set_cookie_header(str): ODictCaseless set of attributes. No attempt is made to parse attribute values - they are treated purely as strings. """ - pairs = _parse_set_cookie_pairs(str) + pairs = _parse_set_cookie_pairs(line) if pairs: return pairs[0][0], pairs[0][1], odict.ODictCaseless(pairs[1:]) @@ -180,12 +180,12 @@ def format_set_cookie_header(name, value, attrs): return _format_set_cookie_pairs(pairs) -def parse_cookie_header(str): +def parse_cookie_header(line): """ Parse a Cookie header value. Returns a (possibly empty) ODict object. """ - pairs, off = _read_pairs(str) + pairs, off = _read_pairs(line) return odict.ODict(pairs) diff --git a/netlib/http_uastrings.py b/netlib/http_uastrings.py index d9869531..c1ef557c 100644 --- a/netlib/http_uastrings.py +++ b/netlib/http_uastrings.py @@ -5,40 +5,42 @@ from __future__ import (absolute_import, print_function, division) kept reasonably current to reflect common usage. """ +# pylint: line-too-long + # A collection of (name, shortcut, string) tuples. UASTRINGS = [ ("android", "a", - "Mozilla/5.0 (Linux; U; Android 4.1.1; en-gb; Nexus 7 Build/JRO03D) AFL/01.04.02"), + "Mozilla/5.0 (Linux; U; Android 4.1.1; en-gb; Nexus 7 Build/JRO03D) AFL/01.04.02"), # noqa ("blackberry", "l", - "Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.346 Mobile Safari/534.11+"), + "Mozilla/5.0 (BlackBerry; U; BlackBerry 9900; en) AppleWebKit/534.11+ (KHTML, like Gecko) Version/7.1.0.346 Mobile Safari/534.11+"), # noqa ("bingbot", "b", - "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"), + "Mozilla/5.0 (compatible; bingbot/2.0; +http://www.bing.com/bingbot.htm)"), # noqa ("chrome", "c", - "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"), + "Mozilla/5.0 (Windows NT 6.1; WOW64) AppleWebKit/537.1 (KHTML, like Gecko) Chrome/22.0.1207.1 Safari/537.1"), # noqa ("firefox", "f", - "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:14.0) Gecko/20120405 Firefox/14.0a1"), + "Mozilla/5.0 (Windows NT 6.1; Win64; x64; rv:14.0) Gecko/20120405 Firefox/14.0a1"), # noqa ("googlebot", "g", - "Googlebot/2.1 (+http://www.googlebot.com/bot.html)"), + "Googlebot/2.1 (+http://www.googlebot.com/bot.html)"), # noqa ("ie9", "i", - "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))"), + "Mozilla/5.0 (Windows; U; MSIE 9.0; WIndows NT 9.0; en-US))"), # noqa ("ipad", "p", - "Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3"), + "Mozilla/5.0 (iPad; CPU OS 5_1 like Mac OS X) AppleWebKit/534.46 (KHTML, like Gecko ) Version/5.1 Mobile/9B176 Safari/7534.48.3"), # noqa ("iphone", "h", - "Mozilla/5.0 (iPhone; CPU iPhone OS 4_2_1 like Mac OS X) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148a Safari/6533.18.5", - ), + "Mozilla/5.0 (iPhone; CPU iPhone OS 4_2_1 like Mac OS X) AppleWebKit/533.17.9 (KHTML, like Gecko) Version/5.0.2 Mobile/8C148a Safari/6533.18.5"), # noqa ("safari", "s", - "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10")] + "Mozilla/5.0 (Macintosh; Intel Mac OS X 10_7_3) AppleWebKit/534.55.3 (KHTML, like Gecko) Version/5.1.3 Safari/534.53.10"), # noqa +] def get_by_shortcut(s): diff --git a/netlib/tcp.py b/netlib/tcp.py index 953cef6e..807015c8 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -297,7 +297,7 @@ def close_socket(sock): """ try: # We already indicate that we close our end. - # may raise "Transport endpoint is not connected" on Linux + # may raise "Transport endpoint is not connected" on Linux sock.shutdown(socket.SHUT_WR) # Section 4.2.2.13 of RFC 1122 tells us that a close() with any pending @@ -368,10 +368,6 @@ class _Connection(object): except SSL.Error: pass - """ - Creates an SSL Context. - """ - def _create_ssl_context(self, method=SSLv23_METHOD, options=(OP_NO_SSLv2 | OP_NO_SSLv3), @@ -383,6 +379,8 @@ class _Connection(object): alpn_select=None, ): """ + Creates an SSL Context. + :param method: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, TLSv1_METHOD, TLSv1_1_METHOD, or TLSv1_2_METHOD :param options: A bit field consisting of OpenSSL.SSL.OP_* values :param verify_options: A bit field consisting of OpenSSL.SSL.VERIFY_* values diff --git a/netlib/utils.py b/netlib/utils.py index 9c5404e6..ac42bd53 100644 --- a/netlib/utils.py +++ b/netlib/utils.py @@ -67,7 +67,7 @@ def getbit(byte, offset): return True -class BiDi: +class BiDi(object): """ A wee utility class for keeping bi-directional mappings, like field diff --git a/netlib/websockets.py b/netlib/websockets.py index 346adf1b..c45db4df 100644 --- a/netlib/websockets.py +++ b/netlib/websockets.py @@ -35,7 +35,7 @@ OPCODE = utils.BiDi( ) -class Masker: +class Masker(object): """ Data sent from the server must be masked to prevent malicious clients @@ -94,15 +94,15 @@ def server_handshake_headers(key): ) -def make_length_code(len): +def make_length_code(length): """ A websockets frame contains an initial length_code, and an optional extended length code to represent the actual length if length code is larger than 125 """ - if len <= 125: - return len - elif len >= 126 and len <= 65535: + if length <= 125: + return length + elif length >= 126 and length <= 65535: return 126 else: return 127 @@ -129,7 +129,7 @@ def create_server_nonce(client_nonce): DEFAULT = object() -class FrameHeader: +class FrameHeader(object): def __init__( self, @@ -216,7 +216,7 @@ class FrameHeader: return b @classmethod - def from_file(klass, fp): + def from_file(cls, fp): """ read a websockets frame header """ @@ -248,7 +248,7 @@ class FrameHeader: else: masking_key = None - return klass( + return cls( fin=fin, rsv1=rsv1, rsv2=rsv2, -- cgit v1.2.3 From 6e301f37d0597d86008c440f62526f906f0ae9f4 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 18 Jun 2015 12:18:22 +1200 Subject: Only set OP_NO_COMPRESSION by default if it exists in our version of OpenSSL We'll need to start testing under both new and old versions of OpenSSL somehow to catch these... --- netlib/tcp.py | 13 ++++++++++++- 1 file changed, 12 insertions(+), 1 deletion(-) (limited to 'netlib') diff --git a/netlib/tcp.py b/netlib/tcp.py index a1d1fe62..52ebc3c0 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -22,6 +22,17 @@ TLSv1_METHOD = SSL.TLSv1_METHOD TLSv1_1_METHOD = SSL.TLSv1_1_METHOD TLSv1_2_METHOD = SSL.TLSv1_2_METHOD + +SSL_DEFAULT_OPTIONS = ( + SSL.OP_NO_SSLv2 | + SSL.OP_NO_SSLv3 | + SSL.OP_CIPHER_SERVER_PREFERENCE +) + +if hasattr(SSL, "OP_NO_COMPRESSION"): + SSL_DEFAULT_OPTIONS |= SSL.OP_NO_COMPRESSION + + class NetLibError(Exception): pass @@ -365,7 +376,7 @@ class _Connection(object): def _create_ssl_context(self, method=SSLv23_METHOD, - options=(SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3 | SSL.OP_CIPHER_SERVER_PREFERENCE | SSL.OP_NO_COMPRESSION), + options=SSL_DEFAULT_OPTIONS, verify_options=SSL.VERIFY_NONE, ca_path=None, ca_pemfile=None, -- cgit v1.2.3