From 5288aa36403bc4b350700a0bf97adc4413f2a398 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 27 May 2015 12:58:55 +0200 Subject: add human_readable() to each frame for debugging --- netlib/h2/frame.py | 75 +++++++++++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 74 insertions(+), 1 deletion(-) (limited to 'netlib') diff --git a/netlib/h2/frame.py b/netlib/h2/frame.py index a7e81f48..51de7d4d 100644 --- a/netlib/h2/frame.py +++ b/netlib/h2/frame.py @@ -25,6 +25,7 @@ class Frame(object): raise ValueError('invalid flags detected.') self.length = length + self.type = self.TYPE self.flags = flags self.stream_id = stream_id @@ -49,10 +50,27 @@ class Frame(object): return b + def payload_bytes(self): # pragma: no cover + raise NotImplementedError() + + def payload_human_readable(self): # pragma: no cover + raise NotImplementedError() + + def human_readable(self): + return "\n".join([ + "============================================================", + "length: %d bytes" % self.length, + "type: %s (%#x)" % (self.__class__.__name__, self.TYPE), + "flags: %#x" % self.flags, + "stream_id: %#x" % self.stream_id, + "------------------------------------------------------------", + self.payload_human_readable(), + "============================================================", + ]) + def __eq__(self, other): return self.to_bytes() == other.to_bytes() - class DataFrame(Frame): TYPE = 0x0 VALID_FLAGS = [Frame.FLAG_END_STREAM, Frame.FLAG_PADDED] @@ -89,6 +107,8 @@ class DataFrame(Frame): return b + def payload_human_readable(self): + return "payload: %s" % str(self.payload) class HeadersFrame(Frame): TYPE = 0x1 @@ -139,6 +159,19 @@ class HeadersFrame(Frame): return b + def payload_human_readable(self): + s = [] + + if self.flags & self.FLAG_PRIORITY: + s.append("exclusive: %d" % self.exclusive) + s.append("stream dependency: %#x" % self.stream_dependency) + s.append("weight: %d" % self.weight) + + if self.flags & self.FLAG_PADDED: + s.append("padding: %d" % self.pad_length) + + s.append("header_block_fragment: %s" % str(self.header_block_fragment)) + return "\n".join(s) class PriorityFrame(Frame): TYPE = 0x2 @@ -169,6 +202,12 @@ class PriorityFrame(Frame): return struct.pack('!LB', (int(self.exclusive) << 31) | self.stream_dependency, self.weight) + def payload_human_readable(self): + s = [] + s.append("exclusive: %d" % self.exclusive) + s.append("stream dependency: %#x" % self.stream_dependency) + s.append("weight: %d" % self.weight) + return "\n".join(s) class RstStreamFrame(Frame): TYPE = 0x3 @@ -190,6 +229,8 @@ class RstStreamFrame(Frame): return struct.pack('!L', self.error_code) + def payload_human_readable(self): + return "error code: %#x" % self.error_code class SettingsFrame(Frame): TYPE = 0x4 @@ -228,6 +269,16 @@ class SettingsFrame(Frame): return b + def payload_human_readable(self): + s = [] + + for identifier, value in self.settings.items(): + s.append("%s: %#x" % (self.SETTINGS.get_name(identifier), value)) + + if not s: + return "settings: None" + else: + return "\n".join(s) class PushPromiseFrame(Frame): TYPE = 0x5 @@ -273,6 +324,15 @@ class PushPromiseFrame(Frame): return b + def payload_human_readable(self): + s = [] + + if self.flags & self.FLAG_PADDED: + s.append("padding: %d" % self.pad_length) + + s.append("promised stream: %#x" % self.promised_stream) + s.append("header_block_fragment: %s" % str(self.header_block_fragment)) + return "\n".join(s) class PingFrame(Frame): TYPE = 0x6 @@ -296,6 +356,8 @@ class PingFrame(Frame): b += b'\0' * (8 - len(b)) return b + def payload_human_readable(self): + return "opaque data: %s" % str(self.payload) class GoAwayFrame(Frame): TYPE = 0x7 @@ -325,6 +387,12 @@ class GoAwayFrame(Frame): b += bytes(self.data) return b + def payload_human_readable(self): + s = [] + s.append("last stream: %#x" % self.last_stream) + s.append("error code: %d" % self.error_code) + s.append("debug data: %s" % str(self.data)) + return "\n".join(s) class WindowUpdateFrame(Frame): TYPE = 0x8 @@ -349,6 +417,8 @@ class WindowUpdateFrame(Frame): return struct.pack('!L', self.window_size_increment & 0x7FFFFFFF) + def payload_human_readable(self): + return "window size increment: %#x" % self.window_size_increment class ContinuationFrame(Frame): TYPE = 0x9 @@ -370,6 +440,9 @@ class ContinuationFrame(Frame): return self.header_block_fragment + def payload_human_readable(self): + return "header_block_fragment: %s" % str(self.header_block_fragment) + _FRAME_CLASSES = [ DataFrame, HeadersFrame, -- cgit v1.2.3 From 754f929187e3954eb05971e38bcd3358d3a5e3be Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 27 May 2015 17:31:18 +0200 Subject: fix default argument Python evaluates default args during method definition. So you get the same dict each time you call this method. Therefore the dict is the SAME actual object each time. --- netlib/h2/frame.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'netlib') diff --git a/netlib/h2/frame.py b/netlib/h2/frame.py index 51de7d4d..ed6af200 100644 --- a/netlib/h2/frame.py +++ b/netlib/h2/frame.py @@ -245,8 +245,12 @@ class SettingsFrame(Frame): SETTINGS_MAX_HEADER_LIST_SIZE=0x6, ) - def __init__(self, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, settings={}): + def __init__(self, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, settings=None): super(SettingsFrame, self).__init__(length, flags, stream_id) + + if settings is None: + settings = {} + self.settings = settings @classmethod -- cgit v1.2.3 From 4c469fdee1b5b01a7e847a75fbbd902dc3bfbd70 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 27 May 2015 17:53:06 +0200 Subject: add hpack to encode and decode headers --- netlib/h2/frame.py | 41 ++++++++++++++++++++++++++++++++--------- 1 file changed, 32 insertions(+), 9 deletions(-) (limited to 'netlib') diff --git a/netlib/h2/frame.py b/netlib/h2/frame.py index ed6af200..179634b0 100644 --- a/netlib/h2/frame.py +++ b/netlib/h2/frame.py @@ -1,4 +1,6 @@ import struct +import io +from hpack.hpack import Encoder, Decoder from .. import utils from functools import reduce @@ -71,6 +73,7 @@ class Frame(object): def __eq__(self, other): return self.to_bytes() == other.to_bytes() + class DataFrame(Frame): TYPE = 0x0 VALID_FLAGS = [Frame.FLAG_END_STREAM, Frame.FLAG_PADDED] @@ -110,14 +113,18 @@ class DataFrame(Frame): def payload_human_readable(self): return "payload: %s" % str(self.payload) + class HeadersFrame(Frame): TYPE = 0x1 VALID_FLAGS = [Frame.FLAG_END_STREAM, Frame.FLAG_END_HEADERS, Frame.FLAG_PADDED, Frame.FLAG_PRIORITY] - def __init__(self, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, header_block_fragment=b'', - pad_length=0, exclusive=False, stream_dependency=0x0, weight=0): + def __init__(self, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, headers=None, pad_length=0, exclusive=False, stream_dependency=0x0, weight=0): super(HeadersFrame, self).__init__(length, flags, stream_id) - self.header_block_fragment = header_block_fragment + + if headers is None: + headers = [] + + self.headers = headers self.pad_length = pad_length self.exclusive = exclusive self.stream_dependency = stream_dependency @@ -129,15 +136,18 @@ class HeadersFrame(Frame): if f.flags & self.FLAG_PADDED: f.pad_length = struct.unpack('!B', payload[0])[0] - f.header_block_fragment = payload[1:-f.pad_length] + header_block_fragment = payload[1:-f.pad_length] else: - f.header_block_fragment = payload[0:] + header_block_fragment = payload[0:] if f.flags & self.FLAG_PRIORITY: - f.stream_dependency, f.weight = struct.unpack('!LB', f.header_block_fragment[:5]) + f.stream_dependency, f.weight = struct.unpack('!LB', header_block_fragment[:5]) f.exclusive = bool(f.stream_dependency >> 31) f.stream_dependency &= 0x7FFFFFFF - f.header_block_fragment = f.header_block_fragment[5:] + header_block_fragment = header_block_fragment[5:] + + for header, value in Decoder().decode(header_block_fragment): + f.headers.append((header, value)) return f @@ -152,7 +162,7 @@ class HeadersFrame(Frame): if self.flags & self.FLAG_PRIORITY: b += struct.pack('!LB', (int(self.exclusive) << 31) | self.stream_dependency, self.weight) - b += bytes(self.header_block_fragment) + b += Encoder().encode(self.headers) if self.flags & self.FLAG_PADDED: b += b'\0' * self.pad_length @@ -170,9 +180,15 @@ class HeadersFrame(Frame): if self.flags & self.FLAG_PADDED: s.append("padding: %d" % self.pad_length) - s.append("header_block_fragment: %s" % str(self.header_block_fragment)) + if not self.headers: + s.append("headers: None") + else: + for header, value in self.headers: + s.append("%s: %s" % (header, value)) + return "\n".join(s) + class PriorityFrame(Frame): TYPE = 0x2 VALID_FLAGS = [] @@ -209,6 +225,7 @@ class PriorityFrame(Frame): s.append("weight: %d" % self.weight) return "\n".join(s) + class RstStreamFrame(Frame): TYPE = 0x3 VALID_FLAGS = [] @@ -232,6 +249,7 @@ class RstStreamFrame(Frame): def payload_human_readable(self): return "error code: %#x" % self.error_code + class SettingsFrame(Frame): TYPE = 0x4 VALID_FLAGS = [Frame.FLAG_ACK] @@ -284,6 +302,7 @@ class SettingsFrame(Frame): else: return "\n".join(s) + class PushPromiseFrame(Frame): TYPE = 0x5 VALID_FLAGS = [Frame.FLAG_END_HEADERS, Frame.FLAG_PADDED] @@ -338,6 +357,7 @@ class PushPromiseFrame(Frame): s.append("header_block_fragment: %s" % str(self.header_block_fragment)) return "\n".join(s) + class PingFrame(Frame): TYPE = 0x6 VALID_FLAGS = [Frame.FLAG_ACK] @@ -363,6 +383,7 @@ class PingFrame(Frame): def payload_human_readable(self): return "opaque data: %s" % str(self.payload) + class GoAwayFrame(Frame): TYPE = 0x7 VALID_FLAGS = [] @@ -398,6 +419,7 @@ class GoAwayFrame(Frame): s.append("debug data: %s" % str(self.data)) return "\n".join(s) + class WindowUpdateFrame(Frame): TYPE = 0x8 VALID_FLAGS = [] @@ -424,6 +446,7 @@ class WindowUpdateFrame(Frame): def payload_human_readable(self): return "window size increment: %#x" % self.window_size_increment + class ContinuationFrame(Frame): TYPE = 0x9 VALID_FLAGS = [Frame.FLAG_END_HEADERS] -- cgit v1.2.3 From d50b9be0d5dab1772f0edcbfa89542ef9425e7bf Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 27 May 2015 17:53:45 +0200 Subject: add generic frame parsing method --- netlib/h2/frame.py | 17 +++++++++++++++++ 1 file changed, 17 insertions(+) (limited to 'netlib') diff --git a/netlib/h2/frame.py b/netlib/h2/frame.py index 179634b0..11687316 100644 --- a/netlib/h2/frame.py +++ b/netlib/h2/frame.py @@ -31,6 +31,23 @@ class Frame(object): self.flags = flags self.stream_id = stream_id + @classmethod + def from_file(self, fp): + """ + read a HTTP/2 frame sent by a server or client + fp is a "file like" object that could be backed by a network + stream or a disk or an in memory stream reader + """ + raw_header = fp.safe_read(9) + + fields = struct.unpack("!HBBBL", raw_header) + length = (fields[0] << 8) + fields[1] + flags = fields[3] + stream_id = fields[4] + + payload = fp.safe_read(length) + return FRAMES[fields[2]].from_bytes(length, flags, stream_id, payload) + @classmethod def from_bytes(self, data): fields = struct.unpack("!HBBBL", data[:9]) -- cgit v1.2.3 From 780836b182cd982b978f16218299f2b77a8ed204 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 28 May 2015 17:46:44 +0200 Subject: add ALPN support to TCP abstraction --- netlib/tcp.py | 35 +++++++++++++++++++++++++++-------- netlib/test.py | 3 ++- 2 files changed, 29 insertions(+), 9 deletions(-) (limited to 'netlib') diff --git a/netlib/tcp.py b/netlib/tcp.py index 49f92e4a..fc2c144e 100644 --- a/netlib/tcp.py +++ b/netlib/tcp.py @@ -360,7 +360,9 @@ class _Connection(object): def _create_ssl_context(self, method=SSLv23_METHOD, options=(OP_NO_SSLv2 | OP_NO_SSLv3), - cipher_list=None + cipher_list=None, + alpn_protos=None, + alpn_select=None, ): """ :param method: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, TLSv1_METHOD or TLSv1_1_METHOD @@ -389,6 +391,17 @@ class _Connection(object): if log_ssl_key: context.set_info_callback(log_ssl_key) + # advertise application layer protocols + if alpn_protos is not None: + context.set_alpn_protos(alpn_protos) + + # select application layer protocol + if alpn_select is not None: + def alpn_select_f(conn, options): + return bytes(alpn_select) + + context.set_alpn_select_callback(alpn_select_f) + return context @@ -413,8 +426,8 @@ class TCPClient(_Connection): self.ssl_established = False self.sni = None - def create_ssl_context(self, cert=None, **sslctx_kwargs): - context = self._create_ssl_context(**sslctx_kwargs) + def create_ssl_context(self, cert=None, alpn_protos=None, **sslctx_kwargs): + context = self._create_ssl_context(alpn_protos=alpn_protos, **sslctx_kwargs) # Client Certs if cert: try: @@ -424,13 +437,13 @@ class TCPClient(_Connection): raise NetLibError("SSL client certificate error: %s" % str(v)) return context - def convert_to_ssl(self, sni=None, **sslctx_kwargs): + def convert_to_ssl(self, sni=None, alpn_protos=None, **sslctx_kwargs): """ cert: Path to a file containing both client cert and private key. options: A bit field consisting of OpenSSL.SSL.OP_* values """ - context = self.create_ssl_context(**sslctx_kwargs) + context = self.create_ssl_context(alpn_protos=alpn_protos, **sslctx_kwargs) self.connection = SSL.Connection(context, self.connection) if sni: self.sni = sni @@ -465,6 +478,9 @@ class TCPClient(_Connection): def gettimeout(self): return self.connection.gettimeout() + def get_alpn_proto_negotiated(self): + return self.connection.get_alpn_proto_negotiated() + class BaseHandler(_Connection): @@ -492,6 +508,7 @@ class BaseHandler(_Connection): request_client_cert=None, chain_file=None, dhparams=None, + alpn_select=None, **sslctx_kwargs): """ cert: A certutils.SSLCert object. @@ -517,7 +534,8 @@ class BaseHandler(_Connection): we may be able to make the proper behaviour the default again, but until then we're conservative. """ - context = self._create_ssl_context(**sslctx_kwargs) + + context = self._create_ssl_context(alpn_select=alpn_select, **sslctx_kwargs) context.use_privatekey(key) context.use_certificate(cert.x509) @@ -542,12 +560,13 @@ class BaseHandler(_Connection): return context - def convert_to_ssl(self, cert, key, **sslctx_kwargs): + def convert_to_ssl(self, cert, key, alpn_select=None, **sslctx_kwargs): """ Convert connection to SSL. For a list of parameters, see BaseHandler._create_ssl_context(...) """ - context = self.create_ssl_context(cert, key, **sslctx_kwargs) + + context = self.create_ssl_context(cert, key, alpn_select=alpn_select, **sslctx_kwargs) self.connection = SSL.Connection(context, self.connection) self.connection.set_accept_state() try: diff --git a/netlib/test.py b/netlib/test.py index b6f94273..63b493a9 100644 --- a/netlib/test.py +++ b/netlib/test.py @@ -82,7 +82,8 @@ class TServer(tcp.TCPServer): request_client_cert=self.ssl["request_client_cert"], cipher_list=self.ssl.get("cipher_list", None), dhparams=self.ssl.get("dhparams", None), - chain_file=self.ssl.get("chain_file", None) + chain_file=self.ssl.get("chain_file", None), + alpn_select=self.ssl.get("alpn_select", None) ) h.handle() h.finish() -- cgit v1.2.3 From e2de49596d0e60e343c71c73e0847b17fb27ac3c Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 28 May 2015 17:46:30 +0200 Subject: add HTTP/2-capable client --- netlib/h2/h2.py | 65 +++++++++++++++++++++++++++++++++++++++++++++++++++++++++ 1 file changed, 65 insertions(+) (limited to 'netlib') diff --git a/netlib/h2/h2.py b/netlib/h2/h2.py index 7a85226f..bfe5832b 100644 --- a/netlib/h2/h2.py +++ b/netlib/h2/h2.py @@ -1,3 +1,5 @@ +from .. import utils, odict, tcp +from frame import * # "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n" CLIENT_CONNECTION_PREFACE = '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a' @@ -18,3 +20,66 @@ ERROR_CODES = utils.BiDi( INADEQUATE_SECURITY=0xc, HTTP_1_1_REQUIRED=0xd ) + + +class H2Client(tcp.TCPClient): + ALPN_PROTO_H2 = b'h2' + + DEFAULT_SETTINGS = { + SettingsFrame.SETTINGS.SETTINGS_HEADER_TABLE_SIZE: 4096, + SettingsFrame.SETTINGS.SETTINGS_ENABLE_PUSH: 1, + SettingsFrame.SETTINGS.SETTINGS_MAX_CONCURRENT_STREAMS: None, + SettingsFrame.SETTINGS.SETTINGS_INITIAL_WINDOW_SIZE: 2 ^ 16 - 1, + SettingsFrame.SETTINGS.SETTINGS_MAX_FRAME_SIZE: 2 ^ 14, + SettingsFrame.SETTINGS.SETTINGS_MAX_HEADER_LIST_SIZE: None, + } + + def __init__(self, address, source_address=None): + super(H2Client, self).__init__(address, source_address) + self.settings = self.DEFAULT_SETTINGS.copy() + + def connect(self, send_preface=True): + super(H2Client, self).connect() + self.convert_to_ssl(alpn_protos=[self.ALPN_PROTO_H2]) + + alp = self.get_alpn_proto_negotiated() + if alp != b'h2': + raise NotImplementedError("H2Client can not handle unknown protocol: %s" % alp) + print "-> Successfully negotiated 'h2' application layer protocol." + + if send_preface: + self.wfile.write(bytes(CLIENT_CONNECTION_PREFACE.decode('hex'))) + self.send_frame(SettingsFrame()) + + frame = Frame.from_file(self.rfile) + print frame.human_readable() + assert isinstance(frame, SettingsFrame) + self.apply_settings(frame.settings) + + print "-> Connection Preface completed." + + print "-> H2Client is ready..." + + def send_frame(self, frame): + self.wfile.write(frame.to_bytes()) + self.wfile.flush() + + def read_frame(self): + frame = Frame.from_file(self.rfile) + if isinstance(frame, SettingsFrame): + self.apply_settings(frame.settings) + + return frame + + def apply_settings(self, settings): + for setting, value in settings.items(): + old_value = self.settings[setting] + if not old_value: + old_value = '-' + + self.settings[setting] = value + print "-> Setting changed: %s to %d (was %s)" % + (SettingsFrame.SETTINGS.get_name(setting), value, str(old_value)) + + self.send_frame(SettingsFrame(flags=Frame.FLAG_ACK)) + print "-> New settings acknowledged." -- cgit v1.2.3 From c32d8189faa24cbe016bb3c859f64c816e0871fe Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 29 May 2015 16:59:50 +0200 Subject: cleanup imports --- netlib/h2/frame.py | 1 - 1 file changed, 1 deletion(-) (limited to 'netlib') diff --git a/netlib/h2/frame.py b/netlib/h2/frame.py index 11687316..d4294052 100644 --- a/netlib/h2/frame.py +++ b/netlib/h2/frame.py @@ -1,5 +1,4 @@ import struct -import io from hpack.hpack import Encoder, Decoder from .. import utils -- cgit v1.2.3