Merge pull request #62 from Kriechi/h2-client

H2 client

4 files changed, 219 insertions, 18 deletions

diff --git a/netlib/h2/frame.py b/netlib/h2/frame.py
index a7e81f48..d4294052 100644
--- a/netlib/h2/frame.py
+++ b/netlib/h2/frame.py
@@ -1,4 +1,5 @@
 import struct
+from hpack.hpack import Encoder, Decoder
 
 from .. import utils
 from functools import reduce
@@ -25,10 +26,28 @@ class Frame(object):
             raise ValueError('invalid flags detected.')
 
         self.length = length
+        self.type = self.TYPE
         self.flags = flags
         self.stream_id = stream_id
 
     @classmethod
+    def from_file(self, fp):
+        """
+          read a HTTP/2 frame sent by a server or client
+          fp is a "file like" object that could be backed by a network
+          stream or a disk or an in memory stream reader
+        """
+        raw_header = fp.safe_read(9)
+
+        fields = struct.unpack("!HBBBL", raw_header)
+        length = (fields[0] << 8) + fields[1]
+        flags = fields[3]
+        stream_id = fields[4]
+
+        payload = fp.safe_read(length)
+        return FRAMES[fields[2]].from_bytes(length, flags, stream_id, payload)
+
+    @classmethod
     def from_bytes(self, data):
         fields = struct.unpack("!HBBBL", data[:9])
         length = (fields[0] << 8) + fields[1]
@@ -49,6 +68,24 @@ class Frame(object):
 
         return b
 
+    def payload_bytes(self):  # pragma: no cover
+        raise NotImplementedError()
+
+    def payload_human_readable(self):  # pragma: no cover
+        raise NotImplementedError()
+
+    def human_readable(self):
+        return "\n".join([
+            "============================================================",
+            "length:    %d bytes" % self.length,
+            "type:      %s (%#x)" % (self.__class__.__name__, self.TYPE),
+            "flags:     %#x" % self.flags,
+            "stream_id: %#x" % self.stream_id,
+            "------------------------------------------------------------",
+            self.payload_human_readable(),
+            "============================================================",
+        ])
+
     def __eq__(self, other):
         return self.to_bytes() == other.to_bytes()
 
@@ -89,15 +126,21 @@ class DataFrame(Frame):
 
         return b
 
+    def payload_human_readable(self):
+        return "payload: %s" % str(self.payload)
+
 
 class HeadersFrame(Frame):
     TYPE = 0x1
     VALID_FLAGS = [Frame.FLAG_END_STREAM, Frame.FLAG_END_HEADERS, Frame.FLAG_PADDED, Frame.FLAG_PRIORITY]
 
-    def __init__(self, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, header_block_fragment=b'',
-                 pad_length=0, exclusive=False, stream_dependency=0x0, weight=0):
+    def __init__(self, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, headers=None, pad_length=0, exclusive=False, stream_dependency=0x0, weight=0):
         super(HeadersFrame, self).__init__(length, flags, stream_id)
-        self.header_block_fragment = header_block_fragment
+
+        if headers is None:
+            headers = []
+
+        self.headers = headers
         self.pad_length = pad_length
         self.exclusive = exclusive
         self.stream_dependency = stream_dependency
@@ -109,15 +152,18 @@ class HeadersFrame(Frame):
 
         if f.flags & self.FLAG_PADDED:
             f.pad_length = struct.unpack('!B', payload[0])[0]
-            f.header_block_fragment = payload[1:-f.pad_length]
+            header_block_fragment = payload[1:-f.pad_length]
         else:
-            f.header_block_fragment = payload[0:]
+            header_block_fragment = payload[0:]
 
         if f.flags & self.FLAG_PRIORITY:
-            f.stream_dependency, f.weight = struct.unpack('!LB', f.header_block_fragment[:5])
+            f.stream_dependency, f.weight = struct.unpack('!LB', header_block_fragment[:5])
             f.exclusive = bool(f.stream_dependency >> 31)
             f.stream_dependency &= 0x7FFFFFFF
-            f.header_block_fragment = f.header_block_fragment[5:]
+            header_block_fragment = header_block_fragment[5:]
+
+        for header, value in Decoder().decode(header_block_fragment):
+            f.headers.append((header, value))
 
         return f
 
@@ -132,13 +178,32 @@ class HeadersFrame(Frame):
         if self.flags & self.FLAG_PRIORITY:
             b += struct.pack('!LB', (int(self.exclusive) << 31) | self.stream_dependency, self.weight)
 
-        b += bytes(self.header_block_fragment)
+        b += Encoder().encode(self.headers)
 
         if self.flags & self.FLAG_PADDED:
             b += b'\0' * self.pad_length
 
         return b
 
+    def payload_human_readable(self):
+        s = []
+
+        if self.flags & self.FLAG_PRIORITY:
+            s.append("exclusive: %d" % self.exclusive)
+            s.append("stream dependency: %#x" % self.stream_dependency)
+            s.append("weight: %d" % self.weight)
+
+        if self.flags & self.FLAG_PADDED:
+            s.append("padding: %d" % self.pad_length)
+
+        if not self.headers:
+            s.append("headers: None")
+        else:
+            for header, value in self.headers:
+                s.append("%s: %s" % (header, value))
+
+        return "\n".join(s)
+
 
 class PriorityFrame(Frame):
     TYPE = 0x2
@@ -169,6 +234,13 @@ class PriorityFrame(Frame):
 
         return struct.pack('!LB', (int(self.exclusive) << 31) | self.stream_dependency, self.weight)
 
+    def payload_human_readable(self):
+        s = []
+        s.append("exclusive: %d" % self.exclusive)
+        s.append("stream dependency: %#x" % self.stream_dependency)
+        s.append("weight: %d" % self.weight)
+        return "\n".join(s)
+
 
 class RstStreamFrame(Frame):
     TYPE = 0x3
@@ -190,6 +262,9 @@ class RstStreamFrame(Frame):
 
         return struct.pack('!L', self.error_code)
 
+    def payload_human_readable(self):
+        return "error code: %#x" % self.error_code
+
 
 class SettingsFrame(Frame):
     TYPE = 0x4
@@ -204,8 +279,12 @@ class SettingsFrame(Frame):
         SETTINGS_MAX_HEADER_LIST_SIZE=0x6,
     )
 
-    def __init__(self, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, settings={}):
+    def __init__(self, length=0, flags=Frame.FLAG_NO_FLAGS, stream_id=0x0, settings=None):
         super(SettingsFrame, self).__init__(length, flags, stream_id)
+
+        if settings is None:
+            settings = {}
+
         self.settings = settings
 
     @classmethod
@@ -228,6 +307,17 @@ class SettingsFrame(Frame):
 
         return b
 
+    def payload_human_readable(self):
+        s = []
+
+        for identifier, value in self.settings.items():
+            s.append("%s: %#x" % (self.SETTINGS.get_name(identifier), value))
+
+        if not s:
+            return "settings: None"
+        else:
+            return "\n".join(s)
+
 
 class PushPromiseFrame(Frame):
     TYPE = 0x5
@@ -273,6 +363,16 @@ class PushPromiseFrame(Frame):
 
         return b
 
+    def payload_human_readable(self):
+        s = []
+
+        if self.flags & self.FLAG_PADDED:
+            s.append("padding: %d" % self.pad_length)
+
+        s.append("promised stream: %#x" % self.promised_stream)
+        s.append("header_block_fragment: %s" % str(self.header_block_fragment))
+        return "\n".join(s)
+
 
 class PingFrame(Frame):
     TYPE = 0x6
@@ -296,6 +396,9 @@ class PingFrame(Frame):
         b += b'\0' * (8 - len(b))
         return b
 
+    def payload_human_readable(self):
+        return "opaque data: %s" % str(self.payload)
+
 
 class GoAwayFrame(Frame):
     TYPE = 0x7
@@ -325,6 +428,13 @@ class GoAwayFrame(Frame):
         b += bytes(self.data)
         return b
 
+    def payload_human_readable(self):
+        s = []
+        s.append("last stream: %#x" % self.last_stream)
+        s.append("error code: %d" % self.error_code)
+        s.append("debug data: %s" % str(self.data))
+        return "\n".join(s)
+
 
 class WindowUpdateFrame(Frame):
     TYPE = 0x8
@@ -349,6 +459,9 @@ class WindowUpdateFrame(Frame):
 
         return struct.pack('!L', self.window_size_increment & 0x7FFFFFFF)
 
+    def payload_human_readable(self):
+        return "window size increment: %#x" % self.window_size_increment
+
 
 class ContinuationFrame(Frame):
     TYPE = 0x9
@@ -370,6 +483,9 @@ class ContinuationFrame(Frame):
 
         return self.header_block_fragment
 
+    def payload_human_readable(self):
+        return "header_block_fragment: %s" % str(self.header_block_fragment)
+
 _FRAME_CLASSES = [
     DataFrame,
     HeadersFrame,
diff --git a/netlib/h2/h2.py b/netlib/h2/h2.py
index 7a85226f..bfe5832b 100644
--- a/netlib/h2/h2.py
+++ b/netlib/h2/h2.py
@@ -1,3 +1,5 @@
+from .. import utils, odict, tcp
+from frame import *
 
 # "PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n"
 CLIENT_CONNECTION_PREFACE = '505249202a20485454502f322e300d0a0d0a534d0d0a0d0a'
@@ -18,3 +20,66 @@ ERROR_CODES = utils.BiDi(
     INADEQUATE_SECURITY=0xc,
     HTTP_1_1_REQUIRED=0xd
 )
+
+
+class H2Client(tcp.TCPClient):
+    ALPN_PROTO_H2 = b'h2'
+
+    DEFAULT_SETTINGS = {
+        SettingsFrame.SETTINGS.SETTINGS_HEADER_TABLE_SIZE: 4096,
+        SettingsFrame.SETTINGS.SETTINGS_ENABLE_PUSH: 1,
+        SettingsFrame.SETTINGS.SETTINGS_MAX_CONCURRENT_STREAMS: None,
+        SettingsFrame.SETTINGS.SETTINGS_INITIAL_WINDOW_SIZE: 2 ^ 16 - 1,
+        SettingsFrame.SETTINGS.SETTINGS_MAX_FRAME_SIZE: 2 ^ 14,
+        SettingsFrame.SETTINGS.SETTINGS_MAX_HEADER_LIST_SIZE: None,
+    }
+
+    def __init__(self, address, source_address=None):
+        super(H2Client, self).__init__(address, source_address)
+        self.settings = self.DEFAULT_SETTINGS.copy()
+
+    def connect(self, send_preface=True):
+        super(H2Client, self).connect()
+        self.convert_to_ssl(alpn_protos=[self.ALPN_PROTO_H2])
+
+        alp = self.get_alpn_proto_negotiated()
+        if alp != b'h2':
+            raise NotImplementedError("H2Client can not handle unknown protocol: %s" % alp)
+        print "-> Successfully negotiated 'h2' application layer protocol."
+
+        if send_preface:
+            self.wfile.write(bytes(CLIENT_CONNECTION_PREFACE.decode('hex')))
+            self.send_frame(SettingsFrame())
+
+            frame = Frame.from_file(self.rfile)
+            print frame.human_readable()
+            assert isinstance(frame, SettingsFrame)
+            self.apply_settings(frame.settings)
+
+            print "-> Connection Preface completed."
+
+        print "-> H2Client is ready..."
+
+    def send_frame(self, frame):
+        self.wfile.write(frame.to_bytes())
+        self.wfile.flush()
+
+    def read_frame(self):
+        frame = Frame.from_file(self.rfile)
+        if isinstance(frame, SettingsFrame):
+            self.apply_settings(frame.settings)
+
+        return frame
+
+    def apply_settings(self, settings):
+        for setting, value in settings.items():
+            old_value = self.settings[setting]
+            if not old_value:
+                old_value = '-'
+
+            self.settings[setting] = value
+            print "-> Setting changed: %s to %d (was %s)" %
+                (SettingsFrame.SETTINGS.get_name(setting), value, str(old_value))
+
+        self.send_frame(SettingsFrame(flags=Frame.FLAG_ACK))
+        print "-> New settings acknowledged."
diff --git a/netlib/tcp.py b/netlib/tcp.py
index 49f92e4a..fc2c144e 100644
--- a/netlib/tcp.py
+++ b/netlib/tcp.py
@@ -360,7 +360,9 @@ class _Connection(object):
     def _create_ssl_context(self,
                             method=SSLv23_METHOD,
                             options=(OP_NO_SSLv2 | OP_NO_SSLv3),
-                            cipher_list=None
+                            cipher_list=None,
+                            alpn_protos=None,
+                            alpn_select=None,
                             ):
         """
         :param method: One of SSLv2_METHOD, SSLv3_METHOD, SSLv23_METHOD, TLSv1_METHOD or TLSv1_1_METHOD
@@ -389,6 +391,17 @@ class _Connection(object):
         if log_ssl_key:
             context.set_info_callback(log_ssl_key)
 
+        # advertise application layer protocols
+        if alpn_protos is not None:
+            context.set_alpn_protos(alpn_protos)
+
+        # select application layer protocol
+        if alpn_select is not None:
+            def alpn_select_f(conn, options):
+                return bytes(alpn_select)
+
+            context.set_alpn_select_callback(alpn_select_f)
+
         return context
 
 
@@ -413,8 +426,8 @@ class TCPClient(_Connection):
         self.ssl_established = False
         self.sni = None
 
-    def create_ssl_context(self, cert=None, **sslctx_kwargs):
-        context = self._create_ssl_context(**sslctx_kwargs)
+    def create_ssl_context(self, cert=None, alpn_protos=None, **sslctx_kwargs):
+        context = self._create_ssl_context(alpn_protos=alpn_protos, **sslctx_kwargs)
         # Client Certs
         if cert:
             try:
@@ -424,13 +437,13 @@ class TCPClient(_Connection):
                 raise NetLibError("SSL client certificate error: %s" % str(v))
         return context
 
-    def convert_to_ssl(self, sni=None, **sslctx_kwargs):
+    def convert_to_ssl(self, sni=None, alpn_protos=None, **sslctx_kwargs):
         """
             cert: Path to a file containing both client cert and private key.
 
             options: A bit field consisting of OpenSSL.SSL.OP_* values
         """
-        context = self.create_ssl_context(**sslctx_kwargs)
+        context = self.create_ssl_context(alpn_protos=alpn_protos, **sslctx_kwargs)
         self.connection = SSL.Connection(context, self.connection)
         if sni:
             self.sni = sni
@@ -465,6 +478,9 @@ class TCPClient(_Connection):
     def gettimeout(self):
         return self.connection.gettimeout()
 
+    def get_alpn_proto_negotiated(self):
+        return self.connection.get_alpn_proto_negotiated()
+
 
 class BaseHandler(_Connection):
 
@@ -492,6 +508,7 @@ class BaseHandler(_Connection):
                            request_client_cert=None,
                            chain_file=None,
                            dhparams=None,
+                           alpn_select=None,
                            **sslctx_kwargs):
         """
             cert: A certutils.SSLCert object.
@@ -517,7 +534,8 @@ class BaseHandler(_Connection):
             we may be able to make the proper behaviour the default again, but
             until then we're conservative.
         """
-        context = self._create_ssl_context(**sslctx_kwargs)
+
+        context = self._create_ssl_context(alpn_select=alpn_select, **sslctx_kwargs)
 
         context.use_privatekey(key)
         context.use_certificate(cert.x509)
@@ -542,12 +560,13 @@ class BaseHandler(_Connection):
 
         return context
 
-    def convert_to_ssl(self, cert, key, **sslctx_kwargs):
+    def convert_to_ssl(self, cert, key, alpn_select=None, **sslctx_kwargs):
         """
         Convert connection to SSL.
         For a list of parameters, see BaseHandler._create_ssl_context(...)
         """
-        context = self.create_ssl_context(cert, key, **sslctx_kwargs)
+
+        context = self.create_ssl_context(cert, key, alpn_select=alpn_select, **sslctx_kwargs)
         self.connection = SSL.Connection(context, self.connection)
         self.connection.set_accept_state()
         try:
diff --git a/netlib/test.py b/netlib/test.py
index b6f94273..63b493a9 100644
--- a/netlib/test.py
+++ b/netlib/test.py
@@ -82,7 +82,8 @@ class TServer(tcp.TCPServer):
                 request_client_cert=self.ssl["request_client_cert"],
                 cipher_list=self.ssl.get("cipher_list", None),
                 dhparams=self.ssl.get("dhparams", None),
-                chain_file=self.ssl.get("chain_file", None)
+                chain_file=self.ssl.get("chain_file", None),
+                alpn_select=self.ssl.get("alpn_select", None)
             )
         h.handle()
         h.finish()