From 863113f989ee2a089c86b06a88a22e92d840348b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 24 Jul 2015 13:31:55 +0200 Subject: first initial proof-of-concept --- libmproxy/proxy/layer.py | 181 +++++++++++++++++++++++++++++++++++++++++++++ libmproxy/proxy/message.py | 40 ++++++++++ libmproxy/proxy/server.py | 69 +++++++++++++++-- 3 files changed, 282 insertions(+), 8 deletions(-) create mode 100644 libmproxy/proxy/layer.py create mode 100644 libmproxy/proxy/message.py (limited to 'libmproxy') diff --git a/libmproxy/proxy/layer.py b/libmproxy/proxy/layer.py new file mode 100644 index 00000000..500fb6ba --- /dev/null +++ b/libmproxy/proxy/layer.py @@ -0,0 +1,181 @@ +from __future__ import (absolute_import, print_function, division, unicode_literals) +from libmproxy.protocol.tcp import TCPHandler +from libmproxy.proxy.connection import ServerConnection +from netlib import tcp +from .primitives import Socks5ProxyMode, ProxyError, Log +from .message import Connect, Reconnect, ChangeServer + + +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible scenarios (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentModeLayer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + RegularModeLayer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentModeLayer + AutoLayer + SslLayer + AutoLayer + Http2Layer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can "yield" commands to lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" + + +class ProxyError2(Exception): + def __init__(self, message, cause=None): + super(ProxyError2, self).__init__(message) + self.cause = cause + + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def __getattr__(self, name): + """ + Accessing a nonexisting attribute does not throw an error but returns None instead. + """ + return None + + +class LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + def __init__(self): + if True: + return + self.config = None + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(LayerCodeCompletion): + def __init__(self, ctx): + """ + Args: + ctx: The (read-only) higher layer. + """ + super(Layer, self).__init__() + self.ctx = ctx + + def __call__(self): + """ + Logic of the layer. + Raises: + ProxyError2 in case of protocol exceptions. + """ + raise NotImplementedError + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "%s:%s: %s" % + (self.client_conn.address.host, + self.client_conn.address.port, + msg)] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + +class _ServerConnectionMixin(object): + def __init__(self): + self._server_address = None + self.server_conn = None + + def _handle_message(self, message): + if message == Reconnect: + self._disconnect() + self._connect() + return True + elif message == Connect: + self._connect() + return True + elif message == ChangeServer: + raise NotImplementedError + return False + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + self.server_conn.finish() + self.server_conn.close() + # self.channel.tell("serverdisconnect", self) + self.server_conn = None + + def _connect(self): + self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProxyError2("Server connection to '%s' failed: %s" % (self._server_address, repr(e)), e) + + def _set_address(self, address): + a = tcp.Address.wrap(address) + self.log("Set new server address: " + repr(a), "debug") + self.server_conn = ServerConnection(a) + + +class Socks5IncomingLayer(Layer, _ServerConnectionMixin): + def __call__(self): + try: + s5mode = Socks5ProxyMode(self.config.ssl_ports) + address = s5mode.get_upstream_server(self.client_conn)[2:] + except ProxyError as e: + raise ProxyError2(str(e), e) + + self._set_address(address) + + layer = TcpLayer(self) + for message in layer(): + if not self._handle_message(message): + yield message + + +class TcpLayer(Layer): + def __call__(self): + yield Connect() + tcp_handler = TCPHandler(self) + tcp_handler.handle_messages() + + def establish_server_connection(self): + print("establish server conn called") \ No newline at end of file diff --git a/libmproxy/proxy/message.py b/libmproxy/proxy/message.py new file mode 100644 index 00000000..a667123c --- /dev/null +++ b/libmproxy/proxy/message.py @@ -0,0 +1,40 @@ +""" +This module contains all valid messages layers can send to the underlying layers. +""" + + +class _Message(object): + def __eq__(self, other): + # Allow message == Connect checks. + # FIXME: make Connect == message work. + if isinstance(self, other): + return True + return self is other + + +class Connect(_Message): + """ + Connect to the server + """ + + +class Reconnect(_Message): + """ + Re-establish the server connection + """ + + +class ChangeServer(_Message): + """ + Change the upstream server. + """ + + def __init__(self, address, server_ssl, sni, depth=1): + self.address = address + self.server_ssl = server_ssl + self.sni = sni + + # upstream proxy scenario: you may want to change either the final target or the upstream proxy. + # We can express this neatly as the "nth-server-providing-layer" + # ServerConnection could get a `via` attribute. + self.depth = depth diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 2f6ee061..a45276d4 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -1,13 +1,14 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function +import traceback +import sys import socket -from OpenSSL import SSL from netlib import tcp +from . import layer from .primitives import ProxyServerError, Log, ProxyError from .connection import ClientConnection, ServerConnection from ..protocol.handle import protocol_handler -from .. import version class DummyServer: @@ -46,7 +47,7 @@ class ProxyServer(tcp.TCPServer): self.channel = channel def handle_client_connection(self, conn, client_address): - h = ConnectionHandler( + h = ConnectionHandler2( self.config, conn, client_address, @@ -56,6 +57,57 @@ class ProxyServer(tcp.TCPServer): h.finish() +class ConnectionHandler2: + # FIXME: parameter ordering + # FIXME: remove server attribute + def __init__(self, config, client_conn, client_address, server, channel): + self.config = config + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = ClientConnection( + client_conn, + client_address, + server) + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = channel + """@type: libmproxy.controller.Channel""" + + def handle(self): + self.log("clientconnect", "info") + + root_context = layer.RootContext( + self.client_conn, + self.config, + self.channel + ) + root_layer = layer.Socks5IncomingLayer(root_context) + + try: + for message in root_layer(): + print("Root layer receveived: %s" % message) + except layer.ProxyError2 as e: + self.log(e, "info") + except Exception: + self.log(traceback.format_exc(), "error") + print(traceback.format_exc(), file=sys.stderr) + print("mitmproxy has crashed!", file=sys.stderr) + print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) + + def finish(self): + self.client_conn.finish() + + def log(self, msg, level, subs=()): + # FIXME: Duplicate code + full_msg = [ + "%s:%s: %s" % + (self.client_conn.address.host, + self.client_conn.address.port, + msg)] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + class ConnectionHandler: def __init__( self, @@ -74,6 +126,7 @@ class ConnectionHandler: self.server_conn = None """@type: libmproxy.proxy.connection.ServerConnection""" self.channel = channel + """@type: libmproxy.controller.Channel""" self.conntype = "http" @@ -144,9 +197,9 @@ class ConnectionHandler: import sys self.log(traceback.format_exc(), "error") - print >> sys.stderr, traceback.format_exc() - print >> sys.stderr, "mitmproxy has crashed!" - print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy" + print(traceback.format_exc(), file=sys.stderr) + print("mitmproxy has crashed!", file=sys.stderr) + print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) finally: # Make sure that we close the server connection in any case. # The client connection is closed by the ProxyServer and does not @@ -201,7 +254,7 @@ class ConnectionHandler: "serverconnect", "debug", [ "%s:%s" % self.server_conn.address()[ - :2]]) + :2]]) if ask: self.channel.ask("serverconnect", self) try: -- cgit v1.2.3 From be995ddbd62579621ed06ed7197c8f22939c16d1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 24 Jul 2015 17:48:27 +0200 Subject: add ssl layer --- libmproxy/proxy/layer.py | 255 ++++++++++++++++++++++++++++++++++++++++-- libmproxy/proxy/message.py | 4 +- libmproxy/proxy/primitives.py | 2 +- 3 files changed, 247 insertions(+), 14 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy/layer.py b/libmproxy/proxy/layer.py index 500fb6ba..e4878bdf 100644 --- a/libmproxy/proxy/layer.py +++ b/libmproxy/proxy/layer.py @@ -1,11 +1,13 @@ from __future__ import (absolute_import, print_function, division, unicode_literals) +import Queue +import threading +import traceback from libmproxy.protocol.tcp import TCPHandler from libmproxy.proxy.connection import ServerConnection from netlib import tcp from .primitives import Socks5ProxyMode, ProxyError, Log from .message import Connect, Reconnect, ChangeServer - """ mitmproxy protocol architecture @@ -65,10 +67,11 @@ class RootContext(object): return None -class LayerCodeCompletion(object): +class _LayerCodeCompletion(object): """ Dummy class that provides type hinting in PyCharm, which simplifies development a lot. """ + def __init__(self): if True: return @@ -80,7 +83,7 @@ class LayerCodeCompletion(object): """@type: libmproxy.controller.Channel""" -class Layer(LayerCodeCompletion): +class Layer(_LayerCodeCompletion): def __init__(self, ctx): """ Args: @@ -116,11 +119,15 @@ class Layer(LayerCodeCompletion): class _ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + def __init__(self): - self._server_address = None + self.server_address = None self.server_conn = None - def _handle_message(self, message): + def _handle_server_message(self, message): if message == Reconnect: self._disconnect() self._connect() @@ -136,23 +143,24 @@ class _ServerConnectionMixin(object): """ Deletes (and closes) an existing server connection. """ - self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + self.log("serverdisconnect", "debug", [repr(self.server_address)]) self.server_conn.finish() self.server_conn.close() # self.channel.tell("serverdisconnect", self) self.server_conn = None def _connect(self): - self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + self.log("serverconnect", "debug", [repr(self.server_address)]) + self.server_conn = ServerConnection(self.server_address) try: self.server_conn.connect() except tcp.NetLibError as e: - raise ProxyError2("Server connection to '%s' failed: %s" % (self._server_address, repr(e)), e) + raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) def _set_address(self, address): a = tcp.Address.wrap(address) self.log("Set new server address: " + repr(a), "debug") - self.server_conn = ServerConnection(a) + self.server_address = address class Socks5IncomingLayer(Layer, _ServerConnectionMixin): @@ -161,13 +169,17 @@ class Socks5IncomingLayer(Layer, _ServerConnectionMixin): s5mode = Socks5ProxyMode(self.config.ssl_ports) address = s5mode.get_upstream_server(self.client_conn)[2:] except ProxyError as e: + # TODO: Unmonkeypatch raise ProxyError2(str(e), e) self._set_address(address) - layer = TcpLayer(self) + if address[1] == 443: + layer = SslLayer(self, True, True) + else: + layer = TcpLayer(self) for message in layer(): - if not self._handle_message(message): + if not self._handle_server_message(message): yield message @@ -178,4 +190,223 @@ class TcpLayer(Layer): tcp_handler.handle_messages() def establish_server_connection(self): - print("establish server conn called") \ No newline at end of file + pass + # FIXME: Remove method, currently just here to mock TCPHandler's call to it. + + +class ReconnectRequest(object): + def __init__(self): + self.done = threading.Event() + + +class SslLayer(Layer): + def __init__(self, ctx, client_ssl, server_ssl): + super(SslLayer, self).__init__(ctx) + self._client_ssl = client_ssl + self._server_ssl = server_ssl + self._connected = False + self._sni_from_handshake = None + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we resort to the following workaround when establishing SSL with the server: + 1. Try to establish SSL with the server without SNI. If this fails, we ignore it. + 2. Establish SSL with client. + - If there's a SNI callback, reconnect to the server with SNI. + - If not and the server connect failed, raise the original exception. + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + client_ssl_requires_server_cert = ( + self._client_ssl and self._server_ssl and not self.config.no_upstream_cert + ) + lazy_server_ssl = ( + self._server_ssl and not client_ssl_requires_server_cert + ) + + if client_ssl_requires_server_cert: + for m in self._establish_ssl_with_client_and_server(): + yield m + elif self.client_ssl: + self._establish_ssl_with_client() + + layer = TcpLayer(self) + for message in layer(): + if message != Connect or not self._connected: + yield message + if message == Connect: + if lazy_server_ssl: + self._establish_ssl_with_server() + if message == ChangeServer and message.depth == 1: + self.server_ssl = message.server_ssl + self._sni_from_server_change = message.sni + if message == Reconnect or message == ChangeServer: + if self.server_ssl: + self._establish_ssl_with_server() + + @property + def sni(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self._sni_from_handshake + + def _establish_ssl_with_client_and_server(self): + """ + This function deals with the problem that the server may require a SNI value from the client. + """ + + # First, try to connect to the server. + yield Connect() + self._connected = True + server_err = None + try: + self._establish_ssl_with_server() + except ProxyError2 as e: + server_err = e + + # The part below is a bit ugly as we cannot yield from the handle_sni callback. + # The workaround is to do that in a separate thread and yield from the main thread. + + # client_ssl_queue may contain the following elements + # - True, if ssl was successfully established + # - An Exception thrown by self._establish_ssl_with_client() + # - A threading.Event, which singnifies a request for a reconnect from the sni callback + self.__client_ssl_queue = Queue.Queue() + + def establish_client_ssl(): + try: + self._establish_ssl_with_client() + self.__client_ssl_queue.put(True) + except Exception as client_err: + self.__client_ssl_queue.put(client_err) + + threading.Thread(target=establish_client_ssl, name="ClientSSLThread").start() + e = self.__client_ssl_queue.get() + if isinstance(e, ReconnectRequest): + yield Reconnect() + self._establish_ssl_with_server() + e.done.set() + e = self.__client_ssl_queue.get() + if e is not True: + raise ProxyError2("Error when establish client SSL: " + repr(e), e) + self.__client_ssl_queue = None + + if server_err and not self._sni_from_handshake: + raise server_err + + def handle_sni(self, connection): + """ + This callback gets called during the SSL handshake with the client. + The client has just sent the Sever Name Indication (SNI). + """ + try: + sn = connection.get_servername() + if not sn: + return + sni = sn.decode("utf8").encode("idna") + + if sni != self.sni: + self._sni_from_handshake = sni + + # Perform reconnect + if self.server_ssl: + reconnect = ReconnectRequest() + self.__client_ssl_queue.put() + reconnect.done.wait() + + # Now, change client context to reflect changed certificate: + cert, key, chain_file = self.find_cert() + new_context = self.client_conn.create_ssl_context( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file + ) + connection.set_context(new_context) + # An unhandled exception in this method will core dump PyOpenSSL, so + # make dang sure it doesn't happen. + except: # pragma: no cover + self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") + + def _establish_ssl_with_client(self): + self.log("Establish SSL with client", "debug") + cert, key, chain_file = self.find_cert() + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + handle_sni=self.handle_sni, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file + ) + except tcp.NetLibError as e: + raise ProxyError2(repr(e), e) + + def _establish_ssl_with_server(self): + self.log("Establish SSL with server", "debug") + try: + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + ) + ssl_cert_err = self.server_conn.ssl_verification_error + if ssl_cert_err is not None: + self.log( + "SSL verification failed for upstream server at depth %s with error: %s" % + (ssl_cert_err['depth'], ssl_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except tcp.NetLibInvalidCertificateError as e: + ssl_cert_err = self.server_conn.ssl_verification_error + self.log( + "SSL verification failed for upstream server at depth %s with error: %s" % + (ssl_cert_err['depth'], ssl_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProxyError2(repr(e), e) + except Exception as e: + raise ProxyError2(repr(e), e) + + def find_cert(self): + host = self.server_conn.address.host + sans = [] + # Incorporate upstream certificate + if self.server_conn.ssl_established and (not self.config.no_upstream_cert): + upstream_cert = self.server_conn.cert + sans.extend(upstream_cert.altnames) + if upstream_cert.cn: + sans.append(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self._sni_from_handshake: + sans.append(self._sni_from_handshake) + if self._sni_from_server_change: + sans.append(self._sni_from_server_change) + + return self.config.certstore.get_cert(host, sans) diff --git a/libmproxy/proxy/message.py b/libmproxy/proxy/message.py index a667123c..7eb59344 100644 --- a/libmproxy/proxy/message.py +++ b/libmproxy/proxy/message.py @@ -6,11 +6,13 @@ This module contains all valid messages layers can send to the underlying layers class _Message(object): def __eq__(self, other): # Allow message == Connect checks. - # FIXME: make Connect == message work. if isinstance(self, other): return True return self is other + def __ne__(self, other): + return not self.__eq__(other) + class Connect(_Message): """ diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index 923f84ca..a9f31181 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -151,7 +151,7 @@ class Socks5ProxyMode(ProxyMode): return ssl, ssl, connect_request.addr.host, connect_request.addr.port except (socks.SocksError, tcp.NetLibError) as e: - raise ProxyError(502, "SOCKS5 mode failure: %s" % str(e)) + raise ProxyError(502, "SOCKS5 mode failure: %s" % repr(e)) class _ConstDestinationProxyMode(ProxyMode): -- cgit v1.2.3 From c1d016823c67fc834a2fdb6c77181d14b5fd8008 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 24 Jul 2015 18:29:13 +0200 Subject: move files around --- libmproxy/protocol2/__init__.py | 157 +++++++++++++++ libmproxy/protocol2/messages.py | 43 +++++ libmproxy/protocol2/rawtcp.py | 13 ++ libmproxy/protocol2/socks.py | 26 +++ libmproxy/protocol2/ssl.py | 228 ++++++++++++++++++++++ libmproxy/proxy/__init__.py | 2 +- libmproxy/proxy/layer.py | 412 ---------------------------------------- libmproxy/proxy/message.py | 42 ---- libmproxy/proxy/primitives.py | 6 + libmproxy/proxy/server.py | 14 +- 10 files changed, 481 insertions(+), 462 deletions(-) create mode 100644 libmproxy/protocol2/__init__.py create mode 100644 libmproxy/protocol2/messages.py create mode 100644 libmproxy/protocol2/rawtcp.py create mode 100644 libmproxy/protocol2/socks.py create mode 100644 libmproxy/protocol2/ssl.py delete mode 100644 libmproxy/proxy/layer.py delete mode 100644 libmproxy/proxy/message.py (limited to 'libmproxy') diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py new file mode 100644 index 00000000..9374a5bf --- /dev/null +++ b/libmproxy/protocol2/__init__.py @@ -0,0 +1,157 @@ +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible scenarios (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentModeLayer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + RegularModeLayer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentModeLayer + AutoLayer + SslLayer + AutoLayer + Http2Layer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can "yield" commands to lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" + +from __future__ import (absolute_import, print_function, division, unicode_literals) +from netlib import tcp +from ..proxy import ProxyError2, Log +from ..proxy.connection import ServerConnection +from .messages import * + + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def __getattr__(self, name): + """ + Accessing a nonexisting attribute does not throw an error but returns None instead. + """ + return None + + +class _LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + + def __init__(self): + if True: + return + self.config = None + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(_LayerCodeCompletion): + def __init__(self, ctx): + """ + Args: + ctx: The (read-only) higher layer. + """ + super(Layer, self).__init__() + self.ctx = ctx + + def __call__(self): + """ + Logic of the layer. + Raises: + ProxyError2 in case of protocol exceptions. + """ + raise NotImplementedError + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "%s:%s: %s" % + (self.client_conn.address.host, + self.client_conn.address.port, + msg)] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + +class ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + + def __init__(self): + self.server_address = None + self.server_conn = None + + def _handle_server_message(self, message): + if message == Reconnect: + self._disconnect() + self._connect() + return True + elif message == Connect: + self._connect() + return True + elif message == ChangeServer: + raise NotImplementedError + return False + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_address)]) + self.server_conn.finish() + self.server_conn.close() + # self.channel.tell("serverdisconnect", self) + self.server_conn = None + + def _connect(self): + self.log("serverconnect", "debug", [repr(self.server_address)]) + self.server_conn = ServerConnection(self.server_address) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) + + def _set_address(self, address): + a = tcp.Address.wrap(address) + self.log("Set new server address: " + repr(a), "debug") + self.server_address = address + + +from .socks import Socks5IncomingLayer +from .rawtcp import TcpLayer \ No newline at end of file diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py new file mode 100644 index 00000000..52bb5a44 --- /dev/null +++ b/libmproxy/protocol2/messages.py @@ -0,0 +1,43 @@ +""" +This module contains all valid messages layers can send to the underlying layers. +""" +from __future__ import (absolute_import, print_function, division, unicode_literals) + + +class _Message(object): + def __eq__(self, other): + # Allow message == Connect checks. + if isinstance(self, other): + return True + return self is other + + def __ne__(self, other): + return not self.__eq__(other) + + +class Connect(_Message): + """ + Connect to the server + """ + + +class Reconnect(_Message): + """ + Re-establish the server connection + """ + + +class ChangeServer(_Message): + """ + Change the upstream server. + """ + + def __init__(self, address, server_ssl, sni, depth=1): + self.address = address + self.server_ssl = server_ssl + self.sni = sni + + # upstream proxy scenario: you may want to change either the final target or the upstream proxy. + # We can express this neatly as the "nth-server-providing-layer" + # ServerConnection could get a `via` attribute. + self.depth = depth diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py new file mode 100644 index 00000000..db9a48fa --- /dev/null +++ b/libmproxy/protocol2/rawtcp.py @@ -0,0 +1,13 @@ +from . import Layer, Connect +from ..protocol.tcp import TCPHandler + + +class TcpLayer(Layer): + def __call__(self): + yield Connect() + tcp_handler = TCPHandler(self) + tcp_handler.handle_messages() + + def establish_server_connection(self): + pass + # FIXME: Remove method, currently just here to mock TCPHandler's call to it. diff --git a/libmproxy/protocol2/socks.py b/libmproxy/protocol2/socks.py new file mode 100644 index 00000000..90771015 --- /dev/null +++ b/libmproxy/protocol2/socks.py @@ -0,0 +1,26 @@ +from __future__ import (absolute_import, print_function, division, unicode_literals) + +from ..proxy import ProxyError, Socks5ProxyMode, ProxyError2 +from . import Layer, ServerConnectionMixin +from .rawtcp import TcpLayer +from .ssl import SslLayer + + +class Socks5IncomingLayer(Layer, ServerConnectionMixin): + def __call__(self): + try: + s5mode = Socks5ProxyMode(self.config.ssl_ports) + address = s5mode.get_upstream_server(self.client_conn)[2:] + except ProxyError as e: + # TODO: Unmonkeypatch + raise ProxyError2(str(e), e) + + self._set_address(address) + + if address[1] == 443: + layer = SslLayer(self, True, True) + else: + layer = TcpLayer(self) + for message in layer(): + if not self._handle_server_message(message): + yield message diff --git a/libmproxy/protocol2/ssl.py b/libmproxy/protocol2/ssl.py new file mode 100644 index 00000000..6b44bf42 --- /dev/null +++ b/libmproxy/protocol2/ssl.py @@ -0,0 +1,228 @@ +from __future__ import (absolute_import, print_function, division, unicode_literals) +import Queue +import threading +import traceback +from netlib import tcp + +from ..proxy import ProxyError2 +from . import Layer +from .messages import Connect, Reconnect, ChangeServer +from .rawtcp import TcpLayer + + +class ReconnectRequest(object): + def __init__(self): + self.done = threading.Event() + + +class SslLayer(Layer): + def __init__(self, ctx, client_ssl, server_ssl): + super(SslLayer, self).__init__(ctx) + self._client_ssl = client_ssl + self._server_ssl = server_ssl + self._connected = False + self._sni_from_handshake = None + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we resort to the following workaround when establishing SSL with the server: + 1. Try to establish SSL with the server without SNI. If this fails, we ignore it. + 2. Establish SSL with client. + - If there's a SNI callback, reconnect to the server with SNI. + - If not and the server connect failed, raise the original exception. + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + client_ssl_requires_server_cert = ( + self._client_ssl and self._server_ssl and not self.config.no_upstream_cert + ) + lazy_server_ssl = ( + self._server_ssl and not client_ssl_requires_server_cert + ) + + if client_ssl_requires_server_cert: + for m in self._establish_ssl_with_client_and_server(): + yield m + elif self.client_ssl: + self._establish_ssl_with_client() + + layer = TcpLayer(self) + for message in layer(): + if message != Connect or not self._connected: + yield message + if message == Connect: + if lazy_server_ssl: + self._establish_ssl_with_server() + if message == ChangeServer and message.depth == 1: + self.server_ssl = message.server_ssl + self._sni_from_server_change = message.sni + if message == Reconnect or message == ChangeServer: + if self.server_ssl: + self._establish_ssl_with_server() + + @property + def sni(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self._sni_from_handshake + + def _establish_ssl_with_client_and_server(self): + """ + This function deals with the problem that the server may require a SNI value from the client. + """ + + # First, try to connect to the server. + yield Connect() + self._connected = True + server_err = None + try: + self._establish_ssl_with_server() + except ProxyError2 as e: + server_err = e + + # The part below is a bit ugly as we cannot yield from the handle_sni callback. + # The workaround is to do that in a separate thread and yield from the main thread. + + # client_ssl_queue may contain the following elements + # - True, if ssl was successfully established + # - An Exception thrown by self._establish_ssl_with_client() + # - A threading.Event, which singnifies a request for a reconnect from the sni callback + self.__client_ssl_queue = Queue.Queue() + + def establish_client_ssl(): + try: + self._establish_ssl_with_client() + self.__client_ssl_queue.put(True) + except Exception as client_err: + self.__client_ssl_queue.put(client_err) + + threading.Thread(target=establish_client_ssl, name="ClientSSLThread").start() + e = self.__client_ssl_queue.get() + if isinstance(e, ReconnectRequest): + yield Reconnect() + self._establish_ssl_with_server() + e.done.set() + e = self.__client_ssl_queue.get() + if e is not True: + raise ProxyError2("Error when establish client SSL: " + repr(e), e) + self.__client_ssl_queue = None + + if server_err and not self._sni_from_handshake: + raise server_err + + def handle_sni(self, connection): + """ + This callback gets called during the SSL handshake with the client. + The client has just sent the Sever Name Indication (SNI). + """ + try: + sn = connection.get_servername() + if not sn: + return + sni = sn.decode("utf8").encode("idna") + + if sni != self.sni: + self._sni_from_handshake = sni + + # Perform reconnect + if self.server_ssl: + reconnect = ReconnectRequest() + self.__client_ssl_queue.put() + reconnect.done.wait() + + # Now, change client context to reflect changed certificate: + cert, key, chain_file = self.find_cert() + new_context = self.client_conn.create_ssl_context( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file + ) + connection.set_context(new_context) + # An unhandled exception in this method will core dump PyOpenSSL, so + # make dang sure it doesn't happen. + except: # pragma: no cover + self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") + + def _establish_ssl_with_client(self): + self.log("Establish SSL with client", "debug") + cert, key, chain_file = self.find_cert() + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + handle_sni=self.handle_sni, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file + ) + except tcp.NetLibError as e: + raise ProxyError2(repr(e), e) + + def _establish_ssl_with_server(self): + self.log("Establish SSL with server", "debug") + try: + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + ) + ssl_cert_err = self.server_conn.ssl_verification_error + if ssl_cert_err is not None: + self.log( + "SSL verification failed for upstream server at depth %s with error: %s" % + (ssl_cert_err['depth'], ssl_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except tcp.NetLibInvalidCertificateError as e: + ssl_cert_err = self.server_conn.ssl_verification_error + self.log( + "SSL verification failed for upstream server at depth %s with error: %s" % + (ssl_cert_err['depth'], ssl_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProxyError2(repr(e), e) + except Exception as e: + raise ProxyError2(repr(e), e) + + def find_cert(self): + host = self.server_conn.address.host + sans = [] + # Incorporate upstream certificate + if self.server_conn.ssl_established and (not self.config.no_upstream_cert): + upstream_cert = self.server_conn.cert + sans.extend(upstream_cert.altnames) + if upstream_cert.cn: + sans.append(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self._sni_from_handshake: + sans.append(self._sni_from_handshake) + if self._sni_from_server_change: + sans.append(self._sni_from_server_change) + + return self.config.certstore.get_cert(host, sans) diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index f33d323b..7d664707 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1,2 +1,2 @@ from .primitives import * -from .config import ProxyConfig, process_proxy_options +from .config import ProxyConfig, process_proxy_options \ No newline at end of file diff --git a/libmproxy/proxy/layer.py b/libmproxy/proxy/layer.py deleted file mode 100644 index e4878bdf..00000000 --- a/libmproxy/proxy/layer.py +++ /dev/null @@ -1,412 +0,0 @@ -from __future__ import (absolute_import, print_function, division, unicode_literals) -import Queue -import threading -import traceback -from libmproxy.protocol.tcp import TCPHandler -from libmproxy.proxy.connection import ServerConnection -from netlib import tcp -from .primitives import Socks5ProxyMode, ProxyError, Log -from .message import Connect, Reconnect, ChangeServer - -""" -mitmproxy protocol architecture - -In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. -For example, the following scenarios depict possible scenarios (lowest layer first): - -Transparent HTTP proxy, no SSL: - TransparentModeLayer - HttpLayer - -Regular proxy, CONNECT request with WebSockets over SSL: - RegularModeLayer - HttpLayer - SslLayer - WebsocketLayer (or TcpLayer) - -Automated protocol detection by peeking into the buffer: - TransparentModeLayer - AutoLayer - SslLayer - AutoLayer - Http2Layer - -Communication between layers is done as follows: - - lower layers provide context information to higher layers - - higher layers can "yield" commands to lower layers, - which are propagated until they reach a suitable layer. - -Further goals: - - Connections should always be peekable to make automatic protocol detection work. - - Upstream connections should be established as late as possible; - inline scripts shall have a chance to handle everything locally. -""" - - -class ProxyError2(Exception): - def __init__(self, message, cause=None): - super(ProxyError2, self).__init__(message) - self.cause = cause - - -class RootContext(object): - """ - The outmost context provided to the root layer. - As a consequence, every layer has .client_conn, .channel and .config. - """ - - def __init__(self, client_conn, config, channel): - self.client_conn = client_conn # Client Connection - self.channel = channel # provides .ask() method to communicate with FlowMaster - self.config = config # Proxy Configuration - - def __getattr__(self, name): - """ - Accessing a nonexisting attribute does not throw an error but returns None instead. - """ - return None - - -class _LayerCodeCompletion(object): - """ - Dummy class that provides type hinting in PyCharm, which simplifies development a lot. - """ - - def __init__(self): - if True: - return - self.config = None - """@type: libmproxy.proxy.config.ProxyConfig""" - self.client_conn = None - """@type: libmproxy.proxy.connection.ClientConnection""" - self.channel = None - """@type: libmproxy.controller.Channel""" - - -class Layer(_LayerCodeCompletion): - def __init__(self, ctx): - """ - Args: - ctx: The (read-only) higher layer. - """ - super(Layer, self).__init__() - self.ctx = ctx - - def __call__(self): - """ - Logic of the layer. - Raises: - ProxyError2 in case of protocol exceptions. - """ - raise NotImplementedError - - def __getattr__(self, name): - """ - Attributes not present on the current layer may exist on a higher layer. - """ - return getattr(self.ctx, name) - - def log(self, msg, level, subs=()): - full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - -class _ServerConnectionMixin(object): - """ - Mixin that provides a layer with the capabilities to manage a server connection. - """ - - def __init__(self): - self.server_address = None - self.server_conn = None - - def _handle_server_message(self, message): - if message == Reconnect: - self._disconnect() - self._connect() - return True - elif message == Connect: - self._connect() - return True - elif message == ChangeServer: - raise NotImplementedError - return False - - def _disconnect(self): - """ - Deletes (and closes) an existing server connection. - """ - self.log("serverdisconnect", "debug", [repr(self.server_address)]) - self.server_conn.finish() - self.server_conn.close() - # self.channel.tell("serverdisconnect", self) - self.server_conn = None - - def _connect(self): - self.log("serverconnect", "debug", [repr(self.server_address)]) - self.server_conn = ServerConnection(self.server_address) - try: - self.server_conn.connect() - except tcp.NetLibError as e: - raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) - - def _set_address(self, address): - a = tcp.Address.wrap(address) - self.log("Set new server address: " + repr(a), "debug") - self.server_address = address - - -class Socks5IncomingLayer(Layer, _ServerConnectionMixin): - def __call__(self): - try: - s5mode = Socks5ProxyMode(self.config.ssl_ports) - address = s5mode.get_upstream_server(self.client_conn)[2:] - except ProxyError as e: - # TODO: Unmonkeypatch - raise ProxyError2(str(e), e) - - self._set_address(address) - - if address[1] == 443: - layer = SslLayer(self, True, True) - else: - layer = TcpLayer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message - - -class TcpLayer(Layer): - def __call__(self): - yield Connect() - tcp_handler = TCPHandler(self) - tcp_handler.handle_messages() - - def establish_server_connection(self): - pass - # FIXME: Remove method, currently just here to mock TCPHandler's call to it. - - -class ReconnectRequest(object): - def __init__(self): - self.done = threading.Event() - - -class SslLayer(Layer): - def __init__(self, ctx, client_ssl, server_ssl): - super(SslLayer, self).__init__(ctx) - self._client_ssl = client_ssl - self._server_ssl = server_ssl - self._connected = False - self._sni_from_handshake = None - self._sni_from_server_change = None - - def __call__(self): - """ - The strategy for establishing SSL is as follows: - First, we determine whether we need the server cert to establish ssl with the client. - If so, we first connect to the server and then to the client. - If not, we only connect to the client and do the server_ssl lazily on a Connect message. - - An additional complexity is that establish ssl with the server may require a SNI value from the client. - In an ideal world, we'd do the following: - 1. Start the SSL handshake with the client - 2. Check if the client sends a SNI. - 3. Pause the client handshake, establish SSL with the server. - 4. Finish the client handshake with the certificate from the server. - There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( - Thus, we resort to the following workaround when establishing SSL with the server: - 1. Try to establish SSL with the server without SNI. If this fails, we ignore it. - 2. Establish SSL with client. - - If there's a SNI callback, reconnect to the server with SNI. - - If not and the server connect failed, raise the original exception. - Further notes: - - OpenSSL 1.0.2 introduces a callback that would help here: - https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html - - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 - """ - client_ssl_requires_server_cert = ( - self._client_ssl and self._server_ssl and not self.config.no_upstream_cert - ) - lazy_server_ssl = ( - self._server_ssl and not client_ssl_requires_server_cert - ) - - if client_ssl_requires_server_cert: - for m in self._establish_ssl_with_client_and_server(): - yield m - elif self.client_ssl: - self._establish_ssl_with_client() - - layer = TcpLayer(self) - for message in layer(): - if message != Connect or not self._connected: - yield message - if message == Connect: - if lazy_server_ssl: - self._establish_ssl_with_server() - if message == ChangeServer and message.depth == 1: - self.server_ssl = message.server_ssl - self._sni_from_server_change = message.sni - if message == Reconnect or message == ChangeServer: - if self.server_ssl: - self._establish_ssl_with_server() - - @property - def sni(self): - if self._sni_from_server_change is False: - return None - else: - return self._sni_from_server_change or self._sni_from_handshake - - def _establish_ssl_with_client_and_server(self): - """ - This function deals with the problem that the server may require a SNI value from the client. - """ - - # First, try to connect to the server. - yield Connect() - self._connected = True - server_err = None - try: - self._establish_ssl_with_server() - except ProxyError2 as e: - server_err = e - - # The part below is a bit ugly as we cannot yield from the handle_sni callback. - # The workaround is to do that in a separate thread and yield from the main thread. - - # client_ssl_queue may contain the following elements - # - True, if ssl was successfully established - # - An Exception thrown by self._establish_ssl_with_client() - # - A threading.Event, which singnifies a request for a reconnect from the sni callback - self.__client_ssl_queue = Queue.Queue() - - def establish_client_ssl(): - try: - self._establish_ssl_with_client() - self.__client_ssl_queue.put(True) - except Exception as client_err: - self.__client_ssl_queue.put(client_err) - - threading.Thread(target=establish_client_ssl, name="ClientSSLThread").start() - e = self.__client_ssl_queue.get() - if isinstance(e, ReconnectRequest): - yield Reconnect() - self._establish_ssl_with_server() - e.done.set() - e = self.__client_ssl_queue.get() - if e is not True: - raise ProxyError2("Error when establish client SSL: " + repr(e), e) - self.__client_ssl_queue = None - - if server_err and not self._sni_from_handshake: - raise server_err - - def handle_sni(self, connection): - """ - This callback gets called during the SSL handshake with the client. - The client has just sent the Sever Name Indication (SNI). - """ - try: - sn = connection.get_servername() - if not sn: - return - sni = sn.decode("utf8").encode("idna") - - if sni != self.sni: - self._sni_from_handshake = sni - - # Perform reconnect - if self.server_ssl: - reconnect = ReconnectRequest() - self.__client_ssl_queue.put() - reconnect.done.wait() - - # Now, change client context to reflect changed certificate: - cert, key, chain_file = self.find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - connection.set_context(new_context) - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except: # pragma: no cover - self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") - - def _establish_ssl_with_client(self): - self.log("Establish SSL with client", "debug") - cert, key, chain_file = self.find_cert() - try: - self.client_conn.convert_to_ssl( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - handle_sni=self.handle_sni, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - except tcp.NetLibError as e: - raise ProxyError2(repr(e), e) - - def _establish_ssl_with_server(self): - self.log("Establish SSL with server", "debug") - try: - self.server_conn.establish_ssl( - self.config.clientcerts, - self.sni, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify_options=self.config.openssl_verification_mode_server, - ca_path=self.config.openssl_trusted_cadir_server, - ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, - ) - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") - except tcp.NetLibInvalidCertificateError as e: - ssl_cert_err = self.server_conn.ssl_verification_error - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") - raise ProxyError2(repr(e), e) - except Exception as e: - raise ProxyError2(repr(e), e) - - def find_cert(self): - host = self.server_conn.address.host - sans = [] - # Incorporate upstream certificate - if self.server_conn.ssl_established and (not self.config.no_upstream_cert): - upstream_cert = self.server_conn.cert - sans.extend(upstream_cert.altnames) - if upstream_cert.cn: - sans.append(host) - host = upstream_cert.cn.decode("utf8").encode("idna") - # Also add SNI values. - if self._sni_from_handshake: - sans.append(self._sni_from_handshake) - if self._sni_from_server_change: - sans.append(self._sni_from_server_change) - - return self.config.certstore.get_cert(host, sans) diff --git a/libmproxy/proxy/message.py b/libmproxy/proxy/message.py deleted file mode 100644 index 7eb59344..00000000 --- a/libmproxy/proxy/message.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -This module contains all valid messages layers can send to the underlying layers. -""" - - -class _Message(object): - def __eq__(self, other): - # Allow message == Connect checks. - if isinstance(self, other): - return True - return self is other - - def __ne__(self, other): - return not self.__eq__(other) - - -class Connect(_Message): - """ - Connect to the server - """ - - -class Reconnect(_Message): - """ - Re-establish the server connection - """ - - -class ChangeServer(_Message): - """ - Change the upstream server. - """ - - def __init__(self, address, server_ssl, sni, depth=1): - self.address = address - self.server_ssl = server_ssl - self.sni = sni - - # upstream proxy scenario: you may want to change either the final target or the upstream proxy. - # We can express this neatly as the "nth-server-providing-layer" - # ServerConnection could get a `via` attribute. - self.depth = depth diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index a9f31181..fd4eb882 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -2,6 +2,12 @@ from __future__ import absolute_import from netlib import socks, tcp +class ProxyError2(Exception): + def __init__(self, message, cause=None): + super(ProxyError2, self).__init__(message) + self.cause = cause + + class ProxyError(Exception): def __init__(self, code, message, headers=None): super(ProxyError, self).__init__(message) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index a45276d4..c8990a9a 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -3,12 +3,12 @@ from __future__ import absolute_import, print_function import traceback import sys import socket - from netlib import tcp -from . import layer -from .primitives import ProxyServerError, Log, ProxyError -from .connection import ClientConnection, ServerConnection + from ..protocol.handle import protocol_handler +from .. import protocol2 +from .primitives import ProxyServerError, Log, ProxyError, ProxyError2 +from .connection import ClientConnection, ServerConnection class DummyServer: @@ -74,17 +74,17 @@ class ConnectionHandler2: def handle(self): self.log("clientconnect", "info") - root_context = layer.RootContext( + root_context = protocol2.RootContext( self.client_conn, self.config, self.channel ) - root_layer = layer.Socks5IncomingLayer(root_context) + root_layer = protocol2.Socks5IncomingLayer(root_context) try: for message in root_layer(): print("Root layer receveived: %s" % message) - except layer.ProxyError2 as e: + except ProxyError2 as e: self.log(e, "info") except Exception: self.log(traceback.format_exc(), "error") -- cgit v1.2.3 From e815915b22ef266ac4122027a10c59d9e036d0b4 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 25 Jul 2015 13:31:55 +0200 Subject: add auto layer, multiple other fixes --- libmproxy/protocol2/__init__.py | 159 +--------------------------------------- libmproxy/protocol2/auto.py | 16 ++++ libmproxy/protocol2/layer.py | 151 ++++++++++++++++++++++++++++++++++++++ libmproxy/protocol2/rawtcp.py | 4 +- libmproxy/protocol2/socks.py | 10 +-- libmproxy/protocol2/ssl.py | 8 +- 6 files changed, 183 insertions(+), 165 deletions(-) create mode 100644 libmproxy/protocol2/auto.py create mode 100644 libmproxy/protocol2/layer.py (limited to 'libmproxy') diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index 9374a5bf..6f4bfe44 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -1,157 +1,6 @@ -""" -mitmproxy protocol architecture - -In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. -For example, the following scenarios depict possible scenarios (lowest layer first): - -Transparent HTTP proxy, no SSL: - TransparentModeLayer - HttpLayer - -Regular proxy, CONNECT request with WebSockets over SSL: - RegularModeLayer - HttpLayer - SslLayer - WebsocketLayer (or TcpLayer) - -Automated protocol detection by peeking into the buffer: - TransparentModeLayer - AutoLayer - SslLayer - AutoLayer - Http2Layer - -Communication between layers is done as follows: - - lower layers provide context information to higher layers - - higher layers can "yield" commands to lower layers, - which are propagated until they reach a suitable layer. - -Further goals: - - Connections should always be peekable to make automatic protocol detection work. - - Upstream connections should be established as late as possible; - inline scripts shall have a chance to handle everything locally. -""" - from __future__ import (absolute_import, print_function, division, unicode_literals) -from netlib import tcp -from ..proxy import ProxyError2, Log -from ..proxy.connection import ServerConnection -from .messages import * - - -class RootContext(object): - """ - The outmost context provided to the root layer. - As a consequence, every layer has .client_conn, .channel and .config. - """ - - def __init__(self, client_conn, config, channel): - self.client_conn = client_conn # Client Connection - self.channel = channel # provides .ask() method to communicate with FlowMaster - self.config = config # Proxy Configuration - - def __getattr__(self, name): - """ - Accessing a nonexisting attribute does not throw an error but returns None instead. - """ - return None - - -class _LayerCodeCompletion(object): - """ - Dummy class that provides type hinting in PyCharm, which simplifies development a lot. - """ - - def __init__(self): - if True: - return - self.config = None - """@type: libmproxy.proxy.config.ProxyConfig""" - self.client_conn = None - """@type: libmproxy.proxy.connection.ClientConnection""" - self.channel = None - """@type: libmproxy.controller.Channel""" - - -class Layer(_LayerCodeCompletion): - def __init__(self, ctx): - """ - Args: - ctx: The (read-only) higher layer. - """ - super(Layer, self).__init__() - self.ctx = ctx - - def __call__(self): - """ - Logic of the layer. - Raises: - ProxyError2 in case of protocol exceptions. - """ - raise NotImplementedError - - def __getattr__(self, name): - """ - Attributes not present on the current layer may exist on a higher layer. - """ - return getattr(self.ctx, name) - - def log(self, msg, level, subs=()): - full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - -class ServerConnectionMixin(object): - """ - Mixin that provides a layer with the capabilities to manage a server connection. - """ - - def __init__(self): - self.server_address = None - self.server_conn = None - - def _handle_server_message(self, message): - if message == Reconnect: - self._disconnect() - self._connect() - return True - elif message == Connect: - self._connect() - return True - elif message == ChangeServer: - raise NotImplementedError - return False - - def _disconnect(self): - """ - Deletes (and closes) an existing server connection. - """ - self.log("serverdisconnect", "debug", [repr(self.server_address)]) - self.server_conn.finish() - self.server_conn.close() - # self.channel.tell("serverdisconnect", self) - self.server_conn = None - - def _connect(self): - self.log("serverconnect", "debug", [repr(self.server_address)]) - self.server_conn = ServerConnection(self.server_address) - try: - self.server_conn.connect() - except tcp.NetLibError as e: - raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) - - def _set_address(self, address): - a = tcp.Address.wrap(address) - self.log("Set new server address: " + repr(a), "debug") - self.server_address = address - - +from .layer import RootContext from .socks import Socks5IncomingLayer -from .rawtcp import TcpLayer \ No newline at end of file +from .rawtcp import TcpLayer +from .auto import AutoLayer +__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext"] \ No newline at end of file diff --git a/libmproxy/protocol2/auto.py b/libmproxy/protocol2/auto.py new file mode 100644 index 00000000..1c4293ac --- /dev/null +++ b/libmproxy/protocol2/auto.py @@ -0,0 +1,16 @@ +from __future__ import (absolute_import, print_function, division, unicode_literals) +from .layer import Layer + + +class AutoLayer(Layer): + def __call__(self): + d = self.client_conn.rfile.peek(1) + if d[0] == "\x16": + layer = SslLayer(self, True, True) + else: + layer = TcpLayer(self) + for m in layer(): + yield m + +from .rawtcp import TcpLayer +from .ssl import SslLayer diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py new file mode 100644 index 00000000..14263f64 --- /dev/null +++ b/libmproxy/protocol2/layer.py @@ -0,0 +1,151 @@ +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible scenarios (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentModeLayer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + RegularModeLayer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentModeLayer + AutoLayer + SslLayer + AutoLayer + Http2Layer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can "yield" commands to lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" +from __future__ import (absolute_import, print_function, division, unicode_literals) +from netlib import tcp +from ..proxy import ProxyError2, Log +from ..proxy.connection import ServerConnection +from .messages import Connect, Reconnect, ChangeServer + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def __getattr__(self, name): + """ + Accessing a nonexisting attribute does not throw an error but returns None instead. + """ + return None + + +class _LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + + def __init__(self): + if True: + return + self.config = None + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(_LayerCodeCompletion): + def __init__(self, ctx): + """ + Args: + ctx: The (read-only) higher layer. + """ + super(Layer, self).__init__() + self.ctx = ctx + + def __call__(self): + """ + Logic of the layer. + Raises: + ProxyError2 in case of protocol exceptions. + """ + raise NotImplementedError + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "%s:%s: %s" % + (self.client_conn.address.host, + self.client_conn.address.port, + msg)] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + +class ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + + def __init__(self): + self.server_address = None + self.server_conn = None + + def _handle_server_message(self, message): + if message == Reconnect: + self._disconnect() + self._connect() + return True + elif message == Connect: + self._connect() + return True + elif message == ChangeServer: + raise NotImplementedError + return False + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_address)]) + self.server_conn.finish() + self.server_conn.close() + # self.channel.tell("serverdisconnect", self) + self.server_conn = None + + def _connect(self): + self.log("serverconnect", "debug", [repr(self.server_address)]) + self.server_conn = ServerConnection(self.server_address) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) + + def _set_address(self, address): + a = tcp.Address.wrap(address) + self.log("Set new server address: " + repr(a), "debug") + self.server_address = address \ No newline at end of file diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index db9a48fa..b40c569f 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -1,5 +1,7 @@ -from . import Layer, Connect +from __future__ import (absolute_import, print_function, division, unicode_literals) from ..protocol.tcp import TCPHandler +from .layer import Layer +from .messages import Connect class TcpLayer(Layer): diff --git a/libmproxy/protocol2/socks.py b/libmproxy/protocol2/socks.py index 90771015..9ca30bb4 100644 --- a/libmproxy/protocol2/socks.py +++ b/libmproxy/protocol2/socks.py @@ -1,9 +1,7 @@ from __future__ import (absolute_import, print_function, division, unicode_literals) from ..proxy import ProxyError, Socks5ProxyMode, ProxyError2 -from . import Layer, ServerConnectionMixin -from .rawtcp import TcpLayer -from .ssl import SslLayer +from .layer import Layer, ServerConnectionMixin class Socks5IncomingLayer(Layer, ServerConnectionMixin): @@ -18,9 +16,11 @@ class Socks5IncomingLayer(Layer, ServerConnectionMixin): self._set_address(address) if address[1] == 443: - layer = SslLayer(self, True, True) + layer = AutoLayer(self) else: - layer = TcpLayer(self) + layer = AutoLayer(self) for message in layer(): if not self._handle_server_message(message): yield message + +from .auto import AutoLayer diff --git a/libmproxy/protocol2/ssl.py b/libmproxy/protocol2/ssl.py index 6b44bf42..e8ff16cf 100644 --- a/libmproxy/protocol2/ssl.py +++ b/libmproxy/protocol2/ssl.py @@ -5,9 +5,9 @@ import traceback from netlib import tcp from ..proxy import ProxyError2 -from . import Layer +from .layer import Layer from .messages import Connect, Reconnect, ChangeServer -from .rawtcp import TcpLayer +from .auto import AutoLayer class ReconnectRequest(object): @@ -61,7 +61,7 @@ class SslLayer(Layer): elif self.client_ssl: self._establish_ssl_with_client() - layer = TcpLayer(self) + layer = AutoLayer(self) for message in layer(): if message != Connect or not self._connected: yield message @@ -225,4 +225,4 @@ class SslLayer(Layer): if self._sni_from_server_change: sans.append(self._sni_from_server_change) - return self.config.certstore.get_cert(host, sans) + return self.config.certstore.get_cert(host, sans) \ No newline at end of file -- cgit v1.2.3 From 531ca4a35684a83e57d4655922e9817814de41f6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 25 Jul 2015 14:48:50 +0200 Subject: minor fixes --- libmproxy/protocol2/__init__.py | 2 +- libmproxy/protocol2/auto.py | 2 ++ libmproxy/protocol2/layer.py | 3 ++- libmproxy/protocol2/socks.py | 8 ++------ 4 files changed, 7 insertions(+), 8 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index 6f4bfe44..20e5a888 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -3,4 +3,4 @@ from .layer import RootContext from .socks import Socks5IncomingLayer from .rawtcp import TcpLayer from .auto import AutoLayer -__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext"] \ No newline at end of file +__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext"] diff --git a/libmproxy/protocol2/auto.py b/libmproxy/protocol2/auto.py index 1c4293ac..a00f1f52 100644 --- a/libmproxy/protocol2/auto.py +++ b/libmproxy/protocol2/auto.py @@ -5,6 +5,8 @@ from .layer import Layer class AutoLayer(Layer): def __call__(self): d = self.client_conn.rfile.peek(1) + + # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello if d[0] == "\x16": layer = SslLayer(self, True, True) else: diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 14263f64..1cc8df70 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -37,6 +37,7 @@ from ..proxy import ProxyError2, Log from ..proxy.connection import ServerConnection from .messages import Connect, Reconnect, ChangeServer + class RootContext(object): """ The outmost context provided to the root layer. @@ -148,4 +149,4 @@ class ServerConnectionMixin(object): def _set_address(self, address): a = tcp.Address.wrap(address) self.log("Set new server address: " + repr(a), "debug") - self.server_address = address \ No newline at end of file + self.server_address = address diff --git a/libmproxy/protocol2/socks.py b/libmproxy/protocol2/socks.py index 9ca30bb4..7835b1a4 100644 --- a/libmproxy/protocol2/socks.py +++ b/libmproxy/protocol2/socks.py @@ -2,6 +2,7 @@ from __future__ import (absolute_import, print_function, division, unicode_liter from ..proxy import ProxyError, Socks5ProxyMode, ProxyError2 from .layer import Layer, ServerConnectionMixin +from .auto import AutoLayer class Socks5IncomingLayer(Layer, ServerConnectionMixin): @@ -15,12 +16,7 @@ class Socks5IncomingLayer(Layer, ServerConnectionMixin): self._set_address(address) - if address[1] == 443: - layer = AutoLayer(self) - else: - layer = AutoLayer(self) + layer = AutoLayer(self) for message in layer(): if not self._handle_server_message(message): yield message - -from .auto import AutoLayer -- cgit v1.2.3 From c46e3f90bbc38080a41a278340aaad27d8881fd9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 6 Aug 2015 11:09:01 +0200 Subject: apply fixes from proxy-refactor-cb branch --- libmproxy/protocol2/__init__.py | 2 +- libmproxy/protocol2/auto.py | 4 +++- libmproxy/protocol2/layer.py | 25 ++++++++++++------------- libmproxy/protocol2/messages.py | 2 +- libmproxy/protocol2/rawtcp.py | 2 +- libmproxy/protocol2/socks.py | 4 ++-- libmproxy/protocol2/ssl.py | 34 ++++++++++++++++++---------------- 7 files changed, 38 insertions(+), 35 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index 20e5a888..95f67c6c 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -1,4 +1,4 @@ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) from .layer import RootContext from .socks import Socks5IncomingLayer from .rawtcp import TcpLayer diff --git a/libmproxy/protocol2/auto.py b/libmproxy/protocol2/auto.py index a00f1f52..fc111758 100644 --- a/libmproxy/protocol2/auto.py +++ b/libmproxy/protocol2/auto.py @@ -1,4 +1,4 @@ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) from .layer import Layer @@ -6,6 +6,8 @@ class AutoLayer(Layer): def __call__(self): d = self.client_conn.rfile.peek(1) + if not d: + return # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello if d[0] == "\x16": layer = SslLayer(self, True, True) diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 1cc8df70..30aed350 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -31,7 +31,7 @@ Further goals: - Upstream connections should be established as late as possible; inline scripts shall have a chance to handle everything locally. """ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) from netlib import tcp from ..proxy import ProxyError2, Log from ..proxy.connection import ServerConnection @@ -49,12 +49,6 @@ class RootContext(object): self.channel = channel # provides .ask() method to communicate with FlowMaster self.config = config # Proxy Configuration - def __getattr__(self, name): - """ - Accessing a nonexisting attribute does not throw an error but returns None instead. - """ - return None - class _LayerCodeCompletion(object): """ @@ -113,7 +107,7 @@ class ServerConnectionMixin(object): """ def __init__(self): - self.server_address = None + self._server_address = None self.server_conn = None def _handle_server_message(self, message): @@ -128,6 +122,16 @@ class ServerConnectionMixin(object): raise NotImplementedError return False + @property + def server_address(self): + return self._server_address + + @server_address.setter + def server_address(self, address): + self._server_address = tcp.Address.wrap(address) + self.log("Set new server address: " + repr(self.server_address), "debug") + + def _disconnect(self): """ Deletes (and closes) an existing server connection. @@ -145,8 +149,3 @@ class ServerConnectionMixin(object): self.server_conn.connect() except tcp.NetLibError as e: raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) - - def _set_address(self, address): - a = tcp.Address.wrap(address) - self.log("Set new server address: " + repr(a), "debug") - self.server_address = address diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index 52bb5a44..baf4312d 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -1,7 +1,7 @@ """ This module contains all valid messages layers can send to the underlying layers. """ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) class _Message(object): diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index b40c569f..39e48e24 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -1,4 +1,4 @@ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) from ..protocol.tcp import TCPHandler from .layer import Layer from .messages import Connect diff --git a/libmproxy/protocol2/socks.py b/libmproxy/protocol2/socks.py index 7835b1a4..14564521 100644 --- a/libmproxy/protocol2/socks.py +++ b/libmproxy/protocol2/socks.py @@ -1,4 +1,4 @@ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) from ..proxy import ProxyError, Socks5ProxyMode, ProxyError2 from .layer import Layer, ServerConnectionMixin @@ -14,7 +14,7 @@ class Socks5IncomingLayer(Layer, ServerConnectionMixin): # TODO: Unmonkeypatch raise ProxyError2(str(e), e) - self._set_address(address) + self.server_address = address layer = AutoLayer(self) for message in layer(): diff --git a/libmproxy/protocol2/ssl.py b/libmproxy/protocol2/ssl.py index e8ff16cf..c21956b7 100644 --- a/libmproxy/protocol2/ssl.py +++ b/libmproxy/protocol2/ssl.py @@ -1,4 +1,4 @@ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) import Queue import threading import traceback @@ -76,7 +76,7 @@ class SslLayer(Layer): self._establish_ssl_with_server() @property - def sni(self): + def sni_for_upstream_connection(self): if self._sni_from_server_change is False: return None else: @@ -132,21 +132,22 @@ class SslLayer(Layer): The client has just sent the Sever Name Indication (SNI). """ try: + old_upstream_sni = self.sni_for_upstream_connection + sn = connection.get_servername() if not sn: return - sni = sn.decode("utf8").encode("idna") - - if sni != self.sni: - self._sni_from_handshake = sni + self._sni_from_handshake = sn.decode("utf8").encode("idna") + if old_upstream_sni != self.sni_for_upstream_connection: # Perform reconnect if self.server_ssl: reconnect = ReconnectRequest() - self.__client_ssl_queue.put() + self.__client_ssl_queue.put(reconnect) reconnect.done.wait() - # Now, change client context to reflect changed certificate: + if self._sni_from_handshake: + # Now, change client context to reflect possibly changed certificate: cert, key, chain_file = self.find_cert() new_context = self.client_conn.create_ssl_context( cert, key, @@ -183,7 +184,7 @@ class SslLayer(Layer): try: self.server_conn.establish_ssl( self.config.clientcerts, - self.sni, + self.sni_for_upstream_connection, method=self.config.openssl_method_server, options=self.config.openssl_options_server, verify_options=self.config.openssl_verification_mode_server, @@ -206,23 +207,24 @@ class SslLayer(Layer): "error") self.log("Aborting connection attempt", "error") raise ProxyError2(repr(e), e) - except Exception as e: + except tcp.NetLibError as e: raise ProxyError2(repr(e), e) def find_cert(self): host = self.server_conn.address.host - sans = [] + # TODO: Better use an OrderedSet here + sans = set() # Incorporate upstream certificate if self.server_conn.ssl_established and (not self.config.no_upstream_cert): upstream_cert = self.server_conn.cert - sans.extend(upstream_cert.altnames) + sans.update(upstream_cert.altnames) if upstream_cert.cn: - sans.append(host) + sans.add(host) host = upstream_cert.cn.decode("utf8").encode("idna") # Also add SNI values. if self._sni_from_handshake: - sans.append(self._sni_from_handshake) + sans.add(self._sni_from_handshake) if self._sni_from_server_change: - sans.append(self._sni_from_server_change) + sans.add(self._sni_from_server_change) - return self.config.certstore.get_cert(host, sans) \ No newline at end of file + return self.config.certstore.get_cert(host, list(sans)) \ No newline at end of file -- cgit v1.2.3 From aac0ab23ebb0e4d88306b12efee1dd31338f7664 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 6 Aug 2015 12:13:23 +0200 Subject: simplify layer code, add yield_from_callback decorator --- libmproxy/protocol2/layer.py | 58 +++++++++++++++++++++++++++++++++++++++++++- libmproxy/protocol2/ssl.py | 51 +++++++++----------------------------- 2 files changed, 69 insertions(+), 40 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 30aed350..aaa51baf 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -32,6 +32,8 @@ Further goals: inline scripts shall have a chance to handle everything locally. """ from __future__ import (absolute_import, print_function, division) +import Queue +import threading from netlib import tcp from ..proxy import ProxyError2, Log from ..proxy.connection import ServerConnection @@ -131,7 +133,6 @@ class ServerConnectionMixin(object): self._server_address = tcp.Address.wrap(address) self.log("Set new server address: " + repr(self.server_address), "debug") - def _disconnect(self): """ Deletes (and closes) an existing server connection. @@ -149,3 +150,58 @@ class ServerConnectionMixin(object): self.server_conn.connect() except tcp.NetLibError as e: raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) + + +def yield_from_callback(fun): + """ + Decorator which makes it possible to yield from callbacks in the original thread. + As a use case, take the pyOpenSSL handle_sni callback: If we receive a new SNI from the client, + we need to reconnect to the server with the new SNI. Reconnecting would normally be done using "yield Reconnect()", + but we're in a pyOpenSSL callback here, outside of the main program flow. With this decorator, it looks as follows: + + def handle_sni(self): + # ... + self.yield_from_callback(Reconnect()) + + @yield_from_callback + def establish_ssl_with_client(): + self.client_conn.convert_to_ssl(...) + + for message in self.establish_ssl_with_client(): # will yield Reconnect at some point + yield message + + + Limitations: + - You cannot yield True. + """ + yield_queue = Queue.Queue() + + def do_yield(self, msg): + yield_queue.put(msg) + yield_queue.get() + + def wrapper(self, *args, **kwargs): + self.yield_from_callback = do_yield + + def run(): + try: + fun(self, *args, **kwargs) + yield_queue.put(True) + except Exception as e: + yield_queue.put(e) + + threading.Thread(target=run, name="YieldFromCallbackThread").start() + while True: + e = yield_queue.get() + if e is True: + break + elif isinstance(e, Exception): + # TODO: Include func name? + raise ProxyError2("Error from callback: " + repr(e), e) + else: + yield e + yield_queue.put(None) + + self.yield_from_callback = None + + return wrapper diff --git a/libmproxy/protocol2/ssl.py b/libmproxy/protocol2/ssl.py index c21956b7..32798e72 100644 --- a/libmproxy/protocol2/ssl.py +++ b/libmproxy/protocol2/ssl.py @@ -1,20 +1,13 @@ from __future__ import (absolute_import, print_function, division) -import Queue -import threading import traceback from netlib import tcp from ..proxy import ProxyError2 -from .layer import Layer +from .layer import Layer, yield_from_callback from .messages import Connect, Reconnect, ChangeServer from .auto import AutoLayer -class ReconnectRequest(object): - def __init__(self): - self.done = threading.Event() - - class SslLayer(Layer): def __init__(self, ctx, client_ssl, server_ssl): super(SslLayer, self).__init__(ctx) @@ -59,7 +52,8 @@ class SslLayer(Layer): for m in self._establish_ssl_with_client_and_server(): yield m elif self.client_ssl: - self._establish_ssl_with_client() + for m in self._establish_ssl_with_client(): + yield m layer = AutoLayer(self) for message in layer(): @@ -96,32 +90,12 @@ class SslLayer(Layer): except ProxyError2 as e: server_err = e - # The part below is a bit ugly as we cannot yield from the handle_sni callback. - # The workaround is to do that in a separate thread and yield from the main thread. - - # client_ssl_queue may contain the following elements - # - True, if ssl was successfully established - # - An Exception thrown by self._establish_ssl_with_client() - # - A threading.Event, which singnifies a request for a reconnect from the sni callback - self.__client_ssl_queue = Queue.Queue() - - def establish_client_ssl(): - try: - self._establish_ssl_with_client() - self.__client_ssl_queue.put(True) - except Exception as client_err: - self.__client_ssl_queue.put(client_err) - - threading.Thread(target=establish_client_ssl, name="ClientSSLThread").start() - e = self.__client_ssl_queue.get() - if isinstance(e, ReconnectRequest): - yield Reconnect() - self._establish_ssl_with_server() - e.done.set() - e = self.__client_ssl_queue.get() - if e is not True: - raise ProxyError2("Error when establish client SSL: " + repr(e), e) - self.__client_ssl_queue = None + for message in self._establish_ssl_with_client(): + if message == Reconnect: + yield message + self._establish_ssl_with_server() + else: + raise RuntimeError("Unexpected Message: %s" % message) if server_err and not self._sni_from_handshake: raise server_err @@ -142,9 +116,7 @@ class SslLayer(Layer): if old_upstream_sni != self.sni_for_upstream_connection: # Perform reconnect if self.server_ssl: - reconnect = ReconnectRequest() - self.__client_ssl_queue.put(reconnect) - reconnect.done.wait() + self.yield_from_callback(Reconnect()) if self._sni_from_handshake: # Now, change client context to reflect possibly changed certificate: @@ -163,6 +135,7 @@ class SslLayer(Layer): except: # pragma: no cover self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") + @yield_from_callback def _establish_ssl_with_client(self): self.log("Establish SSL with client", "debug") cert, key, chain_file = self.find_cert() @@ -227,4 +200,4 @@ class SslLayer(Layer): if self._sni_from_server_change: sans.add(self._sni_from_server_change) - return self.config.certstore.get_cert(host, list(sans)) \ No newline at end of file + return self.config.certstore.get_cert(host, list(sans)) -- cgit v1.2.3 From 314e0f5839fcd4a1c35323f61938b207232de287 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 6 Aug 2015 12:32:33 +0200 Subject: add reverseproxy mode, fix bugs --- libmproxy/protocol2/__init__.py | 3 ++- libmproxy/protocol2/layer.py | 12 ++++++------ libmproxy/protocol2/reverse_proxy.py | 19 +++++++++++++++++++ libmproxy/protocol2/ssl.py | 14 +++++++------- libmproxy/proxy/server.py | 2 +- 5 files changed, 35 insertions(+), 15 deletions(-) create mode 100644 libmproxy/protocol2/reverse_proxy.py (limited to 'libmproxy') diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index 95f67c6c..3f714f62 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -1,6 +1,7 @@ from __future__ import (absolute_import, print_function, division) from .layer import RootContext from .socks import Socks5IncomingLayer +from .reverse_proxy import ReverseProxy from .rawtcp import TcpLayer from .auto import AutoLayer -__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext"] +__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext", "ReverseProxy"] diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index aaa51baf..c18be83c 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -176,7 +176,7 @@ def yield_from_callback(fun): """ yield_queue = Queue.Queue() - def do_yield(self, msg): + def do_yield(msg): yield_queue.put(msg) yield_queue.get() @@ -192,14 +192,14 @@ def yield_from_callback(fun): threading.Thread(target=run, name="YieldFromCallbackThread").start() while True: - e = yield_queue.get() - if e is True: + msg = yield_queue.get() + if msg is True: break - elif isinstance(e, Exception): + elif isinstance(msg, Exception): # TODO: Include func name? - raise ProxyError2("Error from callback: " + repr(e), e) + raise ProxyError2("Error in %s: %s" % (fun.__name__, repr(msg)), msg) else: - yield e + yield msg yield_queue.put(None) self.yield_from_callback = None diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py new file mode 100644 index 00000000..dfffd2f2 --- /dev/null +++ b/libmproxy/protocol2/reverse_proxy.py @@ -0,0 +1,19 @@ +from __future__ import (absolute_import, print_function, division) + +from .layer import Layer, ServerConnectionMixin +from .ssl import SslLayer + + +class ReverseProxy(Layer, ServerConnectionMixin): + + def __init__(self, ctx, server_address, client_ssl, server_ssl): + super(ReverseProxy, self).__init__(ctx) + self.server_address = server_address + self.client_ssl = client_ssl + self.server_ssl = server_ssl + + def __call__(self): + layer = SslLayer(self, self.client_ssl, self.server_ssl) + for message in layer(): + if not self._handle_server_message(message): + yield message diff --git a/libmproxy/protocol2/ssl.py b/libmproxy/protocol2/ssl.py index 32798e72..a744a979 100644 --- a/libmproxy/protocol2/ssl.py +++ b/libmproxy/protocol2/ssl.py @@ -14,7 +14,7 @@ class SslLayer(Layer): self._client_ssl = client_ssl self._server_ssl = server_ssl self._connected = False - self._sni_from_handshake = None + self.client_sni = None self._sni_from_server_change = None def __call__(self): @@ -74,7 +74,7 @@ class SslLayer(Layer): if self._sni_from_server_change is False: return None else: - return self._sni_from_server_change or self._sni_from_handshake + return self._sni_from_server_change or self.client_sni def _establish_ssl_with_client_and_server(self): """ @@ -97,7 +97,7 @@ class SslLayer(Layer): else: raise RuntimeError("Unexpected Message: %s" % message) - if server_err and not self._sni_from_handshake: + if server_err and not self.client_sni: raise server_err def handle_sni(self, connection): @@ -111,14 +111,14 @@ class SslLayer(Layer): sn = connection.get_servername() if not sn: return - self._sni_from_handshake = sn.decode("utf8").encode("idna") + self.client_sni = sn.decode("utf8").encode("idna") if old_upstream_sni != self.sni_for_upstream_connection: # Perform reconnect if self.server_ssl: self.yield_from_callback(Reconnect()) - if self._sni_from_handshake: + if self.client_sni: # Now, change client context to reflect possibly changed certificate: cert, key, chain_file = self.find_cert() new_context = self.client_conn.create_ssl_context( @@ -195,8 +195,8 @@ class SslLayer(Layer): sans.add(host) host = upstream_cert.cn.decode("utf8").encode("idna") # Also add SNI values. - if self._sni_from_handshake: - sans.add(self._sni_from_handshake) + if self.client_sni: + sans.add(self.client_sni) if self._sni_from_server_change: sans.add(self._sni_from_server_change) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index c8990a9a..32d596ad 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -79,7 +79,7 @@ class ConnectionHandler2: self.config, self.channel ) - root_layer = protocol2.Socks5IncomingLayer(root_context) + root_layer = protocol2.ReverseProxy(root_context, ("localhost", 5000), True, True) try: for message in root_layer(): -- cgit v1.2.3 From 026330a3b014f24f095b839b29186036854de3bc Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 8 Aug 2015 16:08:57 +0200 Subject: cleaner Exceptions, ssl -> tls, upstream proxy mode --- libmproxy/exceptions.py | 22 ++++ libmproxy/protocol2/__init__.py | 3 +- libmproxy/protocol2/auto.py | 4 +- libmproxy/protocol2/layer.py | 10 +- libmproxy/protocol2/messages.py | 4 +- libmproxy/protocol2/rawtcp.py | 8 +- libmproxy/protocol2/reverse_proxy.py | 10 +- libmproxy/protocol2/socks.py | 6 +- libmproxy/protocol2/ssl.py | 203 ------------------------------- libmproxy/protocol2/tls.py | 203 +++++++++++++++++++++++++++++++ libmproxy/protocol2/transparent_proxy.py | 24 ++++ libmproxy/protocol2/upstream_proxy.py | 18 +++ libmproxy/proxy/connection.py | 8 ++ libmproxy/proxy/primitives.py | 6 - libmproxy/proxy/server.py | 6 +- 15 files changed, 306 insertions(+), 229 deletions(-) create mode 100644 libmproxy/exceptions.py delete mode 100644 libmproxy/protocol2/ssl.py create mode 100644 libmproxy/protocol2/tls.py create mode 100644 libmproxy/protocol2/transparent_proxy.py create mode 100644 libmproxy/protocol2/upstream_proxy.py (limited to 'libmproxy') diff --git a/libmproxy/exceptions.py b/libmproxy/exceptions.py new file mode 100644 index 00000000..4d98c024 --- /dev/null +++ b/libmproxy/exceptions.py @@ -0,0 +1,22 @@ +from __future__ import (absolute_import, print_function, division) + + +class ProxyException(Exception): + """ + Base class for all exceptions thrown by libmproxy. + """ + def __init__(self, message, cause=None): + """ + :param message: Error Message + :param cause: Exception object that caused this exception to be thrown. + """ + super(ProxyException, self).__init__(message) + self.cause = cause + + +class ProtocolException(ProxyException): + pass + + +class ServerException(ProxyException): + pass \ No newline at end of file diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index 3f714f62..0d232b13 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -2,6 +2,7 @@ from __future__ import (absolute_import, print_function, division) from .layer import RootContext from .socks import Socks5IncomingLayer from .reverse_proxy import ReverseProxy +from .upstream_proxy import UpstreamProxy from .rawtcp import TcpLayer from .auto import AutoLayer -__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext", "ReverseProxy"] +__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext", "ReverseProxy", "UpstreamProxy"] diff --git a/libmproxy/protocol2/auto.py b/libmproxy/protocol2/auto.py index fc111758..4a930720 100644 --- a/libmproxy/protocol2/auto.py +++ b/libmproxy/protocol2/auto.py @@ -10,11 +10,11 @@ class AutoLayer(Layer): return # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello if d[0] == "\x16": - layer = SslLayer(self, True, True) + layer = TlsLayer(self, True, True) else: layer = TcpLayer(self) for m in layer(): yield m from .rawtcp import TcpLayer -from .ssl import SslLayer +from .tls import TlsLayer diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index c18be83c..8aede22e 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -35,9 +35,10 @@ from __future__ import (absolute_import, print_function, division) import Queue import threading from netlib import tcp -from ..proxy import ProxyError2, Log +from ..proxy import Log from ..proxy.connection import ServerConnection from .messages import Connect, Reconnect, ChangeServer +from ..exceptions import ProtocolException class RootContext(object): @@ -51,6 +52,9 @@ class RootContext(object): self.channel = channel # provides .ask() method to communicate with FlowMaster self.config = config # Proxy Configuration + def next_layer(self): + print(type(self)) + class _LayerCodeCompletion(object): """ @@ -149,7 +153,7 @@ class ServerConnectionMixin(object): try: self.server_conn.connect() except tcp.NetLibError as e: - raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) + raise ProtocolException("Server connection to '%s' failed: %s" % (self.server_address, e), e) def yield_from_callback(fun): @@ -197,7 +201,7 @@ def yield_from_callback(fun): break elif isinstance(msg, Exception): # TODO: Include func name? - raise ProxyError2("Error in %s: %s" % (fun.__name__, repr(msg)), msg) + raise ProtocolException("Error in %s: %s" % (fun.__name__, repr(msg)), msg) else: yield msg yield_queue.put(None) diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index baf4312d..3f53fbd4 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -32,9 +32,9 @@ class ChangeServer(_Message): Change the upstream server. """ - def __init__(self, address, server_ssl, sni, depth=1): + def __init__(self, address, server_tls, sni, depth=1): self.address = address - self.server_ssl = server_ssl + self.server_tls = server_tls self.sni = sni # upstream proxy scenario: you may want to change either the final target or the upstream proxy. diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index 39e48e24..608a53e3 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -1,4 +1,6 @@ from __future__ import (absolute_import, print_function, division) +import OpenSSL +from ..exceptions import ProtocolException from ..protocol.tcp import TCPHandler from .layer import Layer from .messages import Connect @@ -8,7 +10,11 @@ class TcpLayer(Layer): def __call__(self): yield Connect() tcp_handler = TCPHandler(self) - tcp_handler.handle_messages() + try: + tcp_handler.handle_messages() + except OpenSSL.SSL.Error as e: + raise ProtocolException("SSL error: %s" % repr(e), e) + def establish_server_connection(self): pass diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index dfffd2f2..cb6d1d78 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -1,19 +1,19 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin -from .ssl import SslLayer +from .tls import TlsLayer class ReverseProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx, server_address, client_ssl, server_ssl): + def __init__(self, ctx, server_address, client_tls, server_tls): super(ReverseProxy, self).__init__(ctx) self.server_address = server_address - self.client_ssl = client_ssl - self.server_ssl = server_ssl + self._client_tls = client_tls + self._server_tls = server_tls def __call__(self): - layer = SslLayer(self, self.client_ssl, self.server_ssl) + layer = TlsLayer(self, self._client_tls, self._server_tls) for message in layer(): if not self._handle_server_message(message): yield message diff --git a/libmproxy/protocol2/socks.py b/libmproxy/protocol2/socks.py index 14564521..1222ef5c 100644 --- a/libmproxy/protocol2/socks.py +++ b/libmproxy/protocol2/socks.py @@ -1,10 +1,10 @@ from __future__ import (absolute_import, print_function, division) -from ..proxy import ProxyError, Socks5ProxyMode, ProxyError2 +from ..exceptions import ProtocolException +from ..proxy import ProxyError, Socks5ProxyMode from .layer import Layer, ServerConnectionMixin from .auto import AutoLayer - class Socks5IncomingLayer(Layer, ServerConnectionMixin): def __call__(self): try: @@ -12,7 +12,7 @@ class Socks5IncomingLayer(Layer, ServerConnectionMixin): address = s5mode.get_upstream_server(self.client_conn)[2:] except ProxyError as e: # TODO: Unmonkeypatch - raise ProxyError2(str(e), e) + raise ProtocolException(str(e), e) self.server_address = address diff --git a/libmproxy/protocol2/ssl.py b/libmproxy/protocol2/ssl.py deleted file mode 100644 index a744a979..00000000 --- a/libmproxy/protocol2/ssl.py +++ /dev/null @@ -1,203 +0,0 @@ -from __future__ import (absolute_import, print_function, division) -import traceback -from netlib import tcp - -from ..proxy import ProxyError2 -from .layer import Layer, yield_from_callback -from .messages import Connect, Reconnect, ChangeServer -from .auto import AutoLayer - - -class SslLayer(Layer): - def __init__(self, ctx, client_ssl, server_ssl): - super(SslLayer, self).__init__(ctx) - self._client_ssl = client_ssl - self._server_ssl = server_ssl - self._connected = False - self.client_sni = None - self._sni_from_server_change = None - - def __call__(self): - """ - The strategy for establishing SSL is as follows: - First, we determine whether we need the server cert to establish ssl with the client. - If so, we first connect to the server and then to the client. - If not, we only connect to the client and do the server_ssl lazily on a Connect message. - - An additional complexity is that establish ssl with the server may require a SNI value from the client. - In an ideal world, we'd do the following: - 1. Start the SSL handshake with the client - 2. Check if the client sends a SNI. - 3. Pause the client handshake, establish SSL with the server. - 4. Finish the client handshake with the certificate from the server. - There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( - Thus, we resort to the following workaround when establishing SSL with the server: - 1. Try to establish SSL with the server without SNI. If this fails, we ignore it. - 2. Establish SSL with client. - - If there's a SNI callback, reconnect to the server with SNI. - - If not and the server connect failed, raise the original exception. - Further notes: - - OpenSSL 1.0.2 introduces a callback that would help here: - https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html - - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 - """ - client_ssl_requires_server_cert = ( - self._client_ssl and self._server_ssl and not self.config.no_upstream_cert - ) - lazy_server_ssl = ( - self._server_ssl and not client_ssl_requires_server_cert - ) - - if client_ssl_requires_server_cert: - for m in self._establish_ssl_with_client_and_server(): - yield m - elif self.client_ssl: - for m in self._establish_ssl_with_client(): - yield m - - layer = AutoLayer(self) - for message in layer(): - if message != Connect or not self._connected: - yield message - if message == Connect: - if lazy_server_ssl: - self._establish_ssl_with_server() - if message == ChangeServer and message.depth == 1: - self.server_ssl = message.server_ssl - self._sni_from_server_change = message.sni - if message == Reconnect or message == ChangeServer: - if self.server_ssl: - self._establish_ssl_with_server() - - @property - def sni_for_upstream_connection(self): - if self._sni_from_server_change is False: - return None - else: - return self._sni_from_server_change or self.client_sni - - def _establish_ssl_with_client_and_server(self): - """ - This function deals with the problem that the server may require a SNI value from the client. - """ - - # First, try to connect to the server. - yield Connect() - self._connected = True - server_err = None - try: - self._establish_ssl_with_server() - except ProxyError2 as e: - server_err = e - - for message in self._establish_ssl_with_client(): - if message == Reconnect: - yield message - self._establish_ssl_with_server() - else: - raise RuntimeError("Unexpected Message: %s" % message) - - if server_err and not self.client_sni: - raise server_err - - def handle_sni(self, connection): - """ - This callback gets called during the SSL handshake with the client. - The client has just sent the Sever Name Indication (SNI). - """ - try: - old_upstream_sni = self.sni_for_upstream_connection - - sn = connection.get_servername() - if not sn: - return - self.client_sni = sn.decode("utf8").encode("idna") - - if old_upstream_sni != self.sni_for_upstream_connection: - # Perform reconnect - if self.server_ssl: - self.yield_from_callback(Reconnect()) - - if self.client_sni: - # Now, change client context to reflect possibly changed certificate: - cert, key, chain_file = self.find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - connection.set_context(new_context) - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except: # pragma: no cover - self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") - - @yield_from_callback - def _establish_ssl_with_client(self): - self.log("Establish SSL with client", "debug") - cert, key, chain_file = self.find_cert() - try: - self.client_conn.convert_to_ssl( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - handle_sni=self.handle_sni, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - except tcp.NetLibError as e: - raise ProxyError2(repr(e), e) - - def _establish_ssl_with_server(self): - self.log("Establish SSL with server", "debug") - try: - self.server_conn.establish_ssl( - self.config.clientcerts, - self.sni_for_upstream_connection, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify_options=self.config.openssl_verification_mode_server, - ca_path=self.config.openssl_trusted_cadir_server, - ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, - ) - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") - except tcp.NetLibInvalidCertificateError as e: - ssl_cert_err = self.server_conn.ssl_verification_error - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") - raise ProxyError2(repr(e), e) - except tcp.NetLibError as e: - raise ProxyError2(repr(e), e) - - def find_cert(self): - host = self.server_conn.address.host - # TODO: Better use an OrderedSet here - sans = set() - # Incorporate upstream certificate - if self.server_conn.ssl_established and (not self.config.no_upstream_cert): - upstream_cert = self.server_conn.cert - sans.update(upstream_cert.altnames) - if upstream_cert.cn: - sans.add(host) - host = upstream_cert.cn.decode("utf8").encode("idna") - # Also add SNI values. - if self.client_sni: - sans.add(self.client_sni) - if self._sni_from_server_change: - sans.add(self._sni_from_server_change) - - return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py new file mode 100644 index 00000000..2362b2b2 --- /dev/null +++ b/libmproxy/protocol2/tls.py @@ -0,0 +1,203 @@ +from __future__ import (absolute_import, print_function, division) +import traceback +from netlib import tcp + +from ..exceptions import ProtocolException +from .layer import Layer, yield_from_callback +from .messages import Connect, Reconnect, ChangeServer +from .auto import AutoLayer + + +class TlsLayer(Layer): + def __init__(self, ctx, client_tls, server_tls): + super(TlsLayer, self).__init__(ctx) + self._client_tls = client_tls + self._server_tls = server_tls + self._connected = False + self.client_sni = None + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we resort to the following workaround when establishing SSL with the server: + 1. Try to establish SSL with the server without SNI. If this fails, we ignore it. + 2. Establish SSL with client. + - If there's a SNI callback, reconnect to the server with SNI. + - If not and the server connect failed, raise the original exception. + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + client_tls_requires_server_cert = ( + self._client_tls and self._server_tls and not self.config.no_upstream_cert + ) + lazy_server_tls = ( + self._server_tls and not client_tls_requires_server_cert + ) + + if client_tls_requires_server_cert: + for m in self._establish_tls_with_client_and_server(): + yield m + elif self._client_tls: + for m in self._establish_tls_with_client(): + yield m + + self.next_layer() + layer = AutoLayer(self) + for message in layer(): + if message != Connect or not self._connected: + yield message + if message == Connect: + if lazy_server_tls: + self._establish_tls_with_server() + if message == ChangeServer and message.depth == 1: + self._server_tls = message.server_tls + self._sni_from_server_change = message.sni + if message == Reconnect or message == ChangeServer: + if self._server_tls: + self._establish_tls_with_server() + + @property + def sni_for_upstream_connection(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self.client_sni + + def _establish_tls_with_client_and_server(self): + """ + This function deals with the problem that the server may require a SNI value from the client. + """ + + # First, try to connect to the server. + yield Connect() + self._connected = True + server_err = None + try: + self._establish_tls_with_server() + except ProtocolException as e: + server_err = e + + for message in self._establish_tls_with_client(): + if message == Reconnect: + yield message + self._establish_tls_with_server() + else: + raise RuntimeError("Unexpected Message: %s" % message) + + if server_err and not self.client_sni: + raise server_err + + def handle_sni(self, connection): + """ + This callback gets called during the TLS handshake with the client. + The client has just sent the Sever Name Indication (SNI). + """ + try: + old_upstream_sni = self.sni_for_upstream_connection + + sn = connection.get_servername() + if not sn: + return + self.client_sni = sn.decode("utf8").encode("idna") + + if old_upstream_sni != self.sni_for_upstream_connection: + # Perform reconnect + if self._server_tls: + self.yield_from_callback(Reconnect()) + + if self.client_sni: + # Now, change client context to reflect possibly changed certificate: + cert, key, chain_file = self.find_cert() + new_context = self.client_conn.create_ssl_context( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file + ) + connection.set_context(new_context) + # An unhandled exception in this method will core dump PyOpenSSL, so + # make dang sure it doesn't happen. + except: # pragma: no cover + self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") + + @yield_from_callback + def _establish_tls_with_client(self): + self.log("Establish TLS with client", "debug") + cert, key, chain_file = self.find_cert() + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + handle_sni=self.handle_sni, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file + ) + except tcp.NetLibError as e: + raise ProtocolException(repr(e), e) + + def _establish_tls_with_server(self): + self.log("Establish TLS with server", "debug") + try: + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni_for_upstream_connection, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + ) + tls_cert_err = self.server_conn.ssl_verification_error + if tls_cert_err is not None: + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except tcp.NetLibInvalidCertificateError as e: + tls_cert_err = self.server_conn.ssl_verification_error + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProtocolException(repr(e), e) + except tcp.NetLibError as e: + raise ProtocolException(repr(e), e) + + def find_cert(self): + host = self.server_conn.address.host + sans = set() + # Incorporate upstream certificate + if self.server_conn.tls_established and (not self.config.no_upstream_cert): + upstream_cert = self.server_conn.cert + sans.update(upstream_cert.altnames) + if upstream_cert.cn: + sans.add(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self.client_sni: + sans.add(self.client_sni) + if self._sni_from_server_change: + sans.add(self._sni_from_server_change) + + return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py new file mode 100644 index 00000000..078954c2 --- /dev/null +++ b/libmproxy/protocol2/transparent_proxy.py @@ -0,0 +1,24 @@ +from __future__ import (absolute_import, print_function, division) + +from ..exceptions import ProtocolException +from .. import platform +from .layer import Layer, ServerConnectionMixin +from .auto import AutoLayer + + +class TransparentProxy(Layer, ServerConnectionMixin): + + def __init__(self, ctx): + super(TransparentProxy, self).__init__(ctx) + self.resolver = platform.resolver() + + def __call__(self): + try: + self.server_address = self.resolver.original_addr(self.client_conn.connection) + except Exception as e: + raise ProtocolException("Transparent mode failure: %s" % repr(e), e) + + layer = AutoLayer(self) + for message in layer(): + if not self._handle_server_message(message): + yield message diff --git a/libmproxy/protocol2/upstream_proxy.py b/libmproxy/protocol2/upstream_proxy.py new file mode 100644 index 00000000..bd920309 --- /dev/null +++ b/libmproxy/protocol2/upstream_proxy.py @@ -0,0 +1,18 @@ +from __future__ import (absolute_import, print_function, division) + +from .layer import Layer, ServerConnectionMixin +#from .http import HttpLayer + + +class UpstreamProxy(Layer, ServerConnectionMixin): + + def __init__(self, ctx, server_address): + super(UpstreamProxy, self).__init__(ctx) + self.server_address = server_address + + def __call__(self): + #layer = HttpLayer(self) + layer = None + for message in layer(): + if not self._handle_server_message(message): + yield message diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index 9e03157a..49210e47 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -32,6 +32,10 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): port=self.address.port ) + @property + def tls_established(self): + return self.ssl_established + _stateobject_attributes = dict( ssl_established=bool, timestamp_start=float, @@ -112,6 +116,10 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): port=self.address.port ) + @property + def tls_established(self): + return self.ssl_established + _stateobject_attributes = dict( state=list, timestamp_start=float, diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index fd4eb882..a9f31181 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -2,12 +2,6 @@ from __future__ import absolute_import from netlib import socks, tcp -class ProxyError2(Exception): - def __init__(self, message, cause=None): - super(ProxyError2, self).__init__(message) - self.cause = cause - - class ProxyError(Exception): def __init__(self, code, message, headers=None): super(ProxyError, self).__init__(message) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 32d596ad..c107cbed 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -7,7 +7,7 @@ from netlib import tcp from ..protocol.handle import protocol_handler from .. import protocol2 -from .primitives import ProxyServerError, Log, ProxyError, ProxyError2 +from .primitives import ProxyServerError, Log, ProxyError from .connection import ClientConnection, ServerConnection @@ -79,12 +79,12 @@ class ConnectionHandler2: self.config, self.channel ) - root_layer = protocol2.ReverseProxy(root_context, ("localhost", 5000), True, True) + root_layer = protocol2.Socks5IncomingLayer(root_context) try: for message in root_layer(): print("Root layer receveived: %s" % message) - except ProxyError2 as e: + except protocol2.ProtocolException as e: self.log(e, "info") except Exception: self.log(traceback.format_exc(), "error") -- cgit v1.2.3 From aef3b626a70de5f385c8f5496c2e49575b5c3e1c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 11 Aug 2015 20:27:34 +0200 Subject: wip commit --- libmproxy/exceptions.py | 10 +++- libmproxy/protocol2/__init__.py | 12 +++-- libmproxy/protocol2/auto.py | 20 ------- libmproxy/protocol2/http.py | 90 +++++++++++++++++++++++++++++++ libmproxy/protocol2/http_protocol_mock.py | 13 +++++ libmproxy/protocol2/http_proxy.py | 23 ++++++++ libmproxy/protocol2/layer.py | 17 +----- libmproxy/protocol2/root_context.py | 32 +++++++++++ libmproxy/protocol2/socks.py | 22 -------- libmproxy/protocol2/socks_proxy.py | 22 ++++++++ libmproxy/protocol2/tls.py | 4 +- libmproxy/protocol2/transparent_proxy.py | 3 +- libmproxy/protocol2/upstream_proxy.py | 18 ------- libmproxy/proxy/server.py | 5 +- 14 files changed, 202 insertions(+), 89 deletions(-) delete mode 100644 libmproxy/protocol2/auto.py create mode 100644 libmproxy/protocol2/http.py create mode 100644 libmproxy/protocol2/http_protocol_mock.py create mode 100644 libmproxy/protocol2/http_proxy.py create mode 100644 libmproxy/protocol2/root_context.py delete mode 100644 libmproxy/protocol2/socks.py create mode 100644 libmproxy/protocol2/socks_proxy.py delete mode 100644 libmproxy/protocol2/upstream_proxy.py (limited to 'libmproxy') diff --git a/libmproxy/exceptions.py b/libmproxy/exceptions.py index 4d98c024..3825c409 100644 --- a/libmproxy/exceptions.py +++ b/libmproxy/exceptions.py @@ -18,5 +18,13 @@ class ProtocolException(ProxyException): pass +class HttpException(ProtocolException): + pass + + +class InvalidCredentials(HttpException): + pass + + class ServerException(ProxyException): - pass \ No newline at end of file + pass diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index 0d232b13..e3f06ad7 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -1,8 +1,10 @@ from __future__ import (absolute_import, print_function, division) -from .layer import RootContext -from .socks import Socks5IncomingLayer +from .root_context import RootContext +from .socks_proxy import Socks5Proxy from .reverse_proxy import ReverseProxy -from .upstream_proxy import UpstreamProxy +from .http_proxy import HttpProxy, HttpUpstreamProxy from .rawtcp import TcpLayer -from .auto import AutoLayer -__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext", "ReverseProxy", "UpstreamProxy"] + +__all__ = [ + "Socks5Proxy", "TcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" +] diff --git a/libmproxy/protocol2/auto.py b/libmproxy/protocol2/auto.py deleted file mode 100644 index 4a930720..00000000 --- a/libmproxy/protocol2/auto.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import (absolute_import, print_function, division) -from .layer import Layer - - -class AutoLayer(Layer): - def __call__(self): - d = self.client_conn.rfile.peek(1) - - if not d: - return - # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello - if d[0] == "\x16": - layer = TlsLayer(self, True, True) - else: - layer = TcpLayer(self) - for m in layer(): - yield m - -from .rawtcp import TcpLayer -from .tls import TlsLayer diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py new file mode 100644 index 00000000..54cc9dbc --- /dev/null +++ b/libmproxy/protocol2/http.py @@ -0,0 +1,90 @@ +from __future__ import (absolute_import, print_function, division) + +from .layer import Layer, ServerConnectionMixin +from libmproxy import version +from libmproxy.exceptions import InvalidCredentials +from libmproxy.protocol.http import HTTPFlow +from libmproxy.protocol.http_wrappers import HTTPResponse +from libmproxy.protocol2.http_protocol_mock import HTTP1 +from netlib import tcp +from netlib.http import status_codes +from netlib import odict + + +def send_http_error_response(status_code, message, headers=odict.ODictCaseless()): + response = status_codes.RESPONSES.get(status_code, "Unknown") + body = """ + + + %d %s + + %s + + """.strip() % (status_code, response, message) + + headers["Server"] = [version.NAMEVERSION] + headers["Connection"] = ["close"] + headers["Content-Length"] = [len(body)] + headers["Content-Type"] = ["text/html"] + + resp = HTTPResponse( + (1, 1), # if HTTP/2 is used, this value is ignored anyway + status_code, + response, + headers, + body, + ) + + protocol = self.c.client_conn.protocol or http1.HTTP1Protocol(self.c.client_conn) + self.c.client_conn.send(protocol.assemble(resp)) + +class HttpLayer(Layer, ServerConnectionMixin): + """ + HTTP 1 Layer + """ + + def __init__(self, ctx): + super(HttpLayer, self).__init__(ctx) + self.skip_authentication = False + + def __call__(self): + while True: + flow = HTTPFlow(self.client_conn, self.server_conn) + try: + request = HTTP1.read_request( + self.client_conn, + body_size_limit=self.c.config.body_size_limit + ) + except tcp.NetLibError: + # don't throw an error for disconnects that happen + # before/between requests. + return + + self.c.log("request", "debug", [repr(request)]) + + self.check_authentication(request) + + if self.mode == "regular" and request.form_in == "authority": + raise NotImplementedError + + + + ret = self.process_request(flow, request) + if ret is True: + continue + if ret is False: + return + + def check_authentication(self, request): + if self.config.authenticator: + if self.config.authenticator.authenticate(request.headers): + self.config.authenticator.clean(request.headers) + else: + self.send_error() + raise InvalidCredentials("Proxy Authentication Required") + raise http.HttpAuthenticationError( + self.c.config.authenticator.auth_challenge_headers()) + return request.headers + + def send_error(self, code, message, headers): + pass \ No newline at end of file diff --git a/libmproxy/protocol2/http_protocol_mock.py b/libmproxy/protocol2/http_protocol_mock.py new file mode 100644 index 00000000..962a76d6 --- /dev/null +++ b/libmproxy/protocol2/http_protocol_mock.py @@ -0,0 +1,13 @@ +""" +Temporary mock to sort out API discrepancies +""" +from netlib.http.http1 import HTTP1Protocol + + +class HTTP1(object): + @staticmethod + def read_request(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection).read_request(*args, **kwargs) \ No newline at end of file diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py new file mode 100644 index 00000000..6b3b6a82 --- /dev/null +++ b/libmproxy/protocol2/http_proxy.py @@ -0,0 +1,23 @@ +from __future__ import (absolute_import, print_function, division) + +from .layer import Layer, ServerConnectionMixin +from .http import HttpLayer + + +class HttpProxy(Layer): + def __call__(self): + layer = HttpLayer(self) + for message in layer(): + yield message + + +class HttpUpstreamProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address): + super(HttpUpstreamProxy, self).__init__(ctx) + self.server_address = server_address + + def __call__(self): + layer = HttpLayer(self) + for message in layer(): + if not self._handle_server_message(message): + yield message diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 8aede22e..0ae64c43 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -41,21 +41,6 @@ from .messages import Connect, Reconnect, ChangeServer from ..exceptions import ProtocolException -class RootContext(object): - """ - The outmost context provided to the root layer. - As a consequence, every layer has .client_conn, .channel and .config. - """ - - def __init__(self, client_conn, config, channel): - self.client_conn = client_conn # Client Connection - self.channel = channel # provides .ask() method to communicate with FlowMaster - self.config = config # Proxy Configuration - - def next_layer(self): - print(type(self)) - - class _LayerCodeCompletion(object): """ Dummy class that provides type hinting in PyCharm, which simplifies development a lot. @@ -208,4 +193,4 @@ def yield_from_callback(fun): self.yield_from_callback = None - return wrapper + return wrapper \ No newline at end of file diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py new file mode 100644 index 00000000..cbe596aa --- /dev/null +++ b/libmproxy/protocol2/root_context.py @@ -0,0 +1,32 @@ +from .rawtcp import TcpLayer +from .tls import TlsLayer + + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel, .next_layer() and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def next_layer(self, top_layer): + """ + This function determines the next layer in the protocol stack. + :param top_layer: the current top layer + :return: The next layer. + """ + + d = top_layer.client_conn.rfile.peek(1) + + if not d: + return + # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + if d[0] == "\x16": + layer = TlsLayer(top_layer, True, True) + else: + layer = TcpLayer(top_layer) + return layer diff --git a/libmproxy/protocol2/socks.py b/libmproxy/protocol2/socks.py deleted file mode 100644 index 1222ef5c..00000000 --- a/libmproxy/protocol2/socks.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from ..exceptions import ProtocolException -from ..proxy import ProxyError, Socks5ProxyMode -from .layer import Layer, ServerConnectionMixin -from .auto import AutoLayer - -class Socks5IncomingLayer(Layer, ServerConnectionMixin): - def __call__(self): - try: - s5mode = Socks5ProxyMode(self.config.ssl_ports) - address = s5mode.get_upstream_server(self.client_conn)[2:] - except ProxyError as e: - # TODO: Unmonkeypatch - raise ProtocolException(str(e), e) - - self.server_address = address - - layer = AutoLayer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py new file mode 100644 index 00000000..c89477ca --- /dev/null +++ b/libmproxy/protocol2/socks_proxy.py @@ -0,0 +1,22 @@ +from __future__ import (absolute_import, print_function, division) + +from ..exceptions import ProtocolException +from ..proxy import ProxyError, Socks5ProxyMode +from .layer import Layer, ServerConnectionMixin + + +class Socks5Proxy(Layer, ServerConnectionMixin): + def __call__(self): + try: + s5mode = Socks5ProxyMode(self.config.ssl_ports) + address = s5mode.get_upstream_server(self.client_conn)[2:] + except ProxyError as e: + # TODO: Unmonkeypatch + raise ProtocolException(str(e), e) + + self.server_address = address + + layer = self.ctx.next_layer(self) + for message in layer(): + if not self._handle_server_message(message): + yield message diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 2362b2b2..999cbea6 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -5,7 +5,6 @@ from netlib import tcp from ..exceptions import ProtocolException from .layer import Layer, yield_from_callback from .messages import Connect, Reconnect, ChangeServer -from .auto import AutoLayer class TlsLayer(Layer): @@ -55,8 +54,7 @@ class TlsLayer(Layer): for m in self._establish_tls_with_client(): yield m - self.next_layer() - layer = AutoLayer(self) + layer = self.ctx.next_layer(self) for message in layer(): if message != Connect or not self._connected: yield message diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py index 078954c2..f073e2f8 100644 --- a/libmproxy/protocol2/transparent_proxy.py +++ b/libmproxy/protocol2/transparent_proxy.py @@ -3,7 +3,6 @@ from __future__ import (absolute_import, print_function, division) from ..exceptions import ProtocolException from .. import platform from .layer import Layer, ServerConnectionMixin -from .auto import AutoLayer class TransparentProxy(Layer, ServerConnectionMixin): @@ -18,7 +17,7 @@ class TransparentProxy(Layer, ServerConnectionMixin): except Exception as e: raise ProtocolException("Transparent mode failure: %s" % repr(e), e) - layer = AutoLayer(self) + layer = self.ctx.next_layer(self) for message in layer(): if not self._handle_server_message(message): yield message diff --git a/libmproxy/protocol2/upstream_proxy.py b/libmproxy/protocol2/upstream_proxy.py deleted file mode 100644 index bd920309..00000000 --- a/libmproxy/protocol2/upstream_proxy.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from .layer import Layer, ServerConnectionMixin -#from .http import HttpLayer - - -class UpstreamProxy(Layer, ServerConnectionMixin): - - def __init__(self, ctx, server_address): - super(UpstreamProxy, self).__init__(ctx) - self.server_address = server_address - - def __call__(self): - #layer = HttpLayer(self) - layer = None - for message in layer(): - if not self._handle_server_message(message): - yield message diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index c107cbed..6a7048e0 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -7,6 +7,7 @@ from netlib import tcp from ..protocol.handle import protocol_handler from .. import protocol2 +from ..exceptions import ProtocolException from .primitives import ProxyServerError, Log, ProxyError from .connection import ClientConnection, ServerConnection @@ -79,12 +80,12 @@ class ConnectionHandler2: self.config, self.channel ) - root_layer = protocol2.Socks5IncomingLayer(root_context) + root_layer = protocol2.Socks5Proxy(root_context) try: for message in root_layer(): print("Root layer receveived: %s" % message) - except protocol2.ProtocolException as e: + except ProtocolException as e: self.log(e, "info") except Exception: self.log(traceback.format_exc(), "error") -- cgit v1.2.3 From 808218f4bc64be8de065604f6509eb75d98fde88 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 14 Aug 2015 10:41:11 +0200 Subject: more work on http layer --- libmproxy/protocol2/__init__.py | 4 +- libmproxy/protocol2/http.py | 174 ++++++++++++++++++++++++++++++----- libmproxy/protocol2/http_proxy.py | 5 +- libmproxy/protocol2/layer.py | 11 ++- libmproxy/protocol2/rawtcp.py | 3 +- libmproxy/protocol2/reverse_proxy.py | 5 +- libmproxy/protocol2/root_context.py | 29 +++++- libmproxy/protocol2/tls.py | 11 ++- 8 files changed, 200 insertions(+), 42 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index e3f06ad7..d5dafaae 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -3,8 +3,8 @@ from .root_context import RootContext from .socks_proxy import Socks5Proxy from .reverse_proxy import ReverseProxy from .http_proxy import HttpProxy, HttpUpstreamProxy -from .rawtcp import TcpLayer +from .rawtcp import RawTcpLayer __all__ = [ - "Socks5Proxy", "TcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" + "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" ] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 54cc9dbc..44ebf6a8 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -1,17 +1,22 @@ from __future__ import (absolute_import, print_function, division) +from .. import version +from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer, ServerConnectionMixin -from libmproxy import version -from libmproxy.exceptions import InvalidCredentials +from .messages import ChangeServer, Connect, Reconnect +from .http_proxy import HttpProxy, HttpUpstreamProxy +from libmproxy.protocol import KILL + from libmproxy.protocol.http import HTTPFlow -from libmproxy.protocol.http_wrappers import HTTPResponse +from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from libmproxy.protocol2.http_protocol_mock import HTTP1 +from libmproxy.protocol2.tls import TlsLayer from netlib import tcp from netlib.http import status_codes from netlib import odict -def send_http_error_response(status_code, message, headers=odict.ODictCaseless()): +def make_error_response(status_code, message, headers=None): response = status_codes.RESPONSES.get(status_code, "Unknown") body = """ @@ -22,21 +27,40 @@ def send_http_error_response(status_code, message, headers=odict.ODictCaseless() """.strip() % (status_code, response, message) + if not headers: + headers = odict.ODictCaseless() headers["Server"] = [version.NAMEVERSION] headers["Connection"] = ["close"] headers["Content-Length"] = [len(body)] headers["Content-Type"] = ["text/html"] - resp = HTTPResponse( - (1, 1), # if HTTP/2 is used, this value is ignored anyway + return HTTPResponse( + (1, 1), # FIXME: Should be a string. status_code, response, headers, body, ) - protocol = self.c.client_conn.protocol or http1.HTTP1Protocol(self.c.client_conn) - self.c.client_conn.send(protocol.assemble(resp)) +def make_connect_request(address): + return HTTPRequest( + "authority", "CONNECT", None, address.host, address.port, None, (1,1), + odict.ODictCaseless(), "" + ) + +def make_connect_response(httpversion): + headers = odict.ODictCaseless([ + ["Content-Length", "0"], + ["Proxy-Agent", version.NAMEVERSION] + ]) + return HTTPResponse( + httpversion, + 200, + "Connection established", + headers, + "", + ) + class HttpLayer(Layer, ServerConnectionMixin): """ @@ -45,11 +69,16 @@ class HttpLayer(Layer, ServerConnectionMixin): def __init__(self, ctx): super(HttpLayer, self).__init__(ctx) - self.skip_authentication = False + if any(isinstance(l, HttpProxy) for l in self.layers): + self.mode = "regular" + elif any(isinstance(l, HttpUpstreamProxy) for l in self.layers): + self.mode = "upstream" + else: + # also includes socks or reverse mode, which are handled similarly on this layer. + self.mode = "transparent" def __call__(self): while True: - flow = HTTPFlow(self.client_conn, self.server_conn) try: request = HTTP1.read_request( self.client_conn, @@ -62,29 +91,126 @@ class HttpLayer(Layer, ServerConnectionMixin): self.c.log("request", "debug", [repr(request)]) - self.check_authentication(request) + # Handle Proxy Authentication + self.authenticate(request) + # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.form_in == "authority": - raise NotImplementedError - + self.server_address = (request.host, request.port) + self.send_to_client(make_connect_response(request.httpversion)) + layer = self.ctx.next_layer(self) + for message in layer(): + if not self._handle_server_message(message): + yield message + return + # Make sure that the incoming request matches our expectations + self.validate_request(request) - ret = self.process_request(flow, request) - if ret is True: - continue - if ret is False: + flow = HTTPFlow(self.client_conn, self.server_conn) + flow.request = request + if not self.process_request_hook(flow): + self.log("Connection killed", "info") return - def check_authentication(self, request): + if not flow.response: + self.establish_server_connection(flow) + + def process_request_hook(self, flow): + # Determine .scheme, .host and .port attributes for inline scripts. + # For absolute-form requests, they are directly given in the request. + # For authority-form requests, we only need to determine the request scheme. + # For relative-form requests, we need to determine host and port as + # well. + if self.mode == "regular": + pass # only absolute-form at this point, nothing to do here. + elif self.mode == "upstream": + if flow.request.form_in == "authority": + flow.request.scheme = "http" # pseudo value + else: + flow.request.host = self.ctx.server_address.host + flow.request.port = self.ctx.server_address.port + flow.request.scheme = self.server_conn.tls_established + + # TODO: Expose ChangeServer functionality to inline scripts somehow? (yield_from_callback?) + request_reply = self.c.channel.ask("request", flow) + if request_reply is None or request_reply == KILL: + return False + if isinstance(request_reply, HTTPResponse): + flow.response = request_reply + return + + def establish_server_connection(self, flow): + + address = tcp.Address((flow.request.host, flow.request.port)) + tls = (flow.request.scheme == "https") + if self.mode == "regular" or self.mode == "transparent": + # If there's an existing connection that doesn't match our expectations, kill it. + if self.server_address != address or tls != self.server_address.ssl_established: + yield ChangeServer(address, tls, address.host) + # Establish connection is neccessary. + if not self.server_conn: + yield Connect() + + # ChangeServer is not guaranteed to work with TLS: + # If there's not TlsLayer below which could catch the exception, + # TLS will not be established. + if tls and not self.server_conn.tls_established: + raise ProtocolException("Cannot upgrade to SSL, no TLS layer on the protocol stack.") + + else: + if tls: + raise HttpException("Cannot change scheme in upstream proxy mode.") + """ + # This is a very ugly (untested) workaround to solve a very ugly problem. + # FIXME: Check if connected first. + if self.server_conn.tls_established and not ssl: + yield Reconnect() + elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: + if self.server_conn.tls_established: + yield Reconnect() + + self.send_to_server(make_connect_request(address)) + tls_layer = TlsLayer(self, False, True) + tls_layer._establish_tls_with_server() + """ + + def validate_request(self, request): + if request.form_in == "absolute" and request.scheme != "http": + self.send_resplonse(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + raise HttpException("Invalid request scheme: %s" % request.scheme) + + expected_request_forms = { + "regular": ("absolute",), # an authority request would already be handled. + "upstream": ("authority", "absolute"), + "transparent": ("regular",) + } + + allowed_request_forms = expected_request_forms[self.mode] + if request.form_in not in allowed_request_forms: + err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( + " or ".join(allowed_request_forms), request.form_in + ) + self.send_to_client(make_error_response(400, err_message)) + raise HttpException(err_message) + + def authenticate(self, request): if self.config.authenticator: if self.config.authenticator.authenticate(request.headers): self.config.authenticator.clean(request.headers) else: - self.send_error() + self.send_to_client(make_error_response( + 407, + "Proxy Authentication Required", + self.config.authenticator.auth_challenge_headers() + )) raise InvalidCredentials("Proxy Authentication Required") - raise http.HttpAuthenticationError( - self.c.config.authenticator.auth_challenge_headers()) - return request.headers - def send_error(self, code, message, headers): - pass \ No newline at end of file + def send_to_server(self, message): + self.server_conn.wfile.wrie(message) + + def send_to_client(self, message): + # FIXME + # - possibly do some http2 stuff here + # - fix message assembly. + self.client_conn.wfile.write(message) diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 6b3b6a82..51d3763c 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -4,11 +4,12 @@ from .layer import Layer, ServerConnectionMixin from .http import HttpLayer -class HttpProxy(Layer): +class HttpProxy(Layer, ServerConnectionMixin): def __call__(self): layer = HttpLayer(self) for message in layer(): - yield message + if not self._handle_server_message(message): + yield message class HttpUpstreamProxy(Layer, ServerConnectionMixin): diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 0ae64c43..e9f5c667 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -16,9 +16,7 @@ Regular proxy, CONNECT request with WebSockets over SSL: Automated protocol detection by peeking into the buffer: TransparentModeLayer - AutoLayer SslLayer - AutoLayer Http2Layer Communication between layers is done as follows: @@ -91,6 +89,13 @@ class Layer(_LayerCodeCompletion): full_msg = "\n".join(full_msg) self.channel.tell("log", Log(full_msg, level)) + @property + def layers(self): + return [self] + self.ctx.layers + + def __repr__(self): + return "%s\r\n %s" % (self.__class__.name__, repr(self.ctx)) + class ServerConnectionMixin(object): """ @@ -133,6 +138,8 @@ class ServerConnectionMixin(object): self.server_conn = None def _connect(self): + if not self.server_address: + raise ProtocolException("Cannot connect to server, no server address given.") self.log("serverconnect", "debug", [repr(self.server_address)]) self.server_conn = ServerConnection(self.server_address) try: diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index 608a53e3..167c8c79 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -1,4 +1,5 @@ from __future__ import (absolute_import, print_function, division) + import OpenSSL from ..exceptions import ProtocolException from ..protocol.tcp import TCPHandler @@ -6,7 +7,7 @@ from .layer import Layer from .messages import Connect -class TcpLayer(Layer): +class RawTcpLayer(Layer): def __call__(self): yield Connect() tcp_handler = TCPHandler(self) diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index cb6d1d78..bb414ec3 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -13,7 +13,10 @@ class ReverseProxy(Layer, ServerConnectionMixin): self._server_tls = server_tls def __call__(self): - layer = TlsLayer(self, self._client_tls, self._server_tls) + if self._client_tls or self._server_tls: + layer = TlsLayer(self, self._client_tls, self._server_tls) + else: + layer = self.ctx.next_layer(self) for message in layer(): if not self._handle_server_message(message): yield message diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index cbe596aa..3b341778 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -1,4 +1,6 @@ -from .rawtcp import TcpLayer +from __future__ import (absolute_import, print_function, division) + +from .rawtcp import RawTcpLayer from .tls import TlsLayer @@ -20,13 +22,30 @@ class RootContext(object): :return: The next layer. """ - d = top_layer.client_conn.rfile.peek(1) + d = top_layer.client_conn.rfile.peek(3) + + # TODO: Handle ignore and tcp passthrough + + # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + is_tls_client_hello = ( + len(d) == 3 and + d[0] == '\x16' and + d[1] == '\x03' and + d[2] in ('\x00', '\x01', '\x02', '\x03') + ) if not d: return - # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello - if d[0] == "\x16": + + if is_tls_client_hello: layer = TlsLayer(top_layer, True, True) else: - layer = TcpLayer(top_layer) + layer = RawTcpLayer(top_layer) return layer + + @property + def layers(self): + return [] + + def __repr__(self): + return "RootContext" diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 999cbea6..988304aa 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -1,4 +1,5 @@ from __future__ import (absolute_import, print_function, division) + import traceback from netlib import tcp @@ -99,7 +100,7 @@ class TlsLayer(Layer): if server_err and not self.client_sni: raise server_err - def handle_sni(self, connection): + def __handle_sni(self, connection): """ This callback gets called during the TLS handshake with the client. The client has just sent the Sever Name Indication (SNI). @@ -119,7 +120,7 @@ class TlsLayer(Layer): if self.client_sni: # Now, change client context to reflect possibly changed certificate: - cert, key, chain_file = self.find_cert() + cert, key, chain_file = self._find_cert() new_context = self.client_conn.create_ssl_context( cert, key, method=self.config.openssl_method_client, @@ -137,13 +138,13 @@ class TlsLayer(Layer): @yield_from_callback def _establish_tls_with_client(self): self.log("Establish TLS with client", "debug") - cert, key, chain_file = self.find_cert() + cert, key, chain_file = self._find_cert() try: self.client_conn.convert_to_ssl( cert, key, method=self.config.openssl_method_client, options=self.config.openssl_options_client, - handle_sni=self.handle_sni, + handle_sni=self.__handle_sni, cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, chain_file=chain_file @@ -182,7 +183,7 @@ class TlsLayer(Layer): except tcp.NetLibError as e: raise ProtocolException(repr(e), e) - def find_cert(self): + def _find_cert(self): host = self.server_conn.address.host sans = set() # Incorporate upstream certificate -- cgit v1.2.3 From 747699b126ab5788aca4541c9c9b4608611e7efa Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 14 Aug 2015 16:49:52 +0200 Subject: more work on http protocol --- libmproxy/protocol2/__init__.py | 3 +- libmproxy/protocol2/http.py | 144 +++++++++++++++++++++++++++--- libmproxy/protocol2/http_protocol_mock.py | 40 ++++++++- libmproxy/protocol2/layer.py | 5 +- libmproxy/protocol2/messages.py | 6 ++ libmproxy/proxy/server.py | 4 + 6 files changed, 187 insertions(+), 15 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index d5dafaae..cf6032da 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -4,7 +4,8 @@ from .socks_proxy import Socks5Proxy from .reverse_proxy import ReverseProxy from .http_proxy import HttpProxy, HttpUpstreamProxy from .rawtcp import RawTcpLayer +from . import messages __all__ = [ - "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" + "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "messages" ] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 44ebf6a8..1e774648 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -3,7 +3,8 @@ from __future__ import (absolute_import, print_function, division) from .. import version from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer, ServerConnectionMixin -from .messages import ChangeServer, Connect, Reconnect +from libmproxy import utils +from .messages import ChangeServer, Connect, Reconnect, Kill from .http_proxy import HttpProxy, HttpUpstreamProxy from libmproxy.protocol import KILL @@ -12,7 +13,8 @@ from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from libmproxy.protocol2.http_protocol_mock import HTTP1 from libmproxy.protocol2.tls import TlsLayer from netlib import tcp -from netlib.http import status_codes +from netlib.http import status_codes, http1 +from netlib.http.semantics import CONTENT_MISSING from netlib import odict @@ -42,12 +44,14 @@ def make_error_response(status_code, message, headers=None): body, ) + def make_connect_request(address): return HTTPRequest( - "authority", "CONNECT", None, address.host, address.port, None, (1,1), + "authority", "CONNECT", None, address.host, address.port, None, (1, 1), odict.ODictCaseless(), "" ) + def make_connect_response(httpversion): headers = odict.ODictCaseless([ ["Content-Length", "0"], @@ -82,14 +86,14 @@ class HttpLayer(Layer, ServerConnectionMixin): try: request = HTTP1.read_request( self.client_conn, - body_size_limit=self.c.config.body_size_limit + body_size_limit=self.config.body_size_limit ) except tcp.NetLibError: # don't throw an error for disconnects that happen # before/between requests. return - self.c.log("request", "debug", [repr(request)]) + self.log("request", "debug", [repr(request)]) # Handle Proxy Authentication self.authenticate(request) @@ -109,12 +113,128 @@ class HttpLayer(Layer, ServerConnectionMixin): flow = HTTPFlow(self.client_conn, self.server_conn) flow.request = request - if not self.process_request_hook(flow): - self.log("Connection killed", "info") - return + for message in self.process_request_hook(flow): + yield message if not flow.response: - self.establish_server_connection(flow) + for message in self.establish_server_connection(flow): + yield message + for message in self.get_response_from_server(flow): + yield message + + self.send_response_to_client(flow) + + if self.check_close_connection(flow): + return + + if flow.request.form_in == "authority" and flow.response.code == 200: + raise NotImplementedError("Upstream mode CONNECT not implemented") + + def check_close_connection(self, flow): + """ + Checks if the connection should be closed depending on the HTTP + semantics. Returns True, if so. + """ + + # TODO: add logic for HTTP/2 + + close_connection = ( + http1.HTTP1Protocol.connection_close( + flow.request.httpversion, + flow.request.headers + ) or http1.HTTP1Protocol.connection_close( + flow.response.httpversion, + flow.response.headers + ) or http1.HTTP1Protocol.expected_http_body_size( + flow.response.headers, + False, + flow.request.method, + flow.response.code) == -1 + ) + if flow.request.form_in == "authority" and flow.response.code == 200: + # Workaround for + # https://github.com/mitmproxy/mitmproxy/issues/313: Some + # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 + # and no Content-Length header + + return False + return close_connection + + def send_response_to_client(self, flow): + if not flow.response.stream: + # no streaming: + # we already received the full response from the server and can + # send it to the client straight away. + self.send_to_client(flow.response) + else: + # streaming: + # First send the headers and then transfer the response + # incrementally: + h = HTTP1._assemble_response_first_line(flow.response) + self.send_to_client(h + "\r\n") + h = HTTP1._assemble_response_headers(flow.response, preserve_transfer_encoding=True) + self.send_to_client(h + "\r\n") + + chunks = HTTP1.read_http_body_chunked( + flow.response.headers, + self.config.body_size_limit, + flow.request.method, + flow.response.code, + False, + 4096 + ) + + if callable(flow.response.stream): + chunks = flow.response.stream(chunks) + + for chunk in chunks: + for part in chunk: + self.send_to_client(part) + self.client_conn.wfile.flush() + + flow.response.timestamp_end = utils.timestamp() + + def get_response_from_server(self, flow): + + self.send_to_server(flow.request) + + flow.response = HTTP1.read_response( + self.server_conn.protocol, + flow.request.method, + body_size_limit=self.config.body_size_limit, + include_body=False, + ) + + # call the appropriate script hook - this is an opportunity for an + # inline script to set flow.stream = True + flow = self.channel.ask("responseheaders", flow) + if flow is None or flow == KILL: + yield Kill() + + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = HTTP1.read_http_body( + flow.response.headers, + self.config.body_size_limit, + flow.request.method, + flow.response.code, + False + ) + flow.response.timestamp_end = utils.timestamp() + + # no further manipulation of self.server_conn beyond this point + # we can safely set it as the final attribute value here. + flow.server_conn = self.server_conn + + self.log( + "response", + "debug", + [repr(flow.response)] + ) + response_reply = self.channel.ask("response", flow) + if response_reply is None or response_reply == KILL: + yield Kill() def process_request_hook(self, flow): # Determine .scheme, .host and .port attributes for inline scripts. @@ -133,9 +253,9 @@ class HttpLayer(Layer, ServerConnectionMixin): flow.request.scheme = self.server_conn.tls_established # TODO: Expose ChangeServer functionality to inline scripts somehow? (yield_from_callback?) - request_reply = self.c.channel.ask("request", flow) + request_reply = self.channel.ask("request", flow) if request_reply is None or request_reply == KILL: - return False + yield Kill() if isinstance(request_reply, HTTPResponse): flow.response = request_reply return @@ -181,7 +301,7 @@ class HttpLayer(Layer, ServerConnectionMixin): raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { - "regular": ("absolute",), # an authority request would already be handled. + "regular": ("absolute",), # an authority request would already be handled. "upstream": ("authority", "absolute"), "transparent": ("regular",) } diff --git a/libmproxy/protocol2/http_protocol_mock.py b/libmproxy/protocol2/http_protocol_mock.py index 962a76d6..5fdb9f2b 100644 --- a/libmproxy/protocol2/http_protocol_mock.py +++ b/libmproxy/protocol2/http_protocol_mock.py @@ -10,4 +10,42 @@ class HTTP1(object): """ :type connection: object """ - return HTTP1Protocol(connection).read_request(*args, **kwargs) \ No newline at end of file + return HTTP1Protocol(connection).read_request(*args, **kwargs) + + @staticmethod + def read_response(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection).read_response(*args, **kwargs) + + @staticmethod + def read_http_body(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection).read_http_body(*args, **kwargs) + + + @staticmethod + def _assemble_response_first_line(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection)._assemble_response_first_line(*args, **kwargs) + + + @staticmethod + def _assemble_response_headers(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection)._assemble_response_headers(*args, **kwargs) + + + @staticmethod + def read_http_body_chunked(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection).read_http_body_chunked(*args, **kwargs) \ No newline at end of file diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index e9f5c667..2775845e 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -35,7 +35,7 @@ import threading from netlib import tcp from ..proxy import Log from ..proxy.connection import ServerConnection -from .messages import Connect, Reconnect, ChangeServer +from .messages import Connect, Reconnect, ChangeServer, Kill from ..exceptions import ProtocolException @@ -116,6 +116,9 @@ class ServerConnectionMixin(object): return True elif message == ChangeServer: raise NotImplementedError + elif message == Kill: + self._disconnect() + return False @property diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index 3f53fbd4..f6b584a1 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -41,3 +41,9 @@ class ChangeServer(_Message): # We can express this neatly as the "nth-server-providing-layer" # ServerConnection could get a `via` attribute. self.depth = depth + + +class Kill(_Message): + """ + Kill a connection. + """ \ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 6a7048e0..defcd464 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -84,6 +84,10 @@ class ConnectionHandler2: try: for message in root_layer(): + if message == protocol2.messages.Kill: + self.log("Connection killed", "info") + break + print("Root layer receveived: %s" % message) except ProtocolException as e: self.log(e, "info") -- cgit v1.2.3 From 0dd243c5e42950de9c8b1193ba9dbdd2d0414a45 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 15 Aug 2015 16:26:12 +0200 Subject: various fixes --- libmproxy/protocol2/http.py | 22 +++++++++++++++------- libmproxy/protocol2/http_protocol_mock.py | 25 ++++++++++++------------- libmproxy/protocol2/layer.py | 11 ++++++++--- libmproxy/proxy/server.py | 2 +- 4 files changed, 36 insertions(+), 24 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 1e774648..7adeb419 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -5,7 +5,6 @@ from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer, ServerConnectionMixin from libmproxy import utils from .messages import ChangeServer, Connect, Reconnect, Kill -from .http_proxy import HttpProxy, HttpUpstreamProxy from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow @@ -66,13 +65,18 @@ def make_connect_response(httpversion): ) -class HttpLayer(Layer, ServerConnectionMixin): +class HttpLayer(Layer): + """ HTTP 1 Layer """ def __init__(self, ctx): super(HttpLayer, self).__init__(ctx) + + # FIXME: Imports + from .http_proxy import HttpProxy, HttpUpstreamProxy + if any(isinstance(l, HttpProxy) for l in self.layers): self.mode = "regular" elif any(isinstance(l, HttpUpstreamProxy) for l in self.layers): @@ -199,7 +203,7 @@ class HttpLayer(Layer, ServerConnectionMixin): self.send_to_server(flow.request) flow.response = HTTP1.read_response( - self.server_conn.protocol, + self.server_conn, flow.request.method, body_size_limit=self.config.body_size_limit, include_body=False, @@ -215,6 +219,7 @@ class HttpLayer(Layer, ServerConnectionMixin): flow.response.content = CONTENT_MISSING else: flow.response.content = HTTP1.read_http_body( + self.server_conn, flow.response.headers, self.config.body_size_limit, flow.request.method, @@ -303,7 +308,7 @@ class HttpLayer(Layer, ServerConnectionMixin): expected_request_forms = { "regular": ("absolute",), # an authority request would already be handled. "upstream": ("authority", "absolute"), - "transparent": ("regular",) + "transparent": ("relative",) } allowed_request_forms = expected_request_forms[self.mode] @@ -314,6 +319,9 @@ class HttpLayer(Layer, ServerConnectionMixin): self.send_to_client(make_error_response(400, err_message)) raise HttpException(err_message) + if self.mode == "regular": + request.form_out = "relative" + def authenticate(self, request): if self.config.authenticator: if self.config.authenticator.authenticate(request.headers): @@ -327,10 +335,10 @@ class HttpLayer(Layer, ServerConnectionMixin): raise InvalidCredentials("Proxy Authentication Required") def send_to_server(self, message): - self.server_conn.wfile.wrie(message) + self.server_conn.send(HTTP1.assemble(message)) + def send_to_client(self, message): # FIXME # - possibly do some http2 stuff here - # - fix message assembly. - self.client_conn.wfile.write(message) + self.client_conn.send(HTTP1.assemble(message)) diff --git a/libmproxy/protocol2/http_protocol_mock.py b/libmproxy/protocol2/http_protocol_mock.py index 5fdb9f2b..22f3dc14 100644 --- a/libmproxy/protocol2/http_protocol_mock.py +++ b/libmproxy/protocol2/http_protocol_mock.py @@ -1,6 +1,7 @@ """ Temporary mock to sort out API discrepancies """ +from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from netlib.http.http1 import HTTP1Protocol @@ -10,14 +11,14 @@ class HTTP1(object): """ :type connection: object """ - return HTTP1Protocol(connection).read_request(*args, **kwargs) + return HTTPRequest.wrap(HTTP1Protocol(connection).read_request(*args, **kwargs)) @staticmethod def read_response(connection, *args, **kwargs): """ :type connection: object """ - return HTTP1Protocol(connection).read_response(*args, **kwargs) + return HTTPResponse.wrap(HTTP1Protocol(connection).read_response(*args, **kwargs)) @staticmethod def read_http_body(connection, *args, **kwargs): @@ -28,19 +29,13 @@ class HTTP1(object): @staticmethod - def _assemble_response_first_line(connection, *args, **kwargs): - """ - :type connection: object - """ - return HTTP1Protocol(connection)._assemble_response_first_line(*args, **kwargs) + def _assemble_response_first_line(*args, **kwargs): + return HTTP1Protocol()._assemble_response_first_line(*args, **kwargs) @staticmethod - def _assemble_response_headers(connection, *args, **kwargs): - """ - :type connection: object - """ - return HTTP1Protocol(connection)._assemble_response_headers(*args, **kwargs) + def _assemble_response_headers(*args, **kwargs): + return HTTP1Protocol()._assemble_response_headers(*args, **kwargs) @staticmethod @@ -48,4 +43,8 @@ class HTTP1(object): """ :type connection: object """ - return HTTP1Protocol(connection).read_http_body_chunked(*args, **kwargs) \ No newline at end of file + return HTTP1Protocol(connection).read_http_body_chunked(*args, **kwargs) + + @staticmethod + def assemble(*args, **kwargs): + return HTTP1Protocol().assemble(*args, **kwargs) \ No newline at end of file diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 2775845e..f2d6b3fb 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -45,6 +45,7 @@ class _LayerCodeCompletion(object): """ def __init__(self): + super(_LayerCodeCompletion, self).__init__() if True: return self.config = None @@ -94,7 +95,7 @@ class Layer(_LayerCodeCompletion): return [self] + self.ctx.layers def __repr__(self): - return "%s\r\n %s" % (self.__class__.name__, repr(self.ctx)) + return type(self).__name__ class ServerConnectionMixin(object): @@ -103,6 +104,7 @@ class ServerConnectionMixin(object): """ def __init__(self): + super(ServerConnectionMixin, self).__init__() self._server_address = None self.server_conn = None @@ -114,8 +116,11 @@ class ServerConnectionMixin(object): elif message == Connect: self._connect() return True - elif message == ChangeServer: - raise NotImplementedError + elif message == ChangeServer and message.depth == 1: + if self.server_conn: + self._disconnect() + self.server_address = message.address + return True elif message == Kill: self._disconnect() diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index defcd464..ffca55ee 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -80,7 +80,7 @@ class ConnectionHandler2: self.config, self.channel ) - root_layer = protocol2.Socks5Proxy(root_context) + root_layer = protocol2.HttpProxy(root_context) try: for message in root_layer(): -- cgit v1.2.3 From a9dd82c986be54d82f6ce9c7b65473f2b052cbe8 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 15 Aug 2015 17:43:46 +0200 Subject: add ALPN to proxy connections --- libmproxy/protocol2/http_proxy.py | 3 ++- libmproxy/protocol2/tls.py | 8 +++++++- libmproxy/proxy/connection.py | 21 ++++++++++----------- 3 files changed, 19 insertions(+), 13 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 51d3763c..b85a65eb 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -1,7 +1,6 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin -from .http import HttpLayer class HttpProxy(Layer, ServerConnectionMixin): @@ -22,3 +21,5 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + +from .http import HttpLayer diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 988304aa..9572912f 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -1,7 +1,9 @@ from __future__ import (absolute_import, print_function, division) import traceback + from netlib import tcp +import netlib.http.http2 from ..exceptions import ProtocolException from .layer import Layer, yield_from_callback @@ -147,7 +149,8 @@ class TlsLayer(Layer): handle_sni=self.__handle_sni, cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, - chain_file=chain_file + chain_file=chain_file, + alpn_select=netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2, # TODO: check if server is capable of h2 first ) except tcp.NetLibError as e: raise ProtocolException(repr(e), e) @@ -164,6 +167,9 @@ class TlsLayer(Layer): ca_path=self.config.openssl_trusted_cadir_server, ca_pemfile=self.config.openssl_trusted_ca_server, cipher_list=self.config.ciphers_server, + alpn_protos=[ + netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1, + netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], # TODO: read this from client_conn first ) tls_cert_err = self.server_conn.ssl_verification_error if tls_cert_err is not None: diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index 49210e47..d2b956f3 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -1,6 +1,8 @@ from __future__ import absolute_import + import copy import os + from netlib import tcp, certutils from .. import stateobject, utils @@ -75,14 +77,14 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): return f def convert_to_ssl(self, *args, **kwargs): - # TODO: read ALPN from server and select same proto for client conn - # alpn_select = 'h2' - # def alpn_select_callback(conn_, options): - # if alpn_select in options: - # return bytes(alpn_select) - # else: # pragma no cover - # return options[0] - # tcp.BaseHandler.convert_to_ssl(self, alpn_select=alpn_select_callback, *args, **kwargs) + if 'alpn_select' in kwargs: + alpn_select = kwargs['alpn_select'] + def alpn_select_callback(conn_, options): + if alpn_select in options: + return bytes(alpn_select) + else: # pragma no cover + return options[0] + kwargs['alpn_select'] = alpn_select_callback tcp.BaseHandler.convert_to_ssl(self, *args, **kwargs) self.timestamp_ssl_setup = utils.timestamp() @@ -184,9 +186,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): if os.path.exists(path): clientcert = path - # TODO: read ALPN from client and use same list for server conn - # self.convert_to_ssl(cert=clientcert, sni=sni, alpn_protos=[netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], **kwargs) - self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) self.sni = sni self.timestamp_ssl_setup = utils.timestamp() -- cgit v1.2.3 From 2a15479cdbda07a4a99f56f6090e479decbeb17c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 15 Aug 2015 20:20:46 +0200 Subject: fix bugs, make https work --- libmproxy/protocol2/http.py | 24 +++++++----------------- libmproxy/protocol2/http_proxy.py | 4 ++-- libmproxy/protocol2/layer.py | 4 ++-- libmproxy/protocol2/messages.py | 2 +- libmproxy/protocol2/root_context.py | 10 ++++++---- libmproxy/protocol2/tls.py | 34 +++++++++++++++++++--------------- 6 files changed, 37 insertions(+), 41 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 7adeb419..f629a6b0 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -4,7 +4,7 @@ from .. import version from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer, ServerConnectionMixin from libmproxy import utils -from .messages import ChangeServer, Connect, Reconnect, Kill +from .messages import SetServer, Connect, Reconnect, Kill from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow @@ -71,19 +71,9 @@ class HttpLayer(Layer): HTTP 1 Layer """ - def __init__(self, ctx): + def __init__(self, ctx, mode): super(HttpLayer, self).__init__(ctx) - - # FIXME: Imports - from .http_proxy import HttpProxy, HttpUpstreamProxy - - if any(isinstance(l, HttpProxy) for l in self.layers): - self.mode = "regular" - elif any(isinstance(l, HttpUpstreamProxy) for l in self.layers): - self.mode = "upstream" - else: - # also includes socks or reverse mode, which are handled similarly on this layer. - self.mode = "transparent" + self.mode = mode def __call__(self): while True: @@ -104,7 +94,7 @@ class HttpLayer(Layer): # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.form_in == "authority": - self.server_address = (request.host, request.port) + yield SetServer((request.host, request.port), False, None) self.send_to_client(make_connect_response(request.httpversion)) layer = self.ctx.next_layer(self) for message in layer(): @@ -255,7 +245,7 @@ class HttpLayer(Layer): else: flow.request.host = self.ctx.server_address.host flow.request.port = self.ctx.server_address.port - flow.request.scheme = self.server_conn.tls_established + flow.request.scheme = "https" if self.server_conn.tls_established else "http" # TODO: Expose ChangeServer functionality to inline scripts somehow? (yield_from_callback?) request_reply = self.channel.ask("request", flow) @@ -271,8 +261,8 @@ class HttpLayer(Layer): tls = (flow.request.scheme == "https") if self.mode == "regular" or self.mode == "transparent": # If there's an existing connection that doesn't match our expectations, kill it. - if self.server_address != address or tls != self.server_address.ssl_established: - yield ChangeServer(address, tls, address.host) + if self.server_address != address or tls != self.server_conn.ssl_established: + yield SetServer(address, tls, address.host) # Establish connection is neccessary. if not self.server_conn: yield Connect() diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 51d3763c..8ac7ea8e 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -6,7 +6,7 @@ from .http import HttpLayer class HttpProxy(Layer, ServerConnectionMixin): def __call__(self): - layer = HttpLayer(self) + layer = HttpLayer(self, "regular") for message in layer(): if not self._handle_server_message(message): yield message @@ -18,7 +18,7 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): self.server_address = server_address def __call__(self): - layer = HttpLayer(self) + layer = HttpLayer(self, "upstream") for message in layer(): if not self._handle_server_message(message): yield message diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index f2d6b3fb..8e985d4d 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -35,7 +35,7 @@ import threading from netlib import tcp from ..proxy import Log from ..proxy.connection import ServerConnection -from .messages import Connect, Reconnect, ChangeServer, Kill +from .messages import Connect, Reconnect, SetServer, Kill from ..exceptions import ProtocolException @@ -116,7 +116,7 @@ class ServerConnectionMixin(object): elif message == Connect: self._connect() return True - elif message == ChangeServer and message.depth == 1: + elif message == SetServer and message.depth == 1: if self.server_conn: self._disconnect() self.server_address = message.address diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index f6b584a1..17e12f11 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -27,7 +27,7 @@ class Reconnect(_Message): """ -class ChangeServer(_Message): +class SetServer(_Message): """ Change the upstream server. """ diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 3b341778..bda8b12b 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -2,7 +2,7 @@ from __future__ import (absolute_import, print_function, division) from .rawtcp import RawTcpLayer from .tls import TlsLayer - +from .http import HttpLayer class RootContext(object): """ @@ -38,10 +38,12 @@ class RootContext(object): return if is_tls_client_hello: - layer = TlsLayer(top_layer, True, True) + return TlsLayer(top_layer, True, True) + elif isinstance(top_layer, TlsLayer) and isinstance(top_layer.ctx, HttpLayer): + return HttpLayer(top_layer, "transparent") else: - layer = RawTcpLayer(top_layer) - return layer + return RawTcpLayer(top_layer) + @property def layers(self): diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 988304aa..55cc9794 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -5,7 +5,7 @@ from netlib import tcp from ..exceptions import ProtocolException from .layer import Layer, yield_from_callback -from .messages import Connect, Reconnect, ChangeServer +from .messages import Connect, Reconnect, SetServer class TlsLayer(Layer): @@ -13,7 +13,6 @@ class TlsLayer(Layer): super(TlsLayer, self).__init__(ctx) self._client_tls = client_tls self._server_tls = server_tls - self._connected = False self.client_sni = None self._sni_from_server_change = None @@ -44,9 +43,6 @@ class TlsLayer(Layer): client_tls_requires_server_cert = ( self._client_tls and self._server_tls and not self.config.no_upstream_cert ) - lazy_server_tls = ( - self._server_tls and not client_tls_requires_server_cert - ) if client_tls_requires_server_cert: for m in self._establish_tls_with_client_and_server(): @@ -56,18 +52,27 @@ class TlsLayer(Layer): yield m layer = self.ctx.next_layer(self) + for message in layer(): - if message != Connect or not self._connected: + self.log("TlsLayer: %s" % message,"debug") + if not (message == Connect and self._connected): yield message - if message == Connect: - if lazy_server_tls: - self._establish_tls_with_server() - if message == ChangeServer and message.depth == 1: - self._server_tls = message.server_tls - self._sni_from_server_change = message.sni - if message == Reconnect or message == ChangeServer: - if self._server_tls: + + if message == Connect or message == Reconnect: + if self._server_tls and not self._server_tls_established: self._establish_tls_with_server() + if message == SetServer and message.depth == 1: + if message.server_tls is not None: + self._sni_from_server_change = message.sni + self._server_tls = message.server_tls + + @property + def _server_tls_established(self): + return self.server_conn and self.server_conn.tls_established + + @property + def _connected(self): + return bool(self.server_conn) @property def sni_for_upstream_connection(self): @@ -83,7 +88,6 @@ class TlsLayer(Layer): # First, try to connect to the server. yield Connect() - self._connected = True server_err = None try: self._establish_tls_with_server() -- cgit v1.2.3 From 1e40d34e942382bbb11234e0e9232794b3bf6acf Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 15 Aug 2015 17:43:46 +0200 Subject: add ALPN to proxy connections --- libmproxy/protocol2/http_proxy.py | 3 ++- libmproxy/protocol2/layer.py | 2 +- libmproxy/protocol2/tls.py | 33 +++++++++++++++++++++++++++++++-- libmproxy/proxy/connection.py | 14 ++------------ 4 files changed, 36 insertions(+), 16 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 8ac7ea8e..b4c506cb 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -1,7 +1,6 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin -from .http import HttpLayer class HttpProxy(Layer, ServerConnectionMixin): @@ -22,3 +21,5 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + +from .http import HttpLayer diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 8e985d4d..de519baa 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -208,4 +208,4 @@ def yield_from_callback(fun): self.yield_from_callback = None - return wrapper \ No newline at end of file + return wrapper diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 55cc9794..fcc12f18 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -1,7 +1,9 @@ from __future__ import (absolute_import, print_function, division) import traceback + from netlib import tcp +import netlib.http.http2 from ..exceptions import ProtocolException from .layer import Layer, yield_from_callback @@ -15,6 +17,9 @@ class TlsLayer(Layer): self._server_tls = server_tls self.client_sni = None self._sni_from_server_change = None + self.client_alpn_protos = None + + # foo alpn protos = [netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1, netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], # TODO: read this from client_conn first def __call__(self): """ @@ -131,7 +136,8 @@ class TlsLayer(Layer): options=self.config.openssl_options_client, cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, - chain_file=chain_file + chain_file=chain_file, + alpn_select_callback=self.__handle_alpn_select, ) connection.set_context(new_context) # An unhandled exception in this method will core dump PyOpenSSL, so @@ -139,10 +145,30 @@ class TlsLayer(Layer): except: # pragma: no cover self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") + def __handle_alpn_select(self, conn_, options): + # TODO: change to something meaningful? + alpn_preference = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 + alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 + ### + + if self.client_alpn_protos != options: + # Perform reconnect + if self._server_tls: + self.yield_from_callback(Reconnect()) + + self.client_alpn_protos = options + print("foo: %s" % options) + + if alpn_preference in options: + return bytes(alpn_preference) + else: # pragma no cover + return options[0] + @yield_from_callback def _establish_tls_with_client(self): self.log("Establish TLS with client", "debug") cert, key, chain_file = self._find_cert() + try: self.client_conn.convert_to_ssl( cert, key, @@ -151,9 +177,11 @@ class TlsLayer(Layer): handle_sni=self.__handle_sni, cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, - chain_file=chain_file + chain_file=chain_file, + alpn_select_callback=self.__handle_alpn_select, ) except tcp.NetLibError as e: + print("alpn: %s" % self.client_alpn_protos) raise ProtocolException(repr(e), e) def _establish_tls_with_server(self): @@ -168,6 +196,7 @@ class TlsLayer(Layer): ca_path=self.config.openssl_trusted_cadir_server, ca_pemfile=self.config.openssl_trusted_ca_server, cipher_list=self.config.ciphers_server, + alpn_protos=self.client_alpn_protos, ) tls_cert_err = self.server_conn.ssl_verification_error if tls_cert_err is not None: diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index 49210e47..f33e84cd 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -1,6 +1,8 @@ from __future__ import absolute_import + import copy import os + from netlib import tcp, certutils from .. import stateobject, utils @@ -75,15 +77,6 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): return f def convert_to_ssl(self, *args, **kwargs): - # TODO: read ALPN from server and select same proto for client conn - # alpn_select = 'h2' - # def alpn_select_callback(conn_, options): - # if alpn_select in options: - # return bytes(alpn_select) - # else: # pragma no cover - # return options[0] - # tcp.BaseHandler.convert_to_ssl(self, alpn_select=alpn_select_callback, *args, **kwargs) - tcp.BaseHandler.convert_to_ssl(self, *args, **kwargs) self.timestamp_ssl_setup = utils.timestamp() @@ -184,9 +177,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): if os.path.exists(path): clientcert = path - # TODO: read ALPN from client and use same list for server conn - # self.convert_to_ssl(cert=clientcert, sni=sni, alpn_protos=[netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], **kwargs) - self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) self.sni = sni self.timestamp_ssl_setup = utils.timestamp() -- cgit v1.2.3 From 4c31ffd90fcc273f798b9a5be96c811fbedb5e2e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 16 Aug 2015 12:43:15 +0200 Subject: minor fixes --- libmproxy/protocol2/http.py | 95 ++++++++++++++++++++------------------- libmproxy/protocol2/http_proxy.py | 5 +-- libmproxy/protocol2/layer.py | 1 - 3 files changed, 51 insertions(+), 50 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index f629a6b0..7cc27652 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -12,9 +12,10 @@ from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from libmproxy.protocol2.http_protocol_mock import HTTP1 from libmproxy.protocol2.tls import TlsLayer from netlib import tcp -from netlib.http import status_codes, http1 +from netlib.http import status_codes, http1, HttpErrorConnClosed from netlib.http.semantics import CONTENT_MISSING from netlib import odict +from netlib.tcp import NetLibError def make_error_response(status_code, message, headers=None): @@ -66,7 +67,6 @@ def make_connect_response(httpversion): class HttpLayer(Layer): - """ HTTP 1 Layer """ @@ -78,51 +78,55 @@ class HttpLayer(Layer): def __call__(self): while True: try: - request = HTTP1.read_request( - self.client_conn, - body_size_limit=self.config.body_size_limit - ) - except tcp.NetLibError: - # don't throw an error for disconnects that happen - # before/between requests. - return - - self.log("request", "debug", [repr(request)]) - - # Handle Proxy Authentication - self.authenticate(request) - - # Regular Proxy Mode: Handle CONNECT - if self.mode == "regular" and request.form_in == "authority": - yield SetServer((request.host, request.port), False, None) - self.send_to_client(make_connect_response(request.httpversion)) - layer = self.ctx.next_layer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message - return - - # Make sure that the incoming request matches our expectations - self.validate_request(request) - - flow = HTTPFlow(self.client_conn, self.server_conn) - flow.request = request - for message in self.process_request_hook(flow): - yield message - - if not flow.response: - for message in self.establish_server_connection(flow): - yield message - for message in self.get_response_from_server(flow): + try: + request = HTTP1.read_request( + self.client_conn, + body_size_limit=self.config.body_size_limit + ) + except tcp.NetLibError: + # don't throw an error for disconnects that happen + # before/between requests. + return + + self.log("request", "debug", [repr(request)]) + + # Handle Proxy Authentication + self.authenticate(request) + + # Regular Proxy Mode: Handle CONNECT + if self.mode == "regular" and request.form_in == "authority": + yield SetServer((request.host, request.port), False, None) + self.send_to_client(make_connect_response(request.httpversion)) + layer = self.ctx.next_layer(self) + for message in layer(): + if not self._handle_server_message(message): + yield message + return + + # Make sure that the incoming request matches our expectations + self.validate_request(request) + + flow = HTTPFlow(self.client_conn, self.server_conn) + flow.request = request + for message in self.process_request_hook(flow): yield message - self.send_response_to_client(flow) + if not flow.response: + for message in self.establish_server_connection(flow): + yield message + for message in self.get_response_from_server(flow): + yield message + + self.send_response_to_client(flow) - if self.check_close_connection(flow): - return + if self.check_close_connection(flow): + return - if flow.request.form_in == "authority" and flow.response.code == 200: - raise NotImplementedError("Upstream mode CONNECT not implemented") + if flow.request.form_in == "authority" and flow.response.code == 200: + raise NotImplementedError("Upstream mode CONNECT not implemented") + except (HttpErrorConnClosed, NetLibError) as e: + make_error_response(502, repr(e)) + raise ProtocolException(repr(e), e) def check_close_connection(self, flow): """ @@ -144,7 +148,7 @@ class HttpLayer(Layer): False, flow.request.method, flow.response.code) == -1 - ) + ) if flow.request.form_in == "authority" and flow.response.code == 200: # Workaround for # https://github.com/mitmproxy/mitmproxy/issues/313: Some @@ -189,7 +193,7 @@ class HttpLayer(Layer): flow.response.timestamp_end = utils.timestamp() def get_response_from_server(self, flow): - + # TODO: Add second attempt. self.send_to_server(flow.request) flow.response = HTTP1.read_response( @@ -327,7 +331,6 @@ class HttpLayer(Layer): def send_to_server(self, message): self.server_conn.send(HTTP1.assemble(message)) - def send_to_client(self, message): # FIXME # - possibly do some http2 stuff here diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index b4c506cb..ca70b012 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -1,6 +1,7 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin +from .http import HttpLayer class HttpProxy(Layer, ServerConnectionMixin): @@ -20,6 +21,4 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): layer = HttpLayer(self, "upstream") for message in layer(): if not self._handle_server_message(message): - yield message - -from .http import HttpLayer + yield message \ No newline at end of file diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 8e985d4d..ca297c0e 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -200,7 +200,6 @@ def yield_from_callback(fun): if msg is True: break elif isinstance(msg, Exception): - # TODO: Include func name? raise ProtocolException("Error in %s: %s" % (fun.__name__, repr(msg)), msg) else: yield msg -- cgit v1.2.3 From c04fa1b233224d28e85be34ab5b6a8718497488c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 16 Aug 2015 12:52:34 +0200 Subject: minor fixes --- libmproxy/protocol2/http_protocol_mock.py | 4 ++-- libmproxy/protocol2/http_proxy.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http_protocol_mock.py b/libmproxy/protocol2/http_protocol_mock.py index 22f3dc14..dd3643f6 100644 --- a/libmproxy/protocol2/http_protocol_mock.py +++ b/libmproxy/protocol2/http_protocol_mock.py @@ -11,14 +11,14 @@ class HTTP1(object): """ :type connection: object """ - return HTTPRequest.wrap(HTTP1Protocol(connection).read_request(*args, **kwargs)) + return HTTPRequest.from_protocol(HTTP1Protocol(connection), *args, **kwargs) @staticmethod def read_response(connection, *args, **kwargs): """ :type connection: object """ - return HTTPResponse.wrap(HTTP1Protocol(connection).read_response(*args, **kwargs)) + return HTTPResponse.from_protocol(HTTP1Protocol(connection), *args, **kwargs) @staticmethod def read_http_body(connection, *args, **kwargs): diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 9488e367..ca70b012 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -1,6 +1,7 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin +from .http import HttpLayer class HttpProxy(Layer, ServerConnectionMixin): -- cgit v1.2.3 From 38c456bb627c4570e0ed983229ec8ef2f120a4b6 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 16 Aug 2015 15:19:11 +0200 Subject: implement Http1 and Http2 protocols as layers --- libmproxy/protocol2/http.py | 51 +++++++++++++++++++++++-------- libmproxy/protocol2/http_protocol_mock.py | 50 ------------------------------ libmproxy/protocol2/http_proxy.py | 8 ++--- libmproxy/protocol2/layer.py | 1 + libmproxy/protocol2/root_context.py | 13 ++++++-- libmproxy/protocol2/tls.py | 1 - 6 files changed, 54 insertions(+), 70 deletions(-) delete mode 100644 libmproxy/protocol2/http_protocol_mock.py (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 7cc27652..cabec806 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -9,15 +9,42 @@ from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest -from libmproxy.protocol2.http_protocol_mock import HTTP1 from libmproxy.protocol2.tls import TlsLayer from netlib import tcp from netlib.http import status_codes, http1, HttpErrorConnClosed from netlib.http.semantics import CONTENT_MISSING from netlib import odict from netlib.tcp import NetLibError +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol + +class Http1Layer(Layer): + def __init__(self, ctx, mode): + super(Http1Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP1Protocol(self.client_conn) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def __call__(self): + from .http import HttpLayer + layer = HttpLayer(self, self.mode) + for message in layer(): + yield message +class Http2Layer(Layer): + def __init__(self, ctx, mode): + super(Http2Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + + def __call__(self): + from .http import HttpLayer + layer = HttpLayer(self, self.mode) + for message in layer(): + yield message + def make_error_response(status_code, message, headers=None): response = status_codes.RESPONSES.get(status_code, "Unknown") body = """ @@ -79,8 +106,8 @@ class HttpLayer(Layer): while True: try: try: - request = HTTP1.read_request( - self.client_conn, + request = HTTPRequest.from_protocol( + self.client_protocol, body_size_limit=self.config.body_size_limit ) except tcp.NetLibError: @@ -168,12 +195,12 @@ class HttpLayer(Layer): # streaming: # First send the headers and then transfer the response # incrementally: - h = HTTP1._assemble_response_first_line(flow.response) + h = self.client_protocol._assemble_response_first_line(flow.response) self.send_to_client(h + "\r\n") - h = HTTP1._assemble_response_headers(flow.response, preserve_transfer_encoding=True) + h = self.client_protocol._assemble_response_headers(flow.response, preserve_transfer_encoding=True) self.send_to_client(h + "\r\n") - chunks = HTTP1.read_http_body_chunked( + chunks = self.client_protocol.read_http_body_chunked( flow.response.headers, self.config.body_size_limit, flow.request.method, @@ -196,8 +223,8 @@ class HttpLayer(Layer): # TODO: Add second attempt. self.send_to_server(flow.request) - flow.response = HTTP1.read_response( - self.server_conn, + flow.response = HTTPResponse.from_protocol( + self.server_protocol, flow.request.method, body_size_limit=self.config.body_size_limit, include_body=False, @@ -211,8 +238,8 @@ class HttpLayer(Layer): if flow.response.stream: flow.response.content = CONTENT_MISSING - else: - flow.response.content = HTTP1.read_http_body( + elif isinstance(self.server_protocol, http1.HTTP1Protocol): + flow.response.content = self.server_protocol.read_http_body( self.server_conn, flow.response.headers, self.config.body_size_limit, @@ -329,9 +356,9 @@ class HttpLayer(Layer): raise InvalidCredentials("Proxy Authentication Required") def send_to_server(self, message): - self.server_conn.send(HTTP1.assemble(message)) + self.server_conn.send(self.server_protocol.assemble(message)) def send_to_client(self, message): # FIXME # - possibly do some http2 stuff here - self.client_conn.send(HTTP1.assemble(message)) + self.client_conn.send(self.client_protocol.assemble(message)) diff --git a/libmproxy/protocol2/http_protocol_mock.py b/libmproxy/protocol2/http_protocol_mock.py deleted file mode 100644 index dd3643f6..00000000 --- a/libmproxy/protocol2/http_protocol_mock.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -Temporary mock to sort out API discrepancies -""" -from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest -from netlib.http.http1 import HTTP1Protocol - - -class HTTP1(object): - @staticmethod - def read_request(connection, *args, **kwargs): - """ - :type connection: object - """ - return HTTPRequest.from_protocol(HTTP1Protocol(connection), *args, **kwargs) - - @staticmethod - def read_response(connection, *args, **kwargs): - """ - :type connection: object - """ - return HTTPResponse.from_protocol(HTTP1Protocol(connection), *args, **kwargs) - - @staticmethod - def read_http_body(connection, *args, **kwargs): - """ - :type connection: object - """ - return HTTP1Protocol(connection).read_http_body(*args, **kwargs) - - - @staticmethod - def _assemble_response_first_line(*args, **kwargs): - return HTTP1Protocol()._assemble_response_first_line(*args, **kwargs) - - - @staticmethod - def _assemble_response_headers(*args, **kwargs): - return HTTP1Protocol()._assemble_response_headers(*args, **kwargs) - - - @staticmethod - def read_http_body_chunked(connection, *args, **kwargs): - """ - :type connection: object - """ - return HTTP1Protocol(connection).read_http_body_chunked(*args, **kwargs) - - @staticmethod - def assemble(*args, **kwargs): - return HTTP1Protocol().assemble(*args, **kwargs) \ No newline at end of file diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index ca70b012..7f5957ac 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -1,12 +1,12 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin -from .http import HttpLayer +from .http import Http1Layer, HttpLayer class HttpProxy(Layer, ServerConnectionMixin): def __call__(self): - layer = HttpLayer(self, "regular") + layer = Http1Layer(self, "regular") for message in layer(): if not self._handle_server_message(message): yield message @@ -18,7 +18,7 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): self.server_address = server_address def __call__(self): - layer = HttpLayer(self, "upstream") + layer = Http1Layer(self, "upstream") for message in layer(): if not self._handle_server_message(message): - yield message \ No newline at end of file + yield message diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index c1648a62..31b74552 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -64,6 +64,7 @@ class Layer(_LayerCodeCompletion): """ super(Layer, self).__init__() self.ctx = ctx + print("%s -> %s" % (repr(ctx), repr(self))) def __call__(self): """ diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index bda8b12b..a68560c2 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -2,7 +2,7 @@ from __future__ import (absolute_import, print_function, division) from .rawtcp import RawTcpLayer from .tls import TlsLayer -from .http import HttpLayer +from .http import Http1Layer, Http2Layer, HttpLayer class RootContext(object): """ @@ -34,13 +34,20 @@ class RootContext(object): d[2] in ('\x00', '\x01', '\x02', '\x03') ) + # TODO: build is_http2_magic check here, maybe this is an easy way to detect h2c + if not d: return if is_tls_client_hello: return TlsLayer(top_layer, True, True) - elif isinstance(top_layer, TlsLayer) and isinstance(top_layer.ctx, HttpLayer): - return HttpLayer(top_layer, "transparent") + elif isinstance(top_layer, TlsLayer): + if top_layer.client_conn.get_alpn_proto_negotiated() == 'h2': + return Http2Layer(top_layer, 'regular') # TODO: regular correct here? + else: + return Http1Layer(top_layer, 'regular') # TODO: regular correct here? + elif isinstance(top_layer, TlsLayer) and isinstance(top_layer.ctx, Http1Layer): + return Http1Layer(top_layer, "transparent") else: return RawTcpLayer(top_layer) diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index fcc12f18..8e367728 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -157,7 +157,6 @@ class TlsLayer(Layer): self.yield_from_callback(Reconnect()) self.client_alpn_protos = options - print("foo: %s" % options) if alpn_preference in options: return bytes(alpn_preference) -- cgit v1.2.3 From a2b85048892626e6834df06e9022498814724636 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 16 Aug 2015 23:25:02 +0200 Subject: improve protocol handling --- libmproxy/protocol2/http.py | 86 +++++++++++++++++++++++++------- libmproxy/protocol2/http_proxy.py | 3 +- libmproxy/protocol2/layer.py | 47 ++++++++--------- libmproxy/protocol2/messages.py | 3 +- libmproxy/protocol2/reverse_proxy.py | 3 +- libmproxy/protocol2/root_context.py | 5 +- libmproxy/protocol2/socks_proxy.py | 4 +- libmproxy/protocol2/tls.py | 3 +- libmproxy/protocol2/transparent_proxy.py | 2 +- libmproxy/proxy/connection.py | 7 ++- libmproxy/proxy/server.py | 11 ++-- 11 files changed, 114 insertions(+), 60 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 7cc27652..90784666 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -2,20 +2,20 @@ from __future__ import (absolute_import, print_function, division) from .. import version from ..exceptions import InvalidCredentials, HttpException, ProtocolException -from .layer import Layer, ServerConnectionMixin +from .layer import Layer from libmproxy import utils +from libmproxy.proxy.connection import ServerConnection from .messages import SetServer, Connect, Reconnect, Kill from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from libmproxy.protocol2.http_protocol_mock import HTTP1 -from libmproxy.protocol2.tls import TlsLayer from netlib import tcp from netlib.http import status_codes, http1, HttpErrorConnClosed from netlib.http.semantics import CONTENT_MISSING from netlib import odict -from netlib.tcp import NetLibError +from netlib.tcp import NetLibError, Address def make_error_response(status_code, message, headers=None): @@ -46,6 +46,7 @@ def make_error_response(status_code, message, headers=None): def make_connect_request(address): + address = Address.wrap(address) return HTTPRequest( "authority", "CONNECT", None, address.host, address.port, None, (1, 1), odict.ODictCaseless(), "" @@ -66,6 +67,22 @@ def make_connect_response(httpversion): ) +class ConnectServerConnection(object): + """ + "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. + """ + def __init__(self, address, ctx): + self.address = tcp.Address.wrap(address) + self._ctx = ctx + + @property + def via(self): + return self._ctx.server_conn + + def __getattr__(self, item): + return getattr(self.via, item) + + class HttpLayer(Layer): """ HTTP 1 Layer @@ -95,12 +112,8 @@ class HttpLayer(Layer): # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.form_in == "authority": - yield SetServer((request.host, request.port), False, None) - self.send_to_client(make_connect_response(request.httpversion)) - layer = self.ctx.next_layer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message + for message in self.handle_regular_mode_connect(request): + yield message return # Make sure that the incoming request matches our expectations @@ -122,12 +135,50 @@ class HttpLayer(Layer): if self.check_close_connection(flow): return + # Upstream Proxy Mode: Handle CONNECT if flow.request.form_in == "authority" and flow.response.code == 200: - raise NotImplementedError("Upstream mode CONNECT not implemented") + for message in self.handle_upstream_mode_connect(flow.request.copy()): + yield message + return + except (HttpErrorConnClosed, NetLibError) as e: make_error_response(502, repr(e)) raise ProtocolException(repr(e), e) + def handle_regular_mode_connect(self, request): + yield SetServer((request.host, request.port), False, None) + self.send_to_client(make_connect_response(request.httpversion)) + layer = self.ctx.next_layer(self) + for message in layer(): + yield message + + def handle_upstream_mode_connect(self, connect_request): + layer = self.ctx.next_layer(self) + self.server_conn = ConnectServerConnection((connect_request.host, connect_request.port), self.ctx) + + for message in layer(): + if message == Connect: + if not self.server_conn: + yield message + self.send_to_server(connect_request) + else: + pass # swallow the message + elif message == Reconnect: + yield message + self.send_to_server(connect_request) + elif message == SetServer: + if message.depth == 1: + if self.ctx.server_conn: + yield Reconnect() + connect_request.host = message.address.host + connect_request.port = message.address.port + self.server_conn.address = message.address + else: + message.depth -= 1 + yield message + else: + yield message + def check_close_connection(self, flow): """ Checks if the connection should be closed depending on the HTTP @@ -247,11 +298,11 @@ class HttpLayer(Layer): if flow.request.form_in == "authority": flow.request.scheme = "http" # pseudo value else: - flow.request.host = self.ctx.server_address.host - flow.request.port = self.ctx.server_address.port + flow.request.host = self.ctx.server_conn.address.host + flow.request.port = self.ctx.server_conn.address.port flow.request.scheme = "https" if self.server_conn.tls_established else "http" - # TODO: Expose ChangeServer functionality to inline scripts somehow? (yield_from_callback?) + # TODO: Expose SetServer functionality to inline scripts somehow? (yield_from_callback?) request_reply = self.channel.ask("request", flow) if request_reply is None or request_reply == KILL: yield Kill() @@ -265,25 +316,26 @@ class HttpLayer(Layer): tls = (flow.request.scheme == "https") if self.mode == "regular" or self.mode == "transparent": # If there's an existing connection that doesn't match our expectations, kill it. - if self.server_address != address or tls != self.server_conn.ssl_established: + if address != self.server_conn.address or tls != self.server_conn.ssl_established: yield SetServer(address, tls, address.host) # Establish connection is neccessary. if not self.server_conn: yield Connect() - # ChangeServer is not guaranteed to work with TLS: + # SetServer is not guaranteed to work with TLS: # If there's not TlsLayer below which could catch the exception, # TLS will not be established. if tls and not self.server_conn.tls_established: raise ProtocolException("Cannot upgrade to SSL, no TLS layer on the protocol stack.") else: + if not self.server_conn: + yield Connect() if tls: raise HttpException("Cannot change scheme in upstream proxy mode.") """ # This is a very ugly (untested) workaround to solve a very ugly problem. - # FIXME: Check if connected first. - if self.server_conn.tls_established and not ssl: + if self.server_conn and self.server_conn.tls_established and not ssl: yield Reconnect() elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: if self.server_conn.tls_established: diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index ca70b012..3cc7fee2 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -14,8 +14,7 @@ class HttpProxy(Layer, ServerConnectionMixin): class HttpUpstreamProxy(Layer, ServerConnectionMixin): def __init__(self, ctx, server_address): - super(HttpUpstreamProxy, self).__init__(ctx) - self.server_address = server_address + super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) def __call__(self): layer = HttpLayer(self, "upstream") diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index c1648a62..67f3d549 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -44,8 +44,8 @@ class _LayerCodeCompletion(object): Dummy class that provides type hinting in PyCharm, which simplifies development a lot. """ - def __init__(self): - super(_LayerCodeCompletion, self).__init__() + def __init__(self, *args, **kwargs): + super(_LayerCodeCompletion, self).__init__(*args, **kwargs) if True: return self.config = None @@ -57,12 +57,12 @@ class _LayerCodeCompletion(object): class Layer(_LayerCodeCompletion): - def __init__(self, ctx): + def __init__(self, ctx, *args, **kwargs): """ Args: ctx: The (read-only) higher layer. """ - super(Layer, self).__init__() + super(Layer, self).__init__(*args, **kwargs) self.ctx = ctx def __call__(self): @@ -103,10 +103,9 @@ class ServerConnectionMixin(object): Mixin that provides a layer with the capabilities to manage a server connection. """ - def __init__(self): + def __init__(self, server_address=None): super(ServerConnectionMixin, self).__init__() - self._server_address = None - self.server_conn = None + self.server_conn = ServerConnection(server_address) def _handle_server_message(self, message): if message == Reconnect: @@ -116,44 +115,38 @@ class ServerConnectionMixin(object): elif message == Connect: self._connect() return True - elif message == SetServer and message.depth == 1: - if self.server_conn: - self._disconnect() - self.server_address = message.address - return True + elif message == SetServer: + if message.depth == 1: + if self.server_conn: + self._disconnect() + self.log("Set new server address: " + repr(message.address), "debug") + self.server_conn.address = message.address + return True + else: + message.depth -= 1 elif message == Kill: self._disconnect() return False - @property - def server_address(self): - return self._server_address - - @server_address.setter - def server_address(self, address): - self._server_address = tcp.Address.wrap(address) - self.log("Set new server address: " + repr(self.server_address), "debug") - def _disconnect(self): """ Deletes (and closes) an existing server connection. """ - self.log("serverdisconnect", "debug", [repr(self.server_address)]) + self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) self.server_conn.finish() self.server_conn.close() # self.channel.tell("serverdisconnect", self) - self.server_conn = None + self.server_conn = ServerConnection(None) def _connect(self): - if not self.server_address: + if not self.server_conn.address: raise ProtocolException("Cannot connect to server, no server address given.") - self.log("serverconnect", "debug", [repr(self.server_address)]) - self.server_conn = ServerConnection(self.server_address) + self.log("serverconnect", "debug", [repr(self.server_conn.address)]) try: self.server_conn.connect() except tcp.NetLibError as e: - raise ProtocolException("Server connection to '%s' failed: %s" % (self.server_address, e), e) + raise ProtocolException("Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) def yield_from_callback(fun): diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index 17e12f11..f5907537 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -2,6 +2,7 @@ This module contains all valid messages layers can send to the underlying layers. """ from __future__ import (absolute_import, print_function, division) +from netlib.tcp import Address class _Message(object): @@ -33,7 +34,7 @@ class SetServer(_Message): """ def __init__(self, address, server_tls, sni, depth=1): - self.address = address + self.address = Address.wrap(address) self.server_tls = server_tls self.sni = sni diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index bb414ec3..2ee3d9d8 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -7,8 +7,7 @@ from .tls import TlsLayer class ReverseProxy(Layer, ServerConnectionMixin): def __init__(self, ctx, server_address, client_tls, server_tls): - super(ReverseProxy, self).__init__(ctx) - self.server_address = server_address + super(ReverseProxy, self).__init__(ctx, server_address=server_address) self._client_tls = client_tls self._server_tls = server_tls diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index bda8b12b..f369fd48 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -1,9 +1,11 @@ from __future__ import (absolute_import, print_function, division) +from .messages import Kill from .rawtcp import RawTcpLayer from .tls import TlsLayer from .http import HttpLayer + class RootContext(object): """ The outmost context provided to the root layer. @@ -35,7 +37,7 @@ class RootContext(object): ) if not d: - return + return iter([]) if is_tls_client_hello: return TlsLayer(top_layer, True, True) @@ -44,7 +46,6 @@ class RootContext(object): else: return RawTcpLayer(top_layer) - @property def layers(self): return [] diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py index c89477ca..c6126a42 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/protocol2/socks_proxy.py @@ -5,7 +5,7 @@ from ..proxy import ProxyError, Socks5ProxyMode from .layer import Layer, ServerConnectionMixin -class Socks5Proxy(Layer, ServerConnectionMixin): +class Socks5Proxy(ServerConnectionMixin, Layer): def __call__(self): try: s5mode = Socks5ProxyMode(self.config.ssl_ports) @@ -14,7 +14,7 @@ class Socks5Proxy(Layer, ServerConnectionMixin): # TODO: Unmonkeypatch raise ProtocolException(str(e), e) - self.server_address = address + self.server_conn.address = address layer = self.ctx.next_layer(self) for message in layer(): diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index fcc12f18..12c67f4e 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -220,7 +220,7 @@ class TlsLayer(Layer): host = self.server_conn.address.host sans = set() # Incorporate upstream certificate - if self.server_conn.tls_established and (not self.config.no_upstream_cert): + if self.server_conn and self.server_conn.tls_established and (not self.config.no_upstream_cert): upstream_cert = self.server_conn.cert sans.update(upstream_cert.altnames) if upstream_cert.cn: @@ -232,4 +232,5 @@ class TlsLayer(Layer): if self._sni_from_server_change: sans.add(self._sni_from_server_change) + sans.discard(host) return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py index f073e2f8..4ed4c14b 100644 --- a/libmproxy/protocol2/transparent_proxy.py +++ b/libmproxy/protocol2/transparent_proxy.py @@ -13,7 +13,7 @@ class TransparentProxy(Layer, ServerConnectionMixin): def __call__(self): try: - self.server_address = self.resolver.original_addr(self.client_conn.connection) + self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) except Exception as e: raise ProtocolException("Transparent mode failure: %s" % repr(e), e) diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index f33e84cd..f92b53aa 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -96,6 +96,9 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.timestamp_ssl_setup = None self.protocol = None + def __nonzero__(self): + return bool(self.connection) + def __repr__(self): if self.ssl_established and self.sni: ssl = "[ssl: {0}] ".format(self.sni) @@ -132,8 +135,8 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): d.update( address={"address": self.address(), "use_ipv6": self.address.use_ipv6}, - source_address= ({"address": self.source_address(), - "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), + source_address=({"address": self.source_address(), + "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), cert=self.cert.to_pem() if self.cert else None ) return d diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index ffca55ee..e23a7d72 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -80,7 +80,12 @@ class ConnectionHandler2: self.config, self.channel ) - root_layer = protocol2.HttpProxy(root_context) + + # FIXME: properly parse config + if self.config.mode == "upstream": + root_layer = protocol2.HttpUpstreamProxy(root_context, ("localhost", 8081)) + else: + root_layer = protocol2.HttpProxy(root_context) try: for message in root_layer(): @@ -302,7 +307,7 @@ class ConnectionHandler: if ssl_cert_err is not None: self.log( "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), + (ssl_cert_err['depth'], ssl_cert_err['errno']), "error") self.log("Ignoring server verification error, continuing with connection", "error") except tcp.NetLibError as v: @@ -318,7 +323,7 @@ class ConnectionHandler: if ssl_cert_err is not None: self.log( "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), + (ssl_cert_err['depth'], ssl_cert_err['errno']), "error") self.log("Aborting connection attempt", "error") raise e -- cgit v1.2.3 From 96de7ad562da9b5110059988b851c66b51874510 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 18 Aug 2015 14:15:08 +0200 Subject: various fixes --- libmproxy/filt.py | 4 +- libmproxy/protocol/http.py | 1 + libmproxy/protocol2/http.py | 64 ++++++++++++++++++++++++-------- libmproxy/protocol2/http_proxy.py | 5 ++- libmproxy/protocol2/layer.py | 2 + libmproxy/protocol2/reverse_proxy.py | 2 + libmproxy/protocol2/root_context.py | 10 +++-- libmproxy/protocol2/socks_proxy.py | 2 + libmproxy/protocol2/transparent_proxy.py | 2 + libmproxy/proxy/connection.py | 10 +++-- 10 files changed, 77 insertions(+), 25 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/filt.py b/libmproxy/filt.py index bd17a807..25747bc6 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -246,14 +246,14 @@ class FSrc(_Rex): help = "Match source address" def __call__(self, f): - return f.client_conn and re.search(self.expr, repr(f.client_conn.address)) + return f.client_conn.address and re.search(self.expr, repr(f.client_conn.address)) class FDst(_Rex): code = "dst" help = "Match destination address" def __call__(self, f): - return f.server_conn and re.search(self.expr, repr(f.server_conn.address)) + return f.server_conn.address and re.search(self.expr, repr(f.server_conn.address)) class _Int(_Action): def __init__(self, num): diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4c15c80d..4472cb2a 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -729,6 +729,7 @@ class RequestReplayThread(threading.Thread): if not self.flow.response: # In all modes, we directly connect to the server displayed if self.config.mode == "upstream": + # FIXME server_address = self.config.mode.get_upstream_server( self.flow.client_conn )[2:] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index eadde3b3..53f40a72 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -10,13 +10,14 @@ from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from netlib import tcp -from netlib.http import status_codes, http1, HttpErrorConnClosed +from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError from netlib.http.semantics import CONTENT_MISSING from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol + # TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. @@ -31,6 +32,7 @@ class Http1Layer(Layer): layer = HttpLayer(self, self.mode) for message in layer(): yield message + self.server_protocol = HTTP1Protocol(self.server_conn) class Http2Layer(Layer): @@ -41,10 +43,10 @@ class Http2Layer(Layer): self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) def __call__(self): - # FIXME: Handle Reconnect etc. layer = HttpLayer(self, self.mode) for message in layer(): yield message + self.server_protocol = HTTP1Protocol(self.server_conn) def make_error_response(status_code, message, headers=None): @@ -100,6 +102,7 @@ class ConnectServerConnection(object): """ "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. """ + def __init__(self, address, ctx): self.address = tcp.Address.wrap(address) self._ctx = ctx @@ -124,6 +127,8 @@ class HttpLayer(Layer): def __call__(self): while True: try: + flow = HTTPFlow(self.client_conn, self.server_conn, live=True) + try: request = HTTPRequest.from_protocol( self.client_protocol, @@ -148,7 +153,6 @@ class HttpLayer(Layer): # Make sure that the incoming request matches our expectations self.validate_request(request) - flow = HTTPFlow(self.client_conn, self.server_conn) flow.request = request for message in self.process_request_hook(flow): yield message @@ -164,15 +168,22 @@ class HttpLayer(Layer): if self.check_close_connection(flow): return + # TODO: Implement HTTP Upgrade + # Upstream Proxy Mode: Handle CONNECT if flow.request.form_in == "authority" and flow.response.code == 200: for message in self.handle_upstream_mode_connect(flow.request.copy()): yield message return - except (HttpErrorConnClosed, NetLibError) as e: - make_error_response(502, repr(e)) + except (HttpErrorConnClosed, NetLibError, HttpError) as e: + self.send_to_client(make_error_response( + getattr(e, "code", 502), + repr(e) + )) raise ProtocolException(repr(e), e) + finally: + flow.live = False def handle_regular_mode_connect(self, request): yield SetServer((request.host, request.port), False, None) @@ -267,21 +278,43 @@ class HttpLayer(Layer): for chunk in chunks: for part in chunk: + # TODO: That's going to fail. self.send_to_client(part) self.client_conn.wfile.flush() flow.response.timestamp_end = utils.timestamp() def get_response_from_server(self, flow): - # TODO: Add second attempt. - self.send_to_server(flow.request) - - flow.response = HTTPResponse.from_protocol( - self.server_protocol, - flow.request.method, - body_size_limit=self.config.body_size_limit, - include_body=False, - ) + def get_response(): + self.send_to_server(flow.request) + # Only get the headers at first... + flow.response = HTTPResponse.from_protocol( + self.server_protocol, + flow.request.method, + body_size_limit=self.config.body_size_limit, + include_body=False, + ) + + try: + get_response() + except (tcp.NetLibError, HttpErrorConnClosed) as v: + self.log( + "server communication error: %s" % repr(v), + level="debug" + ) + # In any case, we try to reconnect at least once. This is + # necessary because it might be possible that we already + # initiated an upstream connection after clientconnect that + # has already been expired, e.g consider the following event + # log: + # > clientconnect (transparent mode destination known) + # > serverconnect (required for client tls handshake) + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + yield Reconnect() + get_response() # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True @@ -293,7 +326,6 @@ class HttpLayer(Layer): flow.response.content = CONTENT_MISSING else: flow.response.content = self.server_protocol.read_http_body( - self.server_conn, flow.response.headers, self.config.body_size_limit, flow.request.method, @@ -405,7 +437,7 @@ class HttpLayer(Layer): self.send_to_client(make_error_response( 407, "Proxy Authentication Required", - self.config.authenticator.auth_challenge_headers() + odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()]) )) raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 19b5f7ef..652aa473 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -10,7 +10,8 @@ class HttpProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message - + if self.server_conn: + self._disconnect() class HttpUpstreamProxy(Layer, ServerConnectionMixin): def __init__(self, ctx, server_address): @@ -21,3 +22,5 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + if self.server_conn: + self._disconnect() diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 67f3d549..eb41bab7 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -109,7 +109,9 @@ class ServerConnectionMixin(object): def _handle_server_message(self, message): if message == Reconnect: + address = self.server_conn.address self._disconnect() + self.server_conn.address = address self._connect() return True elif message == Connect: diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index 2ee3d9d8..767107ad 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -19,3 +19,5 @@ class ReverseProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 6ba6ca9a..f8a645b0 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -1,4 +1,5 @@ from __future__ import (absolute_import, print_function, division) +import string from .messages import Kill from .rawtcp import RawTcpLayer @@ -36,6 +37,8 @@ class RootContext(object): d[2] in ('\x00', '\x01', '\x02', '\x03') ) + is_ascii = all(x in string.ascii_uppercase for x in d) + # TODO: build is_http2_magic check here, maybe this is an easy way to detect h2c if not d: @@ -43,10 +46,11 @@ class RootContext(object): if is_tls_client_hello: return TlsLayer(top_layer, True, True) - elif isinstance(top_layer, TlsLayer) and top_layer.client_conn.get_alpn_proto_negotiated() == 'h2': + elif isinstance(top_layer, TlsLayer) and is_ascii: + if top_layer.client_conn.get_alpn_proto_negotiated() == 'h2': return Http2Layer(top_layer, 'transparent') - elif isinstance(top_layer, TlsLayer) and isinstance(top_layer.ctx, Http1Layer): - return Http1Layer(top_layer, "transparent") + else: + return Http1Layer(top_layer, "transparent") else: return RawTcpLayer(top_layer) diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py index c6126a42..5bb8e5f8 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/protocol2/socks_proxy.py @@ -20,3 +20,5 @@ class Socks5Proxy(ServerConnectionMixin, Layer): for message in layer(): if not self._handle_server_message(message): yield message + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py index 4ed4c14b..28ad3726 100644 --- a/libmproxy/protocol2/transparent_proxy.py +++ b/libmproxy/protocol2/transparent_proxy.py @@ -21,3 +21,5 @@ class TransparentProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index f92b53aa..c9b57998 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -27,6 +27,9 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): self.timestamp_ssl_setup = None self.protocol = None + def __nonzero__(self): + return bool(self.connection) and not self.finished + def __repr__(self): return "".format( ssl="[ssl] " if self.ssl_established else "", @@ -89,7 +92,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def __init__(self, address): tcp.TCPClient.__init__(self, address) - self.state = [] # a list containing (conntype, state) tuples + self.via = None self.timestamp_start = None self.timestamp_end = None self.timestamp_tcp_setup = None @@ -97,7 +100,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.protocol = None def __nonzero__(self): - return bool(self.connection) + return bool(self.connection) and not self.finished def __repr__(self): if self.ssl_established and self.sni: @@ -117,7 +120,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): return self.ssl_established _stateobject_attributes = dict( - state=list, timestamp_start=float, timestamp_end=float, timestamp_tcp_setup=float, @@ -187,3 +189,5 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def finish(self): tcp.TCPClient.finish(self) self.timestamp_end = utils.timestamp() + +ServerConnection._stateobject_attributes["via"] = ServerConnection \ No newline at end of file -- cgit v1.2.3 From ab1549e0eff98588211346aada44549311f04938 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 18 Aug 2015 15:59:44 +0200 Subject: yield -> callbacks --- libmproxy/protocol2/__init__.py | 3 +- libmproxy/protocol2/http.py | 133 ++++++++++++++++++------------- libmproxy/protocol2/http_proxy.py | 20 ++--- libmproxy/protocol2/layer.py | 96 +++++----------------- libmproxy/protocol2/messages.py | 4 - libmproxy/protocol2/rawtcp.py | 3 +- libmproxy/protocol2/reverse_proxy.py | 11 +-- libmproxy/protocol2/root_context.py | 2 +- libmproxy/protocol2/socks_proxy.py | 15 ++-- libmproxy/protocol2/tls.py | 54 ++++++------- libmproxy/protocol2/transparent_proxy.py | 10 +-- libmproxy/proxy/server.py | 10 +-- 12 files changed, 156 insertions(+), 205 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index cf6032da..d5dafaae 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -4,8 +4,7 @@ from .socks_proxy import Socks5Proxy from .reverse_proxy import ReverseProxy from .http_proxy import HttpProxy, HttpUpstreamProxy from .rawtcp import RawTcpLayer -from . import messages __all__ = [ - "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "messages" + "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" ] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 53f40a72..db5aabaf 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -4,7 +4,7 @@ from .. import version from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer from libmproxy import utils -from .messages import SetServer, Connect, Reconnect, Kill +from libmproxy.protocol2.layer import Kill from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow @@ -28,12 +28,21 @@ class Http1Layer(Layer): self.client_protocol = HTTP1Protocol(self.client_conn) self.server_protocol = HTTP1Protocol(self.server_conn) + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP1Protocol(self.server_conn) + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP1Protocol(self.server_conn) + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP1Protocol(self.server_conn) + def __call__(self): layer = HttpLayer(self, self.mode) - for message in layer(): - yield message - self.server_protocol = HTTP1Protocol(self.server_conn) - + layer() class Http2Layer(Layer): def __init__(self, ctx, mode): @@ -42,11 +51,21 @@ class Http2Layer(Layer): self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP2Protocol(self.server_conn) + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP2Protocol(self.server_conn) + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP2Protocol(self.server_conn) + def __call__(self): layer = HttpLayer(self, self.mode) - for message in layer(): - yield message - self.server_protocol = HTTP1Protocol(self.server_conn) + layer() def make_error_response(status_code, message, headers=None): @@ -115,6 +134,37 @@ class ConnectServerConnection(object): return getattr(self.via, item) +class UpstreamConnectLayer(Layer): + def __init__(self, ctx, connect_request): + super(UpstreamConnectLayer, self).__init__(ctx) + self.connect_request = connect_request + self.server_conn = ConnectServerConnection((connect_request.host, connect_request.port), self.ctx) + + def __call__(self): + layer = self.ctx.next_layer(self) + layer() + + def connect(self): + if not self.server_conn: + self.ctx.connect() + self.send_to_server(self.connect_request) + else: + pass # swallow the message + + def reconnect(self): + self.ctx.reconnect() + self.send_to_server(self.connect_request) + + def set_server(self, address, server_tls, sni, depth=1): + if depth == 1: + if self.ctx.server_conn: + self.ctx.reconnect() + self.connect_request.host = address.host + self.connect_request.port = address.port + self.server_conn.address = address + else: + self.ctx.set_server(address, server_tls, sni, depth-1) + class HttpLayer(Layer): """ HTTP 1 Layer @@ -146,22 +196,18 @@ class HttpLayer(Layer): # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.form_in == "authority": - for message in self.handle_regular_mode_connect(request): - yield message + self.handle_regular_mode_connect(request) return # Make sure that the incoming request matches our expectations self.validate_request(request) flow.request = request - for message in self.process_request_hook(flow): - yield message + self.process_request_hook(flow) if not flow.response: - for message in self.establish_server_connection(flow): - yield message - for message in self.get_response_from_server(flow): - yield message + self.establish_server_connection(flow) + self.get_response_from_server(flow) self.send_response_to_client(flow) @@ -172,8 +218,7 @@ class HttpLayer(Layer): # Upstream Proxy Mode: Handle CONNECT if flow.request.form_in == "authority" and flow.response.code == 200: - for message in self.handle_upstream_mode_connect(flow.request.copy()): - yield message + self.handle_upstream_mode_connect(flow.request.copy()) return except (HttpErrorConnClosed, NetLibError, HttpError) as e: @@ -186,38 +231,14 @@ class HttpLayer(Layer): flow.live = False def handle_regular_mode_connect(self, request): - yield SetServer((request.host, request.port), False, None) + self.set_server((request.host, request.port), False, None) self.send_to_client(make_connect_response(request.httpversion)) layer = self.ctx.next_layer(self) - for message in layer(): - yield message + layer() def handle_upstream_mode_connect(self, connect_request): - layer = self.ctx.next_layer(self) - self.server_conn = ConnectServerConnection((connect_request.host, connect_request.port), self.ctx) - - for message in layer(): - if message == Connect: - if not self.server_conn: - yield message - self.send_to_server(connect_request) - else: - pass # swallow the message - elif message == Reconnect: - yield message - self.send_to_server(connect_request) - elif message == SetServer: - if message.depth == 1: - if self.ctx.server_conn: - yield Reconnect() - connect_request.host = message.address.host - connect_request.port = message.address.port - self.server_conn.address = message.address - else: - message.depth -= 1 - yield message - else: - yield message + layer = UpstreamConnectLayer(self, connect_request) + layer() def check_close_connection(self, flow): """ @@ -313,14 +334,14 @@ class HttpLayer(Layer): # > server detects timeout, disconnects # > read (100-n)% of large request # > send large request upstream - yield Reconnect() + self.reconnect() get_response() # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True flow = self.channel.ask("responseheaders", flow) if flow is None or flow == KILL: - yield Kill() + raise Kill() if flow.response.stream and isinstance(self.server_protocol, http1.HTTP1Protocol): flow.response.content = CONTENT_MISSING @@ -345,7 +366,7 @@ class HttpLayer(Layer): ) response_reply = self.channel.ask("response", flow) if response_reply is None or response_reply == KILL: - yield Kill() + raise Kill() def process_request_hook(self, flow): # Determine .scheme, .host and .port attributes for inline scripts. @@ -363,10 +384,10 @@ class HttpLayer(Layer): flow.request.port = self.ctx.server_conn.address.port flow.request.scheme = "https" if self.server_conn.tls_established else "http" - # TODO: Expose SetServer functionality to inline scripts somehow? (yield_from_callback?) + # TODO: Expose .set_server functionality to inline scripts request_reply = self.channel.ask("request", flow) if request_reply is None or request_reply == KILL: - yield Kill() + raise Kill() if isinstance(request_reply, HTTPResponse): flow.response = request_reply return @@ -378,10 +399,10 @@ class HttpLayer(Layer): if self.mode == "regular" or self.mode == "transparent": # If there's an existing connection that doesn't match our expectations, kill it. if address != self.server_conn.address or tls != self.server_conn.ssl_established: - yield SetServer(address, tls, address.host) + self.set_server(address, tls, address.host) # Establish connection is neccessary. if not self.server_conn: - yield Connect() + self.connect() # SetServer is not guaranteed to work with TLS: # If there's not TlsLayer below which could catch the exception, @@ -391,16 +412,16 @@ class HttpLayer(Layer): else: if not self.server_conn: - yield Connect() + self.connect() if tls: raise HttpException("Cannot change scheme in upstream proxy mode.") """ # This is a very ugly (untested) workaround to solve a very ugly problem. if self.server_conn and self.server_conn.tls_established and not ssl: - yield Reconnect() + self.reconnect() elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: if self.server_conn.tls_established: - yield Reconnect() + self.reconnect() self.send_to_server(make_connect_request(address)) tls_layer = TlsLayer(self, False, True) diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 652aa473..c24af6cf 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -7,11 +7,11 @@ from .http import Http1Layer class HttpProxy(Layer, ServerConnectionMixin): def __call__(self): layer = Http1Layer(self, "regular") - for message in layer(): - if not self._handle_server_message(message): - yield message - if self.server_conn: - self._disconnect() + try: + layer() + finally: + if self.server_conn: + self._disconnect() class HttpUpstreamProxy(Layer, ServerConnectionMixin): def __init__(self, ctx, server_address): @@ -19,8 +19,8 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): def __call__(self): layer = Http1Layer(self, "upstream") - for message in layer(): - if not self._handle_server_message(message): - yield message - if self.server_conn: - self._disconnect() + try: + layer() + finally: + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index eb41bab7..7cb76591 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -21,7 +21,7 @@ Automated protocol detection by peeking into the buffer: Communication between layers is done as follows: - lower layers provide context information to higher layers - - higher layers can "yield" commands to lower layers, + - higher layers can call functions provided by lower layers, which are propagated until they reach a suitable layer. Further goals: @@ -35,7 +35,6 @@ import threading from netlib import tcp from ..proxy import Log from ..proxy.connection import ServerConnection -from .messages import Connect, Reconnect, SetServer, Kill from ..exceptions import ProtocolException @@ -69,7 +68,7 @@ class Layer(_LayerCodeCompletion): """ Logic of the layer. Raises: - ProxyError2 in case of protocol exceptions. + ProtocolException in case of protocol exceptions. """ raise NotImplementedError @@ -107,29 +106,20 @@ class ServerConnectionMixin(object): super(ServerConnectionMixin, self).__init__() self.server_conn = ServerConnection(server_address) - def _handle_server_message(self, message): - if message == Reconnect: - address = self.server_conn.address - self._disconnect() + def reconnect(self): + address = self.server_conn.address + self._disconnect() + self.server_conn.address = address + self.connect() + + def set_server(self, address, server_tls, sni, depth=1): + if depth == 1: + if self.server_conn: + self._disconnect() + self.log("Set new server address: " + repr(address), "debug") self.server_conn.address = address - self._connect() - return True - elif message == Connect: - self._connect() - return True - elif message == SetServer: - if message.depth == 1: - if self.server_conn: - self._disconnect() - self.log("Set new server address: " + repr(message.address), "debug") - self.server_conn.address = message.address - return True - else: - message.depth -= 1 - elif message == Kill: - self._disconnect() - - return False + else: + self.ctx.set_server(address, server_tls, sni, depth-1) def _disconnect(self): """ @@ -141,7 +131,7 @@ class ServerConnectionMixin(object): # self.channel.tell("serverdisconnect", self) self.server_conn = ServerConnection(None) - def _connect(self): + def connect(self): if not self.server_conn.address: raise ProtocolException("Cannot connect to server, no server address given.") self.log("serverconnect", "debug", [repr(self.server_conn.address)]) @@ -151,55 +141,7 @@ class ServerConnectionMixin(object): raise ProtocolException("Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) -def yield_from_callback(fun): +class Kill(Exception): """ - Decorator which makes it possible to yield from callbacks in the original thread. - As a use case, take the pyOpenSSL handle_sni callback: If we receive a new SNI from the client, - we need to reconnect to the server with the new SNI. Reconnecting would normally be done using "yield Reconnect()", - but we're in a pyOpenSSL callback here, outside of the main program flow. With this decorator, it looks as follows: - - def handle_sni(self): - # ... - self.yield_from_callback(Reconnect()) - - @yield_from_callback - def establish_ssl_with_client(): - self.client_conn.convert_to_ssl(...) - - for message in self.establish_ssl_with_client(): # will yield Reconnect at some point - yield message - - - Limitations: - - You cannot yield True. - """ - yield_queue = Queue.Queue() - - def do_yield(msg): - yield_queue.put(msg) - yield_queue.get() - - def wrapper(self, *args, **kwargs): - self.yield_from_callback = do_yield - - def run(): - try: - fun(self, *args, **kwargs) - yield_queue.put(True) - except Exception as e: - yield_queue.put(e) - - threading.Thread(target=run, name="YieldFromCallbackThread").start() - while True: - msg = yield_queue.get() - if msg is True: - break - elif isinstance(msg, Exception): - raise ProtocolException("Error in %s: %s" % (fun.__name__, repr(msg)), msg) - else: - yield msg - yield_queue.put(None) - - self.yield_from_callback = None - - return wrapper + Kill a connection. + """ \ No newline at end of file diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index f5907537..de049486 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -44,7 +44,3 @@ class SetServer(_Message): self.depth = depth -class Kill(_Message): - """ - Kill a connection. - """ \ No newline at end of file diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index 167c8c79..6819ad6e 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -4,12 +4,11 @@ import OpenSSL from ..exceptions import ProtocolException from ..protocol.tcp import TCPHandler from .layer import Layer -from .messages import Connect class RawTcpLayer(Layer): def __call__(self): - yield Connect() + self.connect() tcp_handler = TCPHandler(self) try: tcp_handler.handle_messages() diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index 767107ad..9d5a4beb 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -16,8 +16,9 @@ class ReverseProxy(Layer, ServerConnectionMixin): layer = TlsLayer(self, self._client_tls, self._server_tls) else: layer = self.ctx.next_layer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message - if self.server_conn: - self._disconnect() \ No newline at end of file + + try: + layer() + finally: + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index f8a645b0..f0e5b9a7 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -1,7 +1,7 @@ from __future__ import (absolute_import, print_function, division) import string -from .messages import Kill +from libmproxy.protocol2.layer import Kill from .rawtcp import RawTcpLayer from .tls import TlsLayer from .http import Http1Layer, Http2Layer, HttpLayer diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py index 5bb8e5f8..18b363d5 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/protocol2/socks_proxy.py @@ -5,7 +5,7 @@ from ..proxy import ProxyError, Socks5ProxyMode from .layer import Layer, ServerConnectionMixin -class Socks5Proxy(ServerConnectionMixin, Layer): +class Socks5Proxy(Layer, ServerConnectionMixin): def __call__(self): try: s5mode = Socks5ProxyMode(self.config.ssl_ports) @@ -16,9 +16,12 @@ class Socks5Proxy(ServerConnectionMixin, Layer): self.server_conn.address = address + # TODO: Kill event + layer = self.ctx.next_layer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message - if self.server_conn: - self._disconnect() \ No newline at end of file + + try: + layer() + finally: + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 970abe62..28480388 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -6,8 +6,7 @@ from netlib import tcp import netlib.http.http2 from ..exceptions import ProtocolException -from .layer import Layer, yield_from_callback -from .messages import Connect, Reconnect, SetServer +from .layer import Layer class TlsLayer(Layer): @@ -50,35 +49,34 @@ class TlsLayer(Layer): ) if client_tls_requires_server_cert: - for m in self._establish_tls_with_client_and_server(): - yield m + self._establish_tls_with_client_and_server() elif self._client_tls: - for m in self._establish_tls_with_client(): - yield m + self._establish_tls_with_client() layer = self.ctx.next_layer(self) + layer() - for message in layer(): - self.log("TlsLayer: %s" % message,"debug") - if not (message == Connect and self._connected): - yield message + def connect(self): + if not self.server_conn: + self.ctx.connect() + if self._server_tls and not self._server_tls_established: + self._establish_tls_with_server() + + def reconnect(self): + self.ctx.reconnect() + if self._server_tls and not self._server_tls_established: + self._establish_tls_with_server() - if message == Connect or message == Reconnect: - if self._server_tls and not self._server_tls_established: - self._establish_tls_with_server() - if message == SetServer and message.depth == 1: - if message.server_tls is not None: - self._sni_from_server_change = message.sni - self._server_tls = message.server_tls + def set_server(self, address, server_tls, sni, depth=1): + self.ctx.set_server(address, server_tls, sni, depth) + if server_tls is not None: + self._sni_from_server_change = sni + self._server_tls = server_tls @property def _server_tls_established(self): return self.server_conn and self.server_conn.tls_established - @property - def _connected(self): - return bool(self.server_conn) - @property def sni_for_upstream_connection(self): if self._sni_from_server_change is False: @@ -92,19 +90,14 @@ class TlsLayer(Layer): """ # First, try to connect to the server. - yield Connect() + self.ctx.connect() server_err = None try: self._establish_tls_with_server() except ProtocolException as e: server_err = e - for message in self._establish_tls_with_client(): - if message == Reconnect: - yield message - self._establish_tls_with_server() - else: - raise RuntimeError("Unexpected Message: %s" % message) + self._establish_tls_with_client() if server_err and not self.client_sni: raise server_err @@ -125,7 +118,7 @@ class TlsLayer(Layer): if old_upstream_sni != self.sni_for_upstream_connection: # Perform reconnect if self.server_conn and self._server_tls: - self.yield_from_callback(Reconnect()) + self.reconnect() if self.client_sni: # Now, change client context to reflect possibly changed certificate: @@ -156,7 +149,7 @@ class TlsLayer(Layer): # Perform reconnect # TODO: Avoid double reconnect. if self.server_conn and self._server_tls: - self.yield_from_callback(Reconnect()) + self.reconnect() self.client_alpn_protos = options @@ -165,7 +158,6 @@ class TlsLayer(Layer): else: # pragma no cover return options[0] - @yield_from_callback def _establish_tls_with_client(self): self.log("Establish TLS with client", "debug") cert, key, chain_file = self._find_cert() diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py index 28ad3726..9263dbde 100644 --- a/libmproxy/protocol2/transparent_proxy.py +++ b/libmproxy/protocol2/transparent_proxy.py @@ -18,8 +18,8 @@ class TransparentProxy(Layer, ServerConnectionMixin): raise ProtocolException("Transparent mode failure: %s" % repr(e), e) layer = self.ctx.next_layer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message - if self.server_conn: - self._disconnect() \ No newline at end of file + try: + layer() + finally: + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index e23a7d72..9957caa0 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, print_function import traceback import sys import socket +from libmproxy.protocol2.layer import Kill from netlib import tcp from ..protocol.handle import protocol_handler @@ -88,12 +89,9 @@ class ConnectionHandler2: root_layer = protocol2.HttpProxy(root_context) try: - for message in root_layer(): - if message == protocol2.messages.Kill: - self.log("Connection killed", "info") - break - - print("Root layer receveived: %s" % message) + root_layer() + except Kill as e: + self.log("Connection killed", "info") except ProtocolException as e: self.log(e, "info") except Exception: -- cgit v1.2.3 From 9bae97eb17ed66a33b5b988c6857ca6c9fae8e22 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 18 Aug 2015 13:43:26 +0200 Subject: http2: fix connection preface and wrappers --- libmproxy/protocol/http_wrappers.py | 36 ++++++++++-------------------------- libmproxy/protocol2/http.py | 16 +++++++--------- 2 files changed, 17 insertions(+), 35 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index ed5759ea..e41d65d6 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -247,24 +247,11 @@ class HTTPRequest(MessageMixin, semantics.Request): include_body = include_body, body_size_limit = body_size_limit, ) - - return HTTPRequest( - req.form_in, - req.method, - req.scheme, - req.host, - req.port, - req.path, - req.httpversion, - req.headers, - req.body, - req.timestamp_start, - req.timestamp_end, - ) + return self.wrap(req) @classmethod def wrap(self, request): - return HTTPRequest( + req = HTTPRequest( form_in=request.form_in, method=request.method, scheme=request.scheme, @@ -278,6 +265,9 @@ class HTTPRequest(MessageMixin, semantics.Request): timestamp_end=request.timestamp_end, form_out=(request.form_out if hasattr(request, 'form_out') else None), ) + if hasattr(request, 'stream_id'): + req.stream_id = request.stream_id + return req def __hash__(self): return id(self) @@ -371,20 +361,11 @@ class HTTPResponse(MessageMixin, semantics.Response): body_size_limit, include_body=include_body ) - - return HTTPResponse( - resp.httpversion, - resp.status_code, - resp.msg, - resp.headers, - resp.body, - resp.timestamp_start, - resp.timestamp_end, - ) + return self.wrap(resp) @classmethod def wrap(self, response): - return HTTPResponse( + resp = HTTPResponse( httpversion=response.httpversion, status_code=response.status_code, msg=response.msg, @@ -393,6 +374,9 @@ class HTTPResponse(MessageMixin, semantics.Response): timestamp_start=response.timestamp_start, timestamp_end=response.timestamp_end, ) + if hasattr(response, 'stream_id'): + resp.stream_id = response.stream_id + return resp def _refresh_cookie(self, c, delta): """ diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index db5aabaf..e73bbb61 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -10,7 +10,7 @@ from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from netlib import tcp -from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError +from netlib.http import status_codes, http1, http2, HttpErrorConnClosed, HttpError from netlib.http.semantics import CONTENT_MISSING from netlib import odict from netlib.tcp import NetLibError, Address @@ -64,6 +64,7 @@ class Http2Layer(Layer): self.server_protocol = HTTP2Protocol(self.server_conn) def __call__(self): + self.server_protocol.perform_connection_preface() layer = HttpLayer(self, self.mode) layer() @@ -166,10 +167,6 @@ class UpstreamConnectLayer(Layer): self.ctx.set_server(address, server_tls, sni, depth-1) class HttpLayer(Layer): - """ - HTTP 1 Layer - """ - def __init__(self, ctx, mode): super(HttpLayer, self).__init__(ctx) self.mode = mode @@ -337,15 +334,18 @@ class HttpLayer(Layer): self.reconnect() get_response() + if isinstance(self.server_protocol, http2.HTTP2Protocol): + flow.response.stream_id = flow.request.stream_id + # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True flow = self.channel.ask("responseheaders", flow) if flow is None or flow == KILL: raise Kill() - if flow.response.stream and isinstance(self.server_protocol, http1.HTTP1Protocol): + if flow.response.stream: flow.response.content = CONTENT_MISSING - else: + elif isinstance(self.server_protocol, http1.HTTP1Protocol): flow.response.content = self.server_protocol.read_http_body( flow.response.headers, self.config.body_size_limit, @@ -466,6 +466,4 @@ class HttpLayer(Layer): self.server_conn.send(self.server_protocol.assemble(message)) def send_to_client(self, message): - # FIXME - # - possibly do some http2 stuff here self.client_conn.send(self.client_protocol.assemble(message)) -- cgit v1.2.3 From c9fa8491ccc015ddff09ce15a5d718d6b58b515c Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 19 Aug 2015 15:23:52 +0200 Subject: improve next_layer detection --- libmproxy/protocol2/root_context.py | 31 +++++++++++++++++++------------ libmproxy/proxy/connection.py | 2 +- 2 files changed, 20 insertions(+), 13 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index f0e5b9a7..9b18f0aa 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -6,6 +6,7 @@ from .rawtcp import RawTcpLayer from .tls import TlsLayer from .http import Http1Layer, Http2Layer, HttpLayer +from netlib.http.http2 import HTTP2Protocol class RootContext(object): """ @@ -25,11 +26,11 @@ class RootContext(object): :return: The next layer. """ - d = top_layer.client_conn.rfile.peek(3) - # TODO: Handle ignore and tcp passthrough - # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + d = top_layer.client_conn.rfile.peek(3) is_tls_client_hello = ( len(d) == 3 and d[0] == '\x16' and @@ -37,20 +38,26 @@ class RootContext(object): d[2] in ('\x00', '\x01', '\x02', '\x03') ) - is_ascii = all(x in string.ascii_uppercase for x in d) + d = top_layer.client_conn.rfile.peek(3) + is_ascii = ( + len(d) == 3 and + all(x in string.ascii_uppercase for x in d) + ) - # TODO: build is_http2_magic check here, maybe this is an easy way to detect h2c + d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) + is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) - if not d: - return iter([]) + is_alpn_h2_negotiated = ( + isinstance(top_layer, TlsLayer) and + top_layer.client_conn.get_alpn_proto_negotiated() == HTTP2Protocol.ALPN_PROTO_H2 + ) if is_tls_client_hello: return TlsLayer(top_layer, True, True) - elif isinstance(top_layer, TlsLayer) and is_ascii: - if top_layer.client_conn.get_alpn_proto_negotiated() == 'h2': - return Http2Layer(top_layer, 'transparent') - else: - return Http1Layer(top_layer, "transparent") + elif is_alpn_h2_negotiated or is_http2_magic: + return Http2Layer(top_layer, 'transparent') + elif is_ascii: + return Http1Layer(top_layer, 'transparent') else: return RawTcpLayer(top_layer) diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index c9b57998..c329ed64 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -190,4 +190,4 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): tcp.TCPClient.finish(self) self.timestamp_end = utils.timestamp() -ServerConnection._stateobject_attributes["via"] = ServerConnection \ No newline at end of file +ServerConnection._stateobject_attributes["via"] = ServerConnection -- cgit v1.2.3 From 97bfd1d856b26216fffbf84b9e75e49b41fc6fb2 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 19 Aug 2015 16:36:22 +0200 Subject: move send method to lower layers --- libmproxy/protocol2/http.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index e73bbb61..b8b1e8a5 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -28,6 +28,12 @@ class Http1Layer(Layer): self.client_protocol = HTTP1Protocol(self.client_conn) self.server_protocol = HTTP1Protocol(self.server_conn) + def send_to_client(self, message): + self.client_conn.send(self.client_protocol.assemble(message)) + + def send_to_server(self, message): + self.server_conn.send(self.server_protocol.assemble(message)) + def connect(self): self.ctx.connect() self.server_protocol = HTTP1Protocol(self.server_conn) @@ -51,6 +57,14 @@ class Http2Layer(Layer): self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + def send_to_client(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.client_conn.send(self.client_protocol.assemble(message)) + + def send_to_server(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.server_conn.send(self.server_protocol.assemble(message)) + def connect(self): self.ctx.connect() self.server_protocol = HTTP2Protocol(self.server_conn) @@ -166,6 +180,7 @@ class UpstreamConnectLayer(Layer): else: self.ctx.set_server(address, server_tls, sni, depth-1) + class HttpLayer(Layer): def __init__(self, ctx, mode): super(HttpLayer, self).__init__(ctx) @@ -461,9 +476,3 @@ class HttpLayer(Layer): odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()]) )) raise InvalidCredentials("Proxy Authentication Required") - - def send_to_server(self, message): - self.server_conn.send(self.server_protocol.assemble(message)) - - def send_to_client(self, message): - self.client_conn.send(self.client_protocol.assemble(message)) -- cgit v1.2.3 From f2ace5493b1fc6eceff46cff3fb1238559e9aef2 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 19 Aug 2015 18:09:45 +0200 Subject: move read methods to lower HTTP layer --- libmproxy/protocol2/http.py | 53 +++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 16 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index b8b1e8a5..5170660f 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -28,6 +28,20 @@ class Http1Layer(Layer): self.client_protocol = HTTP1Protocol(self.client_conn) self.server_protocol = HTTP1Protocol(self.server_conn) + def read_from_client(self): + return HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + + def read_from_server(self, method): + return HTTPResponse.from_protocol( + self.server_protocol, + method, + body_size_limit=self.config.body_size_limit, + include_body=False, + ) + def send_to_client(self, message): self.client_conn.send(self.client_protocol.assemble(message)) @@ -57,6 +71,20 @@ class Http2Layer(Layer): self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + def read_from_client(self): + return HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + + def read_from_server(self, method): + return HTTPResponse.from_protocol( + self.server_protocol, + method, + body_size_limit=self.config.body_size_limit, + include_body=False, + ) + def send_to_client(self, message): # TODO: implement flow control and WINDOW_UPDATE frames self.client_conn.send(self.client_protocol.assemble(message)) @@ -67,15 +95,18 @@ class Http2Layer(Layer): def connect(self): self.ctx.connect() - self.server_protocol = HTTP2Protocol(self.server_conn) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.server_protocol.perform_connection_preface() def reconnect(self): self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.server_protocol.perform_connection_preface() def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP2Protocol(self.server_conn) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.server_protocol.perform_connection_preface() def __call__(self): self.server_protocol.perform_connection_preface() @@ -192,10 +223,7 @@ class HttpLayer(Layer): flow = HTTPFlow(self.client_conn, self.server_conn, live=True) try: - request = HTTPRequest.from_protocol( - self.client_protocol, - body_size_limit=self.config.body_size_limit - ) + request = self.read_from_client() except tcp.NetLibError: # don't throw an error for disconnects that happen # before/between requests. @@ -320,13 +348,7 @@ class HttpLayer(Layer): def get_response_from_server(self, flow): def get_response(): self.send_to_server(flow.request) - # Only get the headers at first... - flow.response = HTTPResponse.from_protocol( - self.server_protocol, - flow.request.method, - body_size_limit=self.config.body_size_limit, - include_body=False, - ) + flow.response = self.read_from_server(flow.request.method) try: get_response() @@ -408,9 +430,9 @@ class HttpLayer(Layer): return def establish_server_connection(self, flow): - address = tcp.Address((flow.request.host, flow.request.port)) tls = (flow.request.scheme == "https") + if self.mode == "regular" or self.mode == "transparent": # If there's an existing connection that doesn't match our expectations, kill it. if address != self.server_conn.address or tls != self.server_conn.ssl_established: @@ -424,7 +446,6 @@ class HttpLayer(Layer): # TLS will not be established. if tls and not self.server_conn.tls_established: raise ProtocolException("Cannot upgrade to SSL, no TLS layer on the protocol stack.") - else: if not self.server_conn: self.connect() -- cgit v1.2.3 From 4339b8e7fa1140b9138a023e7e61d78cefe6bb02 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 19 Aug 2015 21:09:48 +0200 Subject: http2: use callback for handle unexpected frames --- libmproxy/protocol2/http.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 5170660f..e227f0ba 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -68,8 +68,8 @@ class Http2Layer(Layer): def __init__(self, ctx, mode): super(Http2Layer, self).__init__(ctx) self.mode = mode - self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) def read_from_client(self): return HTTPRequest.from_protocol( @@ -95,17 +95,17 @@ class Http2Layer(Layer): def connect(self): self.ctx.connect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol.perform_connection_preface() def reconnect(self): self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol.perform_connection_preface() def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol.perform_connection_preface() def __call__(self): @@ -113,6 +113,9 @@ class Http2Layer(Layer): layer = HttpLayer(self, self.mode) layer() + def handle_unexpected_frame(self, frm): + print(frm.human_readable()) + def make_error_response(status_code, message, headers=None): response = status_codes.RESPONSES.get(status_code, "Unknown") -- cgit v1.2.3 From 5746472426d3928497e9c8f85664a46598a044af Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 20 Aug 2015 19:53:17 +0200 Subject: fix typo --- libmproxy/protocol2/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index e227f0ba..e5a434f2 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -469,7 +469,7 @@ class HttpLayer(Layer): def validate_request(self, request): if request.form_in == "absolute" and request.scheme != "http": - self.send_resplonse(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + self.send_response(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { -- cgit v1.2.3 From 55cfd259dc8264ca9bca166b50b0d3dc7ab71451 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 20 Aug 2015 20:31:01 +0200 Subject: http2: simplify protocol-related code --- libmproxy/protocol2/http.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index e5a434f2..a2dfc428 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -34,10 +34,10 @@ class Http1Layer(Layer): body_size_limit=self.config.body_size_limit ) - def read_from_server(self, method): + def read_from_server(self, request): return HTTPResponse.from_protocol( self.server_protocol, - method, + request.method, body_size_limit=self.config.body_size_limit, include_body=False, ) @@ -77,13 +77,15 @@ class Http2Layer(Layer): body_size_limit=self.config.body_size_limit ) - def read_from_server(self, method): - return HTTPResponse.from_protocol( + def read_from_server(self, request): + response = HTTPResponse.from_protocol( self.server_protocol, - method, + request.method, body_size_limit=self.config.body_size_limit, include_body=False, ) + response.stream_id = request.stream_id + return response def send_to_client(self, message): # TODO: implement flow control and WINDOW_UPDATE frames @@ -351,7 +353,7 @@ class HttpLayer(Layer): def get_response_from_server(self, flow): def get_response(): self.send_to_server(flow.request) - flow.response = self.read_from_server(flow.request.method) + flow.response = self.read_from_server(flow.request) try: get_response() @@ -374,9 +376,6 @@ class HttpLayer(Layer): self.reconnect() get_response() - if isinstance(self.server_protocol, http2.HTTP2Protocol): - flow.response.stream_id = flow.request.stream_id - # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True flow = self.channel.ask("responseheaders", flow) -- cgit v1.2.3 From 05d26545e4c65e8fd2242142833a40a96ce5fb81 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 21 Aug 2015 10:26:28 +0200 Subject: adapt netlib changes --- libmproxy/protocol/http.py | 6 +++--- libmproxy/protocol/http_wrappers.py | 4 ++-- libmproxy/protocol2/http.py | 4 ++-- 3 files changed, 7 insertions(+), 7 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4472cb2a..1b168569 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -41,7 +41,7 @@ def send_connect_request(conn, host, port, update_state=True): protocol = http1.HTTP1Protocol(conn) conn.send(protocol.assemble(upstream_request)) - resp = HTTPResponse.from_protocol(protocol, upstream_request.method) + resp = HTTPResponse.from_protocol(protocol, upstream_request) if resp.status_code != 200: raise proxy.ProxyError(resp.status_code, "Cannot establish SSL " + @@ -177,7 +177,7 @@ class HTTPHandler(ProtocolHandler): # Only get the headers at first... flow.response = HTTPResponse.from_protocol( self.c.server_conn.protocol, - flow.request.method, + flow.request, body_size_limit=self.c.config.body_size_limit, include_body=False, ) @@ -760,7 +760,7 @@ class RequestReplayThread(threading.Thread): self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) self.flow.response = HTTPResponse.from_protocol( self.flow.server_conn.protocol, - r.method, + r, body_size_limit=self.config.body_size_limit, ) if self.channel: diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index e41d65d6..f91b936c 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -352,12 +352,12 @@ class HTTPResponse(MessageMixin, semantics.Response): def from_protocol( self, protocol, - request_method, + request, include_body=True, body_size_limit=None ): resp = protocol.read_response( - request_method, + request, body_size_limit, include_body=include_body ) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index a2dfc428..f093f7c5 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -37,7 +37,7 @@ class Http1Layer(Layer): def read_from_server(self, request): return HTTPResponse.from_protocol( self.server_protocol, - request.method, + request, body_size_limit=self.config.body_size_limit, include_body=False, ) @@ -80,7 +80,7 @@ class Http2Layer(Layer): def read_from_server(self, request): response = HTTPResponse.from_protocol( self.server_protocol, - request.method, + request, body_size_limit=self.config.body_size_limit, include_body=False, ) -- cgit v1.2.3 From f1f34e7713295adb8f54b13ec50d74d43cd42841 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 24 Aug 2015 16:52:03 +0200 Subject: fix bugs, fix tests --- libmproxy/protocol2/http.py | 9 +++-- libmproxy/protocol2/root_context.py | 2 +- libmproxy/protocol2/tls.py | 79 ++++++++++++++++++++----------------- 3 files changed, 49 insertions(+), 41 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index f093f7c5..973f169c 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -266,12 +266,15 @@ class HttpLayer(Layer): self.handle_upstream_mode_connect(flow.request.copy()) return - except (HttpErrorConnClosed, NetLibError, HttpError) as e: + except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: self.send_to_client(make_error_response( getattr(e, "code", 502), repr(e) )) - raise ProtocolException(repr(e), e) + if isinstance(e, ProtocolException): + raise e + else: + raise ProtocolException(repr(e), e) finally: flow.live = False @@ -468,7 +471,7 @@ class HttpLayer(Layer): def validate_request(self, request): if request.form_in == "absolute" and request.scheme != "http": - self.send_response(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + self.send_to_client(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 9b18f0aa..d0c62be4 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -41,7 +41,7 @@ class RootContext(object): d = top_layer.client_conn.rfile.peek(3) is_ascii = ( len(d) == 3 and - all(x in string.ascii_uppercase for x in d) + all(x in string.ascii_letters for x in d) # better be safe here and don't expect uppercase... ) d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 28480388..98c5d603 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -17,6 +17,7 @@ class TlsLayer(Layer): self.client_sni = None self._sni_from_server_change = None self.client_alpn_protos = None + self.__server_tls_exception = None # foo alpn protos = [netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1, netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], # TODO: read this from client_conn first @@ -107,49 +108,48 @@ class TlsLayer(Layer): This callback gets called during the TLS handshake with the client. The client has just sent the Sever Name Indication (SNI). """ - try: - old_upstream_sni = self.sni_for_upstream_connection - - sn = connection.get_servername() - if not sn: - return - self.client_sni = sn.decode("utf8").encode("idna") - - if old_upstream_sni != self.sni_for_upstream_connection: - # Perform reconnect - if self.server_conn and self._server_tls: - self.reconnect() - - if self.client_sni: - # Now, change client context to reflect possibly changed certificate: - cert, key, chain_file = self._find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file, - alpn_select_callback=self.__handle_alpn_select, - ) - connection.set_context(new_context) - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except: # pragma: no cover - self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") + old_upstream_sni = self.sni_for_upstream_connection + + sn = connection.get_servername() + if not sn: + return + + self.client_sni = sn.decode("utf8").encode("idna") + + server_sni_changed = (old_upstream_sni != self.sni_for_upstream_connection) + server_conn_with_tls_exists = (self.server_conn and self._server_tls) + if server_sni_changed and server_conn_with_tls_exists: + try: + self.reconnect() + except Exception as e: + self.__server_tls_exception = e + + # Now, change client context to reflect possibly changed certificate: + cert, key, chain_file = self._find_cert() + new_context = self.client_conn.create_ssl_context( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file, + alpn_select_callback=self.__handle_alpn_select, + ) + connection.set_context(new_context) def __handle_alpn_select(self, conn_, options): # TODO: change to something meaningful? - alpn_preference = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 + # alpn_preference = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 - ### - # TODO: Not - if self.client_alpn_protos != options: - # Perform reconnect - # TODO: Avoid double reconnect. - if self.server_conn and self._server_tls: + # TODO: Don't reconnect twice? + upstream_alpn_changed = (self.client_alpn_protos != options) + server_conn_with_tls_exists = (self.server_conn and self._server_tls) + if upstream_alpn_changed and server_conn_with_tls_exists: + try: self.reconnect() + except Exception as e: + self.__server_tls_exception = e self.client_alpn_protos = options @@ -177,6 +177,11 @@ class TlsLayer(Layer): print("alpn: %s" % self.client_alpn_protos) raise ProtocolException(repr(e), e) + # Do not raise server tls exceptions immediately. + # We want to try to finish the client handshake so that other layers can send error messages over it. + if self.__server_tls_exception: + raise self.__server_tls_exception + def _establish_tls_with_server(self): self.log("Establish TLS with server", "debug") try: -- cgit v1.2.3 From 56a4bc381efc177c08ed7b9e7c845f74120050e4 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 24 Aug 2015 18:17:04 +0200 Subject: request -> request_method --- libmproxy/protocol/http.py | 6 +++--- libmproxy/protocol/http_wrappers.py | 20 ++++++-------------- libmproxy/protocol2/http.py | 19 ++++++++++--------- 3 files changed, 19 insertions(+), 26 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 1b168569..4472cb2a 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -41,7 +41,7 @@ def send_connect_request(conn, host, port, update_state=True): protocol = http1.HTTP1Protocol(conn) conn.send(protocol.assemble(upstream_request)) - resp = HTTPResponse.from_protocol(protocol, upstream_request) + resp = HTTPResponse.from_protocol(protocol, upstream_request.method) if resp.status_code != 200: raise proxy.ProxyError(resp.status_code, "Cannot establish SSL " + @@ -177,7 +177,7 @@ class HTTPHandler(ProtocolHandler): # Only get the headers at first... flow.response = HTTPResponse.from_protocol( self.c.server_conn.protocol, - flow.request, + flow.request.method, body_size_limit=self.c.config.body_size_limit, include_body=False, ) @@ -760,7 +760,7 @@ class RequestReplayThread(threading.Thread): self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) self.flow.response = HTTPResponse.from_protocol( self.flow.server_conn.protocol, - r, + r.method, body_size_limit=self.config.body_size_limit, ) if self.channel: diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index f91b936c..b1000a79 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -240,13 +240,10 @@ class HTTPRequest(MessageMixin, semantics.Request): def from_protocol( self, protocol, - include_body=True, - body_size_limit=None, + *args, + **kwargs ): - req = protocol.read_request( - include_body = include_body, - body_size_limit = body_size_limit, - ) + req = protocol.read_request(*args, **kwargs) return self.wrap(req) @classmethod @@ -352,15 +349,10 @@ class HTTPResponse(MessageMixin, semantics.Response): def from_protocol( self, protocol, - request, - include_body=True, - body_size_limit=None + *args, + **kwargs ): - resp = protocol.read_response( - request, - body_size_limit, - include_body=include_body - ) + resp = protocol.read_response(*args, **kwargs) return self.wrap(resp) @classmethod diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 973f169c..2c8c8d27 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -34,10 +34,10 @@ class Http1Layer(Layer): body_size_limit=self.config.body_size_limit ) - def read_from_server(self, request): + def read_from_server(self, request_method): return HTTPResponse.from_protocol( self.server_protocol, - request, + request_method, body_size_limit=self.config.body_size_limit, include_body=False, ) @@ -64,6 +64,7 @@ class Http1Layer(Layer): layer = HttpLayer(self, self.mode) layer() + class Http2Layer(Layer): def __init__(self, ctx, mode): super(Http2Layer, self).__init__(ctx) @@ -72,20 +73,20 @@ class Http2Layer(Layer): self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) def read_from_client(self): - return HTTPRequest.from_protocol( + request = HTTPRequest.from_protocol( self.client_protocol, body_size_limit=self.config.body_size_limit ) + self._stream_id = request.stream_id - def read_from_server(self, request): - response = HTTPResponse.from_protocol( + def read_from_server(self, request_method): + return HTTPResponse.from_protocol( self.server_protocol, - request, + request_method, body_size_limit=self.config.body_size_limit, include_body=False, + stream_id=self._stream_id ) - response.stream_id = request.stream_id - return response def send_to_client(self, message): # TODO: implement flow control and WINDOW_UPDATE frames @@ -356,7 +357,7 @@ class HttpLayer(Layer): def get_response_from_server(self, flow): def get_response(): self.send_to_server(flow.request) - flow.response = self.read_from_server(flow.request) + flow.response = self.read_from_server(flow.request.method) try: get_response() -- cgit v1.2.3 From 8ce0de8bed5fbd2c42e7b43ee553e065e1c08a4c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 25 Aug 2015 18:24:17 +0200 Subject: minor fixes --- libmproxy/protocol2/http.py | 11 +++++++---- libmproxy/proxy/server.py | 2 ++ 2 files changed, 9 insertions(+), 4 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 2c8c8d27..5a25c317 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -268,10 +268,13 @@ class HttpLayer(Layer): return except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: - self.send_to_client(make_error_response( - getattr(e, "code", 502), - repr(e) - )) + try: + self.send_to_client(make_error_response( + getattr(e, "code", 502), + repr(e) + )) + except NetLibError: + pass if isinstance(e, ProtocolException): raise e else: diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 9957caa0..19ddb930 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -100,6 +100,8 @@ class ConnectionHandler2: print("mitmproxy has crashed!", file=sys.stderr) print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) + self.log("clientdisconnect", "info") + def finish(self): self.client_conn.finish() -- cgit v1.2.3 From 3fa65c48dd2880a806985e273b3fa280103e2a7b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Aug 2015 05:39:00 +0200 Subject: manually read tls clienthello [wip] --- libmproxy/contrib/tls/__init__.py | 5 + libmproxy/contrib/tls/_constructs.py | 213 ++++++++++++++++++++ libmproxy/contrib/tls/alert_message.py | 64 ++++++ libmproxy/contrib/tls/ciphersuites.py | 343 +++++++++++++++++++++++++++++++++ libmproxy/contrib/tls/exceptions.py | 2 + libmproxy/contrib/tls/hello_message.py | 178 +++++++++++++++++ libmproxy/contrib/tls/message.py | 313 ++++++++++++++++++++++++++++++ libmproxy/contrib/tls/record.py | 110 +++++++++++ libmproxy/contrib/tls/utils.py | 52 +++++ libmproxy/protocol2/tls.py | 44 +++-- 10 files changed, 1313 insertions(+), 11 deletions(-) create mode 100644 libmproxy/contrib/tls/__init__.py create mode 100644 libmproxy/contrib/tls/_constructs.py create mode 100644 libmproxy/contrib/tls/alert_message.py create mode 100644 libmproxy/contrib/tls/ciphersuites.py create mode 100644 libmproxy/contrib/tls/exceptions.py create mode 100644 libmproxy/contrib/tls/hello_message.py create mode 100644 libmproxy/contrib/tls/message.py create mode 100644 libmproxy/contrib/tls/record.py create mode 100644 libmproxy/contrib/tls/utils.py (limited to 'libmproxy') diff --git a/libmproxy/contrib/tls/__init__.py b/libmproxy/contrib/tls/__init__.py new file mode 100644 index 00000000..4b540884 --- /dev/null +++ b/libmproxy/contrib/tls/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function diff --git a/libmproxy/contrib/tls/_constructs.py b/libmproxy/contrib/tls/_constructs.py new file mode 100644 index 00000000..49661efb --- /dev/null +++ b/libmproxy/contrib/tls/_constructs.py @@ -0,0 +1,213 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from construct import Array, Bytes, Struct, UBInt16, UBInt32, UBInt8, PascalString, Embed, \ + TunnelAdapter, GreedyRange, Switch + +from .utils import UBInt24 + +ProtocolVersion = Struct( + "version", + UBInt8("major"), + UBInt8("minor"), +) + +TLSPlaintext = Struct( + "TLSPlaintext", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + Bytes("fragment", lambda ctx: ctx.length), +) + +TLSCompressed = Struct( + "TLSCompressed", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + 1024 + Bytes("fragment", lambda ctx: ctx.length), +) + +TLSCiphertext = Struct( + "TLSCiphertext", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + 2048 + Bytes("fragment", lambda ctx: ctx.length), +) + +Random = Struct( + "random", + UBInt32("gmt_unix_time"), + Bytes("random_bytes", 28), +) + +SessionID = Struct( + "session_id", + UBInt8("length"), + Bytes("session_id", lambda ctx: ctx.length), +) + +CipherSuites = Struct( + "cipher_suites", + UBInt16("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length // 2, UBInt16("cipher_suites")), +) + +CompressionMethods = Struct( + "compression_methods", + UBInt8("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length, UBInt8("compression_methods")), +) + +ServerName = Struct( + "", + UBInt8("type"), + PascalString("name", length_field=UBInt16("length")), +) + +SNIExtension = Struct( + "", + TunnelAdapter( + PascalString("server_names", length_field=UBInt16("length")), + TunnelAdapter( + PascalString("", length_field=UBInt16("length")), + GreedyRange(ServerName) + ), + ), +) + +ALPNExtension = Struct( + "", + TunnelAdapter( + PascalString("alpn_protocols", length_field=UBInt16("length")), + TunnelAdapter( + PascalString("", length_field=UBInt16("length")), + GreedyRange(PascalString("name")) + ), + ), +) + +UnknownExtension = Struct( + "", + PascalString("bytes", length_field=UBInt16("extensions_length")) +) + +Extension = Struct( + "Extension", + UBInt16("type"), + Embed( + Switch( + "data", lambda ctx: ctx.type, + { + 0x00: SNIExtension, + 0x10: ALPNExtension + }, + default=UnknownExtension + ) + ) +) + +extensions = TunnelAdapter( + PascalString("extensions", length_field=UBInt16("extensions_length")), + GreedyRange(Extension) +) + +ClientHello = Struct( + "ClientHello", + ProtocolVersion, + Random, + SessionID, + CipherSuites, + CompressionMethods, + extensions, +) + +ServerHello = Struct( + "ServerHello", + ProtocolVersion, + Random, + SessionID, + Bytes("cipher_suite", 2), + UBInt8("compression_method"), + extensions, +) + +ClientCertificateType = Struct( + "certificate_types", + UBInt8("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length, UBInt8("certificate_types")), +) + +SignatureAndHashAlgorithm = Struct( + "algorithms", + UBInt8("hash"), + UBInt8("signature"), +) + +SupportedSignatureAlgorithms = Struct( + "supported_signature_algorithms", + UBInt16("supported_signature_algorithms_length"), + # TODO: Reject packets of length 0 + Array( + lambda ctx: ctx.supported_signature_algorithms_length / 2, + SignatureAndHashAlgorithm, + ), +) + +DistinguishedName = Struct( + "certificate_authorities", + UBInt16("length"), + Bytes("certificate_authorities", lambda ctx: ctx.length), +) + +CertificateRequest = Struct( + "CertificateRequest", + ClientCertificateType, + SupportedSignatureAlgorithms, + DistinguishedName, +) + +ServerDHParams = Struct( + "ServerDHParams", + UBInt16("dh_p_length"), + Bytes("dh_p", lambda ctx: ctx.dh_p_length), + UBInt16("dh_g_length"), + Bytes("dh_g", lambda ctx: ctx.dh_g_length), + UBInt16("dh_Ys_length"), + Bytes("dh_Ys", lambda ctx: ctx.dh_Ys_length), +) + +PreMasterSecret = Struct( + "pre_master_secret", + ProtocolVersion, + Bytes("random_bytes", 46), +) + +ASN1Cert = Struct( + "ASN1Cert", + UBInt32("length"), # TODO: Reject packets with length not in 1..2^24-1 + Bytes("asn1_cert", lambda ctx: ctx.length), +) + +Certificate = Struct( + "Certificate", # TODO: Reject packets with length > 2 ** 24 - 1 + UBInt32("certificates_length"), + Bytes("certificates_bytes", lambda ctx: ctx.certificates_length), +) + +Handshake = Struct( + "Handshake", + UBInt8("msg_type"), + UBInt24("length"), # TODO: Reject packets with length > 2 ** 24 + Bytes("body", lambda ctx: ctx.length), +) + +Alert = Struct( + "Alert", + UBInt8("level"), + UBInt8("description"), +) diff --git a/libmproxy/contrib/tls/alert_message.py b/libmproxy/contrib/tls/alert_message.py new file mode 100644 index 00000000..ef02f56d --- /dev/null +++ b/libmproxy/contrib/tls/alert_message.py @@ -0,0 +1,64 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +from characteristic import attributes + +from . import _constructs + + +class AlertLevel(Enum): + WARNING = 1 + FATAL = 2 + + +class AlertDescription(Enum): + CLOSE_NOTIFY = 0 + UNEXPECTED_MESSAGE = 10 + BAD_RECORD_MAC = 20 + DECRYPTION_FAILED_RESERVED = 21 + RECORD_OVERFLOW = 22 + DECOMPRESSION_FAILURE = 30 + HANDSHAKE_FAILURE = 40 + NO_CERTIFICATE_RESERVED = 41 + BAD_CERTIFICATE = 42 + UNSUPPORTED_CERTIFICATE = 43 + CERTIFICATE_REVOKED = 44 + CERTIFICATE_EXPIRED = 45 + CERTIFICATE_UNKNOWN = 46 + ILLEGAL_PARAMETER = 47 + UNKNOWN_CA = 48 + ACCESS_DENIED = 49 + DECODE_ERROR = 50 + DECRYPT_ERROR = 51 + EXPORT_RESTRICTION_RESERVED = 60 + PROTOCOL_VERSION = 70 + INSUFFICIENT_SECURITY = 71 + INTERNAL_ERROR = 80 + USER_CANCELED = 90 + NO_RENEGOTIATION = 100 + UNSUPPORTED_EXTENSION = 110 + + +@attributes(['level', 'description']) +class Alert(object): + """ + An object representing an Alert message. + """ + @classmethod + def from_bytes(cls, bytes): + """ + Parse an ``Alert`` struct. + + :param bytes: the bytes representing the input. + :return: Alert object. + """ + construct = _constructs.Alert.parse(bytes) + return cls( + level=AlertLevel(construct.level), + description=AlertDescription(construct.description) + ) diff --git a/libmproxy/contrib/tls/ciphersuites.py b/libmproxy/contrib/tls/ciphersuites.py new file mode 100644 index 00000000..86298f80 --- /dev/null +++ b/libmproxy/contrib/tls/ciphersuites.py @@ -0,0 +1,343 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +from .exceptions import UnsupportedCipherException + + +class CipherSuites(Enum): + TLS_NULL_WITH_NULL_NULL = 0x0000 + TLS_RSA_WITH_NULL_MD5 = 0x0001 + TLS_RSA_WITH_NULL_SHA = 0x0002 + TLS_RSA_EXPORT_WITH_RC4_40_MD5 = 0x0003 + TLS_RSA_WITH_RC4_128_MD5 = 0x0004 + TLS_RSA_WITH_RC4_128_SHA = 0x0005 + TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5 = 0x0006 + TLS_RSA_WITH_IDEA_CBC_SHA = 0x0007 + TLS_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0008 + TLS_RSA_WITH_DES_CBC_SHA = 0x0009 + TLS_RSA_WITH_3DES_EDE_CBC_SHA = 0x000A + TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x000B + TLS_DH_DSS_WITH_DES_CBC_SHA = 0x000C + TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA = 0x000D + TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x000E + TLS_DH_RSA_WITH_DES_CBC_SHA = 0x000F + TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA = 0x0010 + TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x0011 + TLS_DHE_DSS_WITH_DES_CBC_SHA = 0x0012 + TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA = 0x0013 + TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0014 + TLS_DHE_RSA_WITH_DES_CBC_SHA = 0x0015 + TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA = 0x0016 + TLS_DH_anon_EXPORT_WITH_RC4_40_MD5 = 0x0017 + TLS_DH_anon_WITH_RC4_128_MD5 = 0x0018 + TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA = 0x0019 + TLS_DH_anon_WITH_DES_CBC_SHA = 0x001A + TLS_DH_anon_WITH_3DES_EDE_CBC_SHA = 0x001B + TLS_KRB5_WITH_DES_CBC_SHA = 0x001E + TLS_KRB5_WITH_3DES_EDE_CBC_SHA = 0x001F + TLS_KRB5_WITH_RC4_128_SHA = 0x0020 + TLS_KRB5_WITH_IDEA_CBC_SHA = 0x0021 + TLS_KRB5_WITH_DES_CBC_MD5 = 0x0022 + TLS_KRB5_WITH_3DES_EDE_CBC_MD5 = 0x0023 + TLS_KRB5_WITH_RC4_128_MD5 = 0x0024 + TLS_KRB5_WITH_IDEA_CBC_MD5 = 0x0025 + TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA = 0x0026 + TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA = 0x0027 + TLS_KRB5_EXPORT_WITH_RC4_40_SHA = 0x0028 + TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5 = 0x0029 + TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5 = 0x002A + TLS_KRB5_EXPORT_WITH_RC4_40_MD5 = 0x002B + TLS_PSK_WITH_NULL_SHA = 0x002C + TLS_DHE_PSK_WITH_NULL_SHA = 0x002D + TLS_RSA_PSK_WITH_NULL_SHA = 0x002E + TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F + TLS_DH_DSS_WITH_AES_128_CBC_SHA = 0x0030 + TLS_DH_RSA_WITH_AES_128_CBC_SHA = 0x0031 + TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032 + TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033 + TLS_DH_anon_WITH_AES_128_CBC_SHA = 0x0034 + TLS_RSA_WITH_AES_256_CBC_SHA = 0x0035 + TLS_DH_DSS_WITH_AES_256_CBC_SHA = 0x0036 + TLS_DH_RSA_WITH_AES_256_CBC_SHA = 0x0037 + TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038 + TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039 + TLS_DH_anon_WITH_AES_256_CBC_SHA = 0x003A + TLS_RSA_WITH_NULL_SHA256 = 0x003B + TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x003C + TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D + TLS_DH_DSS_WITH_AES_128_CBC_SHA256 = 0x003E + TLS_DH_RSA_WITH_AES_128_CBC_SHA256 = 0x003F + TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040 + TLS_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0041 + TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA = 0x0042 + TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0043 + TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA = 0x0044 + TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0045 + TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA = 0x0046 + TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067 + TLS_DH_DSS_WITH_AES_256_CBC_SHA256 = 0x0068 + TLS_DH_RSA_WITH_AES_256_CBC_SHA256 = 0x0069 + TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006A + TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B + TLS_DH_anon_WITH_AES_128_CBC_SHA256 = 0x006C + TLS_DH_anon_WITH_AES_256_CBC_SHA256 = 0x006D + TLS_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0084 + TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA = 0x0085 + TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0086 + TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA = 0x0087 + TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0088 + TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA = 0x0089 + TLS_PSK_WITH_RC4_128_SHA = 0x008A + TLS_PSK_WITH_3DES_EDE_CBC_SHA = 0x008B + TLS_PSK_WITH_AES_128_CBC_SHA = 0x008C + TLS_PSK_WITH_AES_256_CBC_SHA = 0x008D + TLS_DHE_PSK_WITH_RC4_128_SHA = 0x008E + TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA = 0x008F + TLS_DHE_PSK_WITH_AES_128_CBC_SHA = 0x0090 + TLS_DHE_PSK_WITH_AES_256_CBC_SHA = 0x0091 + TLS_RSA_PSK_WITH_RC4_128_SHA = 0x0092 + TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA = 0x0093 + TLS_RSA_PSK_WITH_AES_128_CBC_SHA = 0x0094 + TLS_RSA_PSK_WITH_AES_256_CBC_SHA = 0x0095 + TLS_RSA_WITH_SEED_CBC_SHA = 0x0096 + TLS_DH_DSS_WITH_SEED_CBC_SHA = 0x0097 + TLS_DH_RSA_WITH_SEED_CBC_SHA = 0x0098 + TLS_DHE_DSS_WITH_SEED_CBC_SHA = 0x0099 + TLS_DHE_RSA_WITH_SEED_CBC_SHA = 0x009A + TLS_DH_anon_WITH_SEED_CBC_SHA = 0x009B + TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C + TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D + TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E + TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F + TLS_DH_RSA_WITH_AES_128_GCM_SHA256 = 0x00A0 + TLS_DH_RSA_WITH_AES_256_GCM_SHA384 = 0x00A1 + TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 = 0x00A2 + TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 = 0x00A3 + TLS_DH_DSS_WITH_AES_128_GCM_SHA256 = 0x00A4 + TLS_DH_DSS_WITH_AES_256_GCM_SHA384 = 0x00A5 + TLS_DH_anon_WITH_AES_128_GCM_SHA256 = 0x00A6 + TLS_DH_anon_WITH_AES_256_GCM_SHA384 = 0x00A7 + TLS_PSK_WITH_AES_128_GCM_SHA256 = 0x00A8 + TLS_PSK_WITH_AES_256_GCM_SHA384 = 0x00A9 + TLS_DHE_PSK_WITH_AES_128_GCM_SHA256 = 0x00AA + TLS_DHE_PSK_WITH_AES_256_GCM_SHA384 = 0x00AB + TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 = 0x00AC + TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 = 0x00AD + TLS_PSK_WITH_AES_128_CBC_SHA256 = 0x00AE + TLS_PSK_WITH_AES_256_CBC_SHA384 = 0x00AF + TLS_PSK_WITH_NULL_SHA256 = 0x00B0 + TLS_PSK_WITH_NULL_SHA384 = 0x00B1 + TLS_DHE_PSK_WITH_AES_128_CBC_SHA256 = 0x00B2 + TLS_DHE_PSK_WITH_AES_256_CBC_SHA384 = 0x00B3 + TLS_DHE_PSK_WITH_NULL_SHA256 = 0x00B4 + TLS_DHE_PSK_WITH_NULL_SHA384 = 0x00B5 + TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 = 0x00B6 + TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 = 0x00B7 + TLS_RSA_PSK_WITH_NULL_SHA256 = 0x00B8 + TLS_RSA_PSK_WITH_NULL_SHA384 = 0x00B9 + TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BA + TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BB + TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BC + TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BD + TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BE + TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BF + TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C0 + TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C1 + TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C2 + TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C3 + TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C4 + TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C5 + TLS_EMPTY_RENEGOTIATION_INFO_SCSV = 0x00FF + TLS_ECDH_ECDSA_WITH_NULL_SHA = 0xC001 + TLS_ECDH_ECDSA_WITH_RC4_128_SHA = 0xC002 + TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA = 0xC003 + TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA = 0xC004 + TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA = 0xC005 + TLS_ECDHE_ECDSA_WITH_NULL_SHA = 0xC006 + TLS_ECDHE_ECDSA_WITH_RC4_128_SHA = 0xC007 + TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA = 0xC008 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009 + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A + TLS_ECDH_RSA_WITH_NULL_SHA = 0xC00B + TLS_ECDH_RSA_WITH_RC4_128_SHA = 0xC00C + TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA = 0xC00D + TLS_ECDH_RSA_WITH_AES_128_CBC_SHA = 0xC00E + TLS_ECDH_RSA_WITH_AES_256_CBC_SHA = 0xC00F + TLS_ECDHE_RSA_WITH_NULL_SHA = 0xC010 + TLS_ECDHE_RSA_WITH_RC4_128_SHA = 0xC011 + TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA = 0xC012 + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013 + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014 + TLS_ECDH_anon_WITH_NULL_SHA = 0xC015 + TLS_ECDH_anon_WITH_RC4_128_SHA = 0xC016 + TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA = 0xC017 + TLS_ECDH_anon_WITH_AES_128_CBC_SHA = 0xC018 + TLS_ECDH_anon_WITH_AES_256_CBC_SHA = 0xC019 + TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA = 0xC01A + TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA = 0xC01B + TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA = 0xC01C + TLS_SRP_SHA_WITH_AES_128_CBC_SHA = 0xC01D + TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA = 0xC01E + TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA = 0xC01F + TLS_SRP_SHA_WITH_AES_256_CBC_SHA = 0xC020 + TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA = 0xC021 + TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA = 0xC022 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023 + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024 + TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC025 + TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC026 + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027 + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028 + TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 = 0xC029 + TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 = 0xC02A + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B + TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C + TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02D + TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02E + TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F + TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030 + TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 = 0xC031 + TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 = 0xC032 + TLS_ECDHE_PSK_WITH_RC4_128_SHA = 0xC033 + TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA = 0xC034 + TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA = 0xC035 + TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA = 0xC036 + TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 = 0xC037 + TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 = 0xC038 + TLS_ECDHE_PSK_WITH_NULL_SHA = 0xC039 + TLS_ECDHE_PSK_WITH_NULL_SHA256 = 0xC03A + TLS_ECDHE_PSK_WITH_NULL_SHA384 = 0xC03B + TLS_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC03C + TLS_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC03D + TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256 = 0xC03E + TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384 = 0xC03F + TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC040 + TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC041 + TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256 = 0xC042 + TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384 = 0xC043 + TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC044 + TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC045 + TLS_DH_anon_WITH_ARIA_128_CBC_SHA256 = 0xC046 + TLS_DH_anon_WITH_ARIA_256_CBC_SHA384 = 0xC047 + TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256 = 0xC048 + TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384 = 0xC049 + TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256 = 0xC04A + TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384 = 0xC04B + TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC04C + TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC04D + TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC04E + TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC04F + TLS_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC050 + TLS_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC051 + TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC052 + TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC053 + TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC054 + TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC055 + TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256 = 0xC056 + TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384 = 0xC057 + TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256 = 0xC058 + TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384 = 0xC059 + TLS_DH_anon_WITH_ARIA_128_GCM_SHA256 = 0xC05A + TLS_DH_anon_WITH_ARIA_256_GCM_SHA384 = 0xC05B + TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256 = 0xC05C + TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384 = 0xC05D + TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 = 0xC05E + TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 = 0xC05F + TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC060 + TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC061 + TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC062 + TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC063 + TLS_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC064 + TLS_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC065 + TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC066 + TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC067 + TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC068 + TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC069 + TLS_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06A + TLS_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06B + TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06C + TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06D + TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06E + TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06F + TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC070 + TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC071 + TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC072 + TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC073 + TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC074 + TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC075 + TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC076 + TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC077 + TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC078 + TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC079 + TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07A + TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07B + TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07C + TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07D + TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07E + TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07F + TLS_DHE_DSS_WITH_CAMELLIA_128_GCM_SHA256 = 0xC080 + TLS_DHE_DSS_WITH_CAMELLIA_256_GCM_SHA384 = 0xC081 + TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256 = 0xC082 + TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384 = 0xC083 + TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256 = 0xC084 + TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384 = 0xC085 + TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC086 + TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC087 + TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC088 + TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC089 + TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08A + TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08B + TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08C + TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08D + TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08E + TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08F + TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC090 + TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC091 + TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC092 + TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC093 + TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC094 + TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC095 + TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC096 + TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC097 + TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC098 + TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC099 + TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC09A + TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC09B + TLS_RSA_WITH_AES_128_CCM = 0xC09C + TLS_RSA_WITH_AES_256_CCM = 0xC09D + TLS_DHE_RSA_WITH_AES_128_CCM = 0xC09E + TLS_DHE_RSA_WITH_AES_256_CCM = 0xC09F + TLS_RSA_WITH_AES_128_CCM_8 = 0xC0A0 + TLS_RSA_WITH_AES_256_CCM_8 = 0xC0A1 + TLS_DHE_RSA_WITH_AES_128_CCM_8 = 0xC0A2 + TLS_DHE_RSA_WITH_AES_256_CCM_8 = 0xC0A3 + TLS_PSK_WITH_AES_128_CCM = 0xC0A4 + TLS_PSK_WITH_AES_256_CCM = 0xC0A5 + TLS_DHE_PSK_WITH_AES_128_CCM = 0xC0A6 + TLS_DHE_PSK_WITH_AES_256_CCM = 0xC0A7 + TLS_PSK_WITH_AES_128_CCM_8 = 0xC0A8 + TLS_PSK_WITH_AES_256_CCM_8 = 0xC0A9 + TLS_PSK_DHE_WITH_AES_128_CCM_8 = 0xC0AA + TLS_PSK_DHE_WITH_AES_256_CCM_8 = 0xC0AB + TLS_ECDHE_ECDSA_WITH_AES_128_CCM = 0xC0AC + TLS_ECDHE_ECDSA_WITH_AES_256_CCM = 0xC0AD + TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 = 0xC0AE + TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8 = 0xC0AF + TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCC14 + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCC13 + + +def select_preferred_ciphersuite(client_supported, server_supported): + for i in server_supported: + assert isinstance(i, CipherSuites) + if i in client_supported: + return i + + raise UnsupportedCipherException( + "Client supported ciphersuites are not supported on the server." + ) diff --git a/libmproxy/contrib/tls/exceptions.py b/libmproxy/contrib/tls/exceptions.py new file mode 100644 index 00000000..75b34d11 --- /dev/null +++ b/libmproxy/contrib/tls/exceptions.py @@ -0,0 +1,2 @@ +class UnsupportedCipherException(Exception): + pass diff --git a/libmproxy/contrib/tls/hello_message.py b/libmproxy/contrib/tls/hello_message.py new file mode 100644 index 00000000..23cd872b --- /dev/null +++ b/libmproxy/contrib/tls/hello_message.py @@ -0,0 +1,178 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +from characteristic import attributes + +from construct import Container + +from six import BytesIO + +from . import _constructs + + +@attributes(['major', 'minor']) +class ProtocolVersion(object): + """ + An object representing a ProtocolVersion struct. + """ + + +@attributes(['gmt_unix_time', 'random_bytes']) +class Random(object): + """ + An object representing a Random struct. + """ + + +@attributes(['type', 'data']) +class Extension(object): + """ + An object representing an Extension struct. + """ + def as_bytes(self): + return _constructs.Extension.build(Container( + type=self.type.value, length=len(self.data), data=self.data)) + + +@attributes(['client_version', 'random', 'session_id', 'cipher_suites', + 'compression_methods', 'extensions']) +class ClientHello(object): + """ + An object representing a ClientHello message. + """ + def as_bytes(self): + return _constructs.ClientHello.build( + Container( + version=Container(major=self.client_version.major, + minor=self.client_version.minor), + random=Container( + gmt_unix_time=self.random.gmt_unix_time, + random_bytes=self.random.random_bytes + ), + session_id=Container(length=len(self.session_id), + session_id=self.session_id), + cipher_suites=Container(length=len(self.cipher_suites) * 2, + cipher_suites=self.cipher_suites), + compression_methods=Container( + length=len(self.compression_methods), + compression_methods=self.compression_methods + ), + extensions_length=sum([2 + 2 + len(ext.data) + for ext in self.extensions]), + extensions_bytes=b''.join( + [ext.as_bytes() for ext in self.extensions] + ) + ) + ) + + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``ClientHello`` struct. + + :param bytes: the bytes representing the input. + :return: ClientHello object. + """ + construct = _constructs.ClientHello.parse(bytes) + # XXX Is there a better way in Construct to parse an array of + # variable-length structs? + extensions = [] + extensions_io = BytesIO(construct.extensions_bytes) + while extensions_io.tell() < construct.extensions_length: + extension_construct = _constructs.Extension.parse_stream( + extensions_io) + extensions.append( + Extension(type=ExtensionType(extension_construct.type), + data=extension_construct.data)) + return ClientHello( + client_version=ProtocolVersion( + major=construct.version.major, + minor=construct.version.minor, + ), + random=Random( + gmt_unix_time=construct.random.gmt_unix_time, + random_bytes=construct.random.random_bytes, + ), + session_id=construct.session_id.session_id, + # TODO: cipher suites should be enums + cipher_suites=construct.cipher_suites.cipher_suites, + compression_methods=( + construct.compression_methods.compression_methods + ), + extensions=extensions, + ) + + +class ExtensionType(Enum): + SIGNATURE_ALGORITHMS = 13 + # XXX: See http://tools.ietf.org/html/rfc5246#ref-TLSEXT + + +@attributes(['server_version', 'random', 'session_id', 'cipher_suite', + 'compression_method', 'extensions']) +class ServerHello(object): + """ + An object representing a ServerHello message. + """ + def as_bytes(self): + return _constructs.ServerHello.build( + Container( + version=Container(major=self.server_version.major, + minor=self.server_version.minor), + random=Container( + gmt_unix_time=self.random.gmt_unix_time, + random_bytes=self.random.random_bytes + ), + session_id=Container(length=len(self.session_id), + session_id=self.session_id), + cipher_suite=self.cipher_suite, + compression_method=self.compression_method.value, + extensions_length=sum([2 + 2 + len(ext.data) + for ext in self.extensions]), + extensions_bytes=b''.join( + [ext.as_bytes() for ext in self.extensions] + ) + ) + ) + + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``ServerHello`` struct. + + :param bytes: the bytes representing the input. + :return: ServerHello object. + """ + construct = _constructs.ServerHello.parse(bytes) + # XXX: Find a better way to parse extensions + extensions = [] + extensions_io = BytesIO(construct.extensions_bytes) + while extensions_io.tell() < construct.extensions_length: + extension_construct = _constructs.Extension.parse_stream( + extensions_io) + extensions.append( + Extension(type=ExtensionType(extension_construct.type), + data=extension_construct.data)) + return ServerHello( + server_version=ProtocolVersion( + major=construct.version.major, + minor=construct.version.minor, + ), + random=Random( + gmt_unix_time=construct.random.gmt_unix_time, + random_bytes=construct.random.random_bytes, + ), + session_id=construct.session_id.session_id, + cipher_suite=construct.cipher_suite, + compression_method=CompressionMethod(construct.compression_method), + extensions=extensions, + ) + + +class CompressionMethod(Enum): + NULL = 0 diff --git a/libmproxy/contrib/tls/message.py b/libmproxy/contrib/tls/message.py new file mode 100644 index 00000000..b372859f --- /dev/null +++ b/libmproxy/contrib/tls/message.py @@ -0,0 +1,313 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +from characteristic import attributes + +from construct import Container + +from six import BytesIO + +from . import _constructs + +from .hello_message import ( + ClientHello, ProtocolVersion, ServerHello +) + + +class ClientCertificateType(Enum): + RSA_SIGN = 1 + DSS_SIGN = 2 + RSA_FIXED_DH = 3 + DSS_FIXED_DH = 4 + RSA_EPHEMERAL_DH_RESERVED = 5 + DSS_EPHEMERAL_DH_RESERVED = 6 + FORTEZZA_DMS_RESERVED = 20 + + +class HashAlgorithm(Enum): + NONE = 0 + MD5 = 1 + SHA1 = 2 + SHA224 = 3 + SHA256 = 4 + SHA384 = 5 + SHA512 = 6 + + +class SignatureAlgorithm(Enum): + ANONYMOUS = 0 + RSA = 1 + DSA = 2 + ECDSA = 3 + + +class HandshakeType(Enum): + HELLO_REQUEST = 0 + CLIENT_HELLO = 1 + SERVER_HELLO = 2 + CERTIFICATE = 11 + SERVER_KEY_EXCHANGE = 12 + CERTIFICATE_REQUEST = 13 + SERVER_HELLO_DONE = 14 + CERTIFICATE_VERIFY = 15 + CLIENT_KEY_EXCHANGE = 16 + FINISHED = 20 + + +class HelloRequest(object): + """ + An object representing a HelloRequest struct. + """ + def as_bytes(self): + return b'' + + +class ServerHelloDone(object): + """ + An object representing a ServerHelloDone struct. + """ + def as_bytes(self): + return b'' + + +@attributes(['certificate_types', 'supported_signature_algorithms', + 'certificate_authorities']) +class CertificateRequest(object): + """ + An object representing a CertificateRequest struct. + """ + def as_bytes(self): + return _constructs.CertificateRequest.build(Container( + certificate_types=Container( + length=len(self.certificate_types), + certificate_types=[cert_type.value + for cert_type in self.certificate_types] + ), + supported_signature_algorithms=Container( + supported_signature_algorithms_length=2 * len( + self.supported_signature_algorithms + ), + algorithms=[Container( + hash=algorithm.hash.value, + signature=algorithm.signature.value, + ) + for algorithm in self.supported_signature_algorithms + ] + ), + certificate_authorities=Container( + length=len(self.certificate_authorities), + certificate_authorities=self.certificate_authorities + ) + )) + + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``CertificateRequest`` struct. + + :param bytes: the bytes representing the input. + :return: CertificateRequest object. + """ + construct = _constructs.CertificateRequest.parse(bytes) + return cls( + certificate_types=[ + ClientCertificateType(cert_type) + for cert_type in construct.certificate_types.certificate_types + ], + supported_signature_algorithms=[ + SignatureAndHashAlgorithm( + hash=HashAlgorithm(algorithm.hash), + signature=SignatureAlgorithm(algorithm.signature), + ) + for algorithm in ( + construct.supported_signature_algorithms.algorithms + ) + ], + certificate_authorities=( + construct.certificate_authorities.certificate_authorities + ) + ) + + +@attributes(['hash', 'signature']) +class SignatureAndHashAlgorithm(object): + """ + An object representing a SignatureAndHashAlgorithm struct. + """ + + +@attributes(['dh_p', 'dh_g', 'dh_Ys']) +class ServerDHParams(object): + """ + An object representing a ServerDHParams struct. + """ + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``ServerDHParams`` struct. + + :param bytes: the bytes representing the input. + :return: ServerDHParams object. + """ + construct = _constructs.ServerDHParams.parse(bytes) + return cls( + dh_p=construct.dh_p, + dh_g=construct.dh_g, + dh_Ys=construct.dh_Ys + ) + + +@attributes(['client_version', 'random']) +class PreMasterSecret(object): + """ + An object representing a PreMasterSecret struct. + """ + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``PreMasterSecret`` struct. + + :param bytes: the bytes representing the input. + :return: CertificateRequest object. + """ + construct = _constructs.PreMasterSecret.parse(bytes) + return cls( + client_version=ProtocolVersion( + major=construct.version.major, + minor=construct.version.minor, + ), + random=construct.random_bytes, + ) + + +@attributes(['asn1_cert']) +class ASN1Cert(object): + """ + An object representing ASN.1 Certificate + """ + def as_bytes(self): + return _constructs.ASN1Cert.build(Container( + length=len(self.asn1_cert), + asn1_cert=self.asn1_cert + )) + + +@attributes(['certificate_list']) +class Certificate(object): + """ + An object representing a Certificate struct. + """ + def as_bytes(self): + return _constructs.Certificate.build(Container( + certificates_length=sum([4 + len(asn1cert.asn1_cert) + for asn1cert in self.certificate_list]), + certificates_bytes=b''.join( + [asn1cert.as_bytes() for asn1cert in self.certificate_list] + ) + + )) + + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``Certificate`` struct. + + :param bytes: the bytes representing the input. + :return: Certificate object. + """ + construct = _constructs.Certificate.parse(bytes) + # XXX: Find a better way to parse an array of variable-length objects + certificates = [] + certificates_io = BytesIO(construct.certificates_bytes) + + while certificates_io.tell() < construct.certificates_length: + certificate_construct = _constructs.ASN1Cert.parse_stream( + certificates_io + ) + certificates.append( + ASN1Cert(asn1_cert=certificate_construct.asn1_cert) + ) + return cls( + certificate_list=certificates + ) + + +@attributes(['verify_data']) +class Finished(object): + def as_bytes(self): + return self.verify_data + + +@attributes(['msg_type', 'length', 'body']) +class Handshake(object): + """ + An object representing a Handshake struct. + """ + def as_bytes(self): + if self.msg_type in [ + HandshakeType.SERVER_HELLO, HandshakeType.CLIENT_HELLO, + HandshakeType.CERTIFICATE, HandshakeType.CERTIFICATE_REQUEST, + HandshakeType.HELLO_REQUEST, HandshakeType.SERVER_HELLO_DONE, + HandshakeType.FINISHED + ]: + _body_as_bytes = self.body.as_bytes() + else: + _body_as_bytes = b'' + return _constructs.Handshake.build( + Container( + msg_type=self.msg_type.value, + length=self.length, + body=_body_as_bytes + ) + ) + + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``Handshake`` struct. + + :param bytes: the bytes representing the input. + :return: Handshake object. + """ + construct = _constructs.Handshake.parse(bytes) + return cls( + msg_type=HandshakeType(construct.msg_type), + length=construct.length, + body=cls._get_handshake_message( + HandshakeType(construct.msg_type), construct.body + ), + ) + + @staticmethod + def _get_handshake_message(msg_type, body): + _handshake_message_parser = { + HandshakeType.CLIENT_HELLO: ClientHello.from_bytes, + HandshakeType.SERVER_HELLO: ServerHello.from_bytes, + HandshakeType.CERTIFICATE: Certificate.from_bytes, + # 12: parse_server_key_exchange, + HandshakeType.CERTIFICATE_REQUEST: CertificateRequest.from_bytes, + # 15: parse_certificate_verify, + # 16: parse_client_key_exchange, + } + + try: + if msg_type == HandshakeType.HELLO_REQUEST: + return HelloRequest() + elif msg_type == HandshakeType.SERVER_HELLO_DONE: + return ServerHelloDone() + elif msg_type == HandshakeType.FINISHED: + return Finished(verify_data=body) + elif msg_type in [HandshakeType.SERVER_KEY_EXCHANGE, + HandshakeType.CERTIFICATE_VERIFY, + HandshakeType.CLIENT_KEY_EXCHANGE, + ]: + raise NotImplementedError + else: + return _handshake_message_parser[msg_type](body) + except NotImplementedError: + return None # TODO diff --git a/libmproxy/contrib/tls/record.py b/libmproxy/contrib/tls/record.py new file mode 100644 index 00000000..481c93bc --- /dev/null +++ b/libmproxy/contrib/tls/record.py @@ -0,0 +1,110 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +from characteristic import attributes + +from construct import Container + +from . import _constructs + + +@attributes(['major', 'minor']) +class ProtocolVersion(object): + """ + An object representing a ProtocolVersion struct. + """ + + +@attributes(['type', 'version', 'fragment']) +class TLSPlaintext(object): + """ + An object representing a TLSPlaintext struct. + """ + def as_bytes(self): + return _constructs.TLSPlaintext.build( + Container( + type=self.type.value, + version=Container(major=self.version.major, + minor=self.version.minor), + length=len(self.fragment), + fragment=self.fragment + ) + ) + + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``TLSPlaintext`` struct. + + :param bytes: the bytes representing the input. + :return: TLSPlaintext object. + """ + construct = _constructs.TLSPlaintext.parse(bytes) + return cls( + type=ContentType(construct.type), + version=ProtocolVersion( + major=construct.version.major, + minor=construct.version.minor + ), + fragment=construct.fragment + ) + + +@attributes(['type', 'version', 'fragment']) +class TLSCompressed(object): + """ + An object representing a TLSCompressed struct. + """ + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``TLSCompressed`` struct. + + :param bytes: the bytes representing the input. + :return: TLSCompressed object. + """ + construct = _constructs.TLSCompressed.parse(bytes) + return cls( + type=ContentType(construct.type), + version=ProtocolVersion( + major=construct.version.major, + minor=construct.version.minor + ), + fragment=construct.fragment + ) + + +@attributes(['type', 'version', 'fragment']) +class TLSCiphertext(object): + """ + An object representing a TLSCiphertext struct. + """ + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``TLSCiphertext`` struct. + + :param bytes: the bytes representing the input. + :return: TLSCiphertext object. + """ + construct = _constructs.TLSCiphertext.parse(bytes) + return cls( + type=ContentType(construct.type), + version=ProtocolVersion( + major=construct.version.major, + minor=construct.version.minor + ), + fragment=construct.fragment + ) + + +class ContentType(Enum): + CHANGE_CIPHER_SPEC = 20 + ALERT = 21 + HANDSHAKE = 22 + APPLICATION_DATA = 23 diff --git a/libmproxy/contrib/tls/utils.py b/libmproxy/contrib/tls/utils.py new file mode 100644 index 00000000..a971af49 --- /dev/null +++ b/libmproxy/contrib/tls/utils.py @@ -0,0 +1,52 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import construct + +import six + + +class _UBInt24(construct.Adapter): + def _encode(self, obj, context): + return ( + six.int2byte((obj & 0xFF0000) >> 16) + + six.int2byte((obj & 0x00FF00) >> 8) + + six.int2byte(obj & 0x0000FF) + ) + + def _decode(self, obj, context): + obj = bytearray(obj) + return (obj[0] << 16 | obj[1] << 8 | obj[2]) + + +def UBInt24(name): # noqa + return _UBInt24(construct.Bytes(name, 3)) + + +def LengthPrefixedArray(subcon, length_field=construct.UBInt8("length")): + """ + An array prefixed by a byte length field. + + In contrast to construct.macros.PrefixedArray, + the length field signifies the number of bytes, not the number of elements. + """ + subcon_with_pos = construct.Struct( + subcon.name, + construct.Embed(subcon), + construct.Anchor("__current_pos") + ) + + return construct.Embed( + construct.Struct( + "", + length_field, + construct.Anchor("__start_pos"), + construct.RepeatUntil( + lambda obj, ctx: obj.__current_pos == ctx.__start_pos + getattr(ctx, length_field.name), + subcon_with_pos + ), + ) + ) \ No newline at end of file diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 98c5d603..78372284 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -1,6 +1,6 @@ from __future__ import (absolute_import, print_function, division) -import traceback +from ..contrib.tls._constructs import ClientHello from netlib import tcp import netlib.http.http2 @@ -11,16 +11,16 @@ from .layer import Layer class TlsLayer(Layer): def __init__(self, ctx, client_tls, server_tls): + self.client_sni = None + self.client_alpn_protos = None + super(TlsLayer, self).__init__(ctx) self._client_tls = client_tls self._server_tls = server_tls - self.client_sni = None + self._sni_from_server_change = None - self.client_alpn_protos = None self.__server_tls_exception = None - # foo alpn protos = [netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1, netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], # TODO: read this from client_conn first - def __call__(self): """ The strategy for establishing SSL is as follows: @@ -45,6 +45,28 @@ class TlsLayer(Layer): https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 """ + import struct + + # Read all records that contain the initial Client Hello message. + client_hello = "" + client_hello_size = 1 + offset = 0 + while len(client_hello) < client_hello_size: + record_header = self.client_conn.rfile.peek(offset+5)[offset:] + record_size = struct.unpack("!H", record_header[3:])[0] + 5 + record_body = self.client_conn.rfile.peek(offset+record_size)[offset+5:] + client_hello += record_body + offset += record_size + client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 + + client_hello = ClientHello.parse(client_hello[4:]) + + for extension in client_hello.extensions: + if extension.type == 0x00: + host = extension.server_names[0].name + if extension.type == 0x10: + alpn = extension.alpn_protocols + client_tls_requires_server_cert = ( self._client_tls and self._server_tls and not self.config.no_upstream_cert ) @@ -60,12 +82,12 @@ class TlsLayer(Layer): def connect(self): if not self.server_conn: self.ctx.connect() - if self._server_tls and not self._server_tls_established: + if self._server_tls and not self.server_conn.tls_established: self._establish_tls_with_server() def reconnect(self): self.ctx.reconnect() - if self._server_tls and not self._server_tls_established: + if self._server_tls and not self.server_conn.tls_established: self._establish_tls_with_server() def set_server(self, address, server_tls, sni, depth=1): @@ -74,10 +96,6 @@ class TlsLayer(Layer): self._sni_from_server_change = sni self._server_tls = server_tls - @property - def _server_tls_established(self): - return self.server_conn and self.server_conn.tls_established - @property def sni_for_upstream_connection(self): if self._sni_from_server_change is False: @@ -138,6 +156,10 @@ class TlsLayer(Layer): connection.set_context(new_context) def __handle_alpn_select(self, conn_, options): + """ + Once the client signals the alternate protocols it supports, + we reconnect upstream with the same list and pass the server's choice down to the client. + """ # TODO: change to something meaningful? # alpn_preference = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 -- cgit v1.2.3 From 1093d185ec78cdfff4fb425b902a52f61991cf5e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Aug 2015 06:38:03 +0200 Subject: manually read tls clienthello --- libmproxy/contrib/tls/_constructs.py | 4 +- libmproxy/protocol2/root_context.py | 3 +- libmproxy/protocol2/tls.py | 157 ++++++++++++----------------------- 3 files changed, 57 insertions(+), 107 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/contrib/tls/_constructs.py b/libmproxy/contrib/tls/_constructs.py index 49661efb..a5f8b524 100644 --- a/libmproxy/contrib/tls/_constructs.py +++ b/libmproxy/contrib/tls/_constructs.py @@ -101,7 +101,7 @@ Extension = Struct( UBInt16("type"), Embed( Switch( - "data", lambda ctx: ctx.type, + "", lambda ctx: ctx.type, { 0x00: SNIExtension, 0x10: ALPNExtension @@ -202,7 +202,7 @@ Certificate = Struct( Handshake = Struct( "Handshake", UBInt8("msg_type"), - UBInt24("length"), # TODO: Reject packets with length > 2 ** 24 + UBInt24("length"), Bytes("body", lambda ctx: ctx.length), ) diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index d0c62be4..880bc160 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -47,9 +47,10 @@ class RootContext(object): d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) + alpn_proto_negotiated = top_layer.client_conn.get_alpn_proto_negotiated() is_alpn_h2_negotiated = ( isinstance(top_layer, TlsLayer) and - top_layer.client_conn.get_alpn_proto_negotiated() == HTTP2Protocol.ALPN_PROTO_H2 + alpn_proto_negotiated == HTTP2Protocol.ALPN_PROTO_H2 ) if is_tls_client_hello: diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 78372284..24b8989d 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -1,10 +1,12 @@ from __future__ import (absolute_import, print_function, division) -from ..contrib.tls._constructs import ClientHello +import struct +from construct import ConstructError from netlib import tcp import netlib.http.http2 +from ..contrib.tls._constructs import ClientHello from ..exceptions import ProtocolException from .layer import Layer @@ -12,14 +14,13 @@ from .layer import Layer class TlsLayer(Layer): def __init__(self, ctx, client_tls, server_tls): self.client_sni = None - self.client_alpn_protos = None + self.client_alpn_protocols = None super(TlsLayer, self).__init__(ctx) self._client_tls = client_tls self._server_tls = server_tls self._sni_from_server_change = None - self.__server_tls_exception = None def __call__(self): """ @@ -35,18 +36,31 @@ class TlsLayer(Layer): 3. Pause the client handshake, establish SSL with the server. 4. Finish the client handshake with the certificate from the server. There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( - Thus, we resort to the following workaround when establishing SSL with the server: - 1. Try to establish SSL with the server without SNI. If this fails, we ignore it. - 2. Establish SSL with client. - - If there's a SNI callback, reconnect to the server with SNI. - - If not and the server connect failed, raise the original exception. + Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values. + Further notes: - OpenSSL 1.0.2 introduces a callback that would help here: https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 """ - import struct + client_tls_requires_server_cert = ( + self._client_tls and self._server_tls and not self.config.no_upstream_cert + ) + + self._parse_client_hello() + + if client_tls_requires_server_cert: + self.ctx.connect() + self._establish_tls_with_server() + self._establish_tls_with_client() + elif self._client_tls: + self._establish_tls_with_client() + + layer = self.ctx.next_layer(self) + layer() + + def _get_client_hello(self): # Read all records that contain the initial Client Hello message. client_hello = "" client_hello_size = 1 @@ -58,26 +72,25 @@ class TlsLayer(Layer): client_hello += record_body offset += record_size client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 + return client_hello - client_hello = ClientHello.parse(client_hello[4:]) + def _parse_client_hello(self): + try: + client_hello = ClientHello.parse(self._get_client_hello()[4:]) + except ConstructError as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + return for extension in client_hello.extensions: if extension.type == 0x00: - host = extension.server_names[0].name - if extension.type == 0x10: - alpn = extension.alpn_protocols - - client_tls_requires_server_cert = ( - self._client_tls and self._server_tls and not self.config.no_upstream_cert - ) - - if client_tls_requires_server_cert: - self._establish_tls_with_client_and_server() - elif self._client_tls: - self._establish_tls_with_client() + if len(extension.server_names) != 1 or extension.server_names[0].type != 0: + self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") + self.client_sni = extension.server_names[0].name + elif extension.type == 0x10: + self.client_alpn_protocols = extension.alpn_protocols - layer = self.ctx.next_layer(self) - layer() + print("sni: %s" % self.client_sni) + print("alpn: %s" % self.client_alpn_protocols) def connect(self): if not self.server_conn: @@ -97,88 +110,31 @@ class TlsLayer(Layer): self._server_tls = server_tls @property - def sni_for_upstream_connection(self): + def sni_for_server_connection(self): if self._sni_from_server_change is False: return None else: return self._sni_from_server_change or self.client_sni - def _establish_tls_with_client_and_server(self): - """ - This function deals with the problem that the server may require a SNI value from the client. - """ - - # First, try to connect to the server. - self.ctx.connect() - server_err = None - try: - self._establish_tls_with_server() - except ProtocolException as e: - server_err = e - - self._establish_tls_with_client() - - if server_err and not self.client_sni: - raise server_err - - def __handle_sni(self, connection): - """ - This callback gets called during the TLS handshake with the client. - The client has just sent the Sever Name Indication (SNI). - """ - old_upstream_sni = self.sni_for_upstream_connection - - sn = connection.get_servername() - if not sn: - return - - self.client_sni = sn.decode("utf8").encode("idna") - - server_sni_changed = (old_upstream_sni != self.sni_for_upstream_connection) - server_conn_with_tls_exists = (self.server_conn and self._server_tls) - if server_sni_changed and server_conn_with_tls_exists: - try: - self.reconnect() - except Exception as e: - self.__server_tls_exception = e - - # Now, change client context to reflect possibly changed certificate: - cert, key, chain_file = self._find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file, - alpn_select_callback=self.__handle_alpn_select, - ) - connection.set_context(new_context) + @property + def alpn_for_client_connection(self): + return self.server_conn.get_alpn_proto_negotiated() - def __handle_alpn_select(self, conn_, options): + def __alpn_select_callback(self, conn_, options): """ Once the client signals the alternate protocols it supports, we reconnect upstream with the same list and pass the server's choice down to the client. """ - # TODO: change to something meaningful? - # alpn_preference = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 - alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 - - # TODO: Don't reconnect twice? - upstream_alpn_changed = (self.client_alpn_protos != options) - server_conn_with_tls_exists = (self.server_conn and self._server_tls) - if upstream_alpn_changed and server_conn_with_tls_exists: - try: - self.reconnect() - except Exception as e: - self.__server_tls_exception = e - self.client_alpn_protos = options + # This gets triggered if we haven't established an upstream connection yet. + default_alpn = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 + # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 - if alpn_preference in options: - return bytes(alpn_preference) - else: # pragma no cover - return options[0] + if self.alpn_for_client_connection in options: + return bytes(self.alpn_for_client_connection) + if default_alpn in options: + return bytes(default_alpn) + return options[0] def _establish_tls_with_client(self): self.log("Establish TLS with client", "debug") @@ -189,34 +145,27 @@ class TlsLayer(Layer): cert, key, method=self.config.openssl_method_client, options=self.config.openssl_options_client, - handle_sni=self.__handle_sni, cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, chain_file=chain_file, - alpn_select_callback=self.__handle_alpn_select, + alpn_select_callback=self.__alpn_select_callback, ) except tcp.NetLibError as e: - print("alpn: %s" % self.client_alpn_protos) raise ProtocolException(repr(e), e) - # Do not raise server tls exceptions immediately. - # We want to try to finish the client handshake so that other layers can send error messages over it. - if self.__server_tls_exception: - raise self.__server_tls_exception - def _establish_tls_with_server(self): self.log("Establish TLS with server", "debug") try: self.server_conn.establish_ssl( self.config.clientcerts, - self.sni_for_upstream_connection, + self.sni_for_server_connection, method=self.config.openssl_method_server, options=self.config.openssl_options_server, verify_options=self.config.openssl_verification_mode_server, ca_path=self.config.openssl_trusted_cadir_server, ca_pemfile=self.config.openssl_trusted_ca_server, cipher_list=self.config.ciphers_server, - alpn_protos=self.client_alpn_protos, + alpn_protos=self.client_alpn_protocols, ) tls_cert_err = self.server_conn.ssl_verification_error if tls_cert_err is not None: -- cgit v1.2.3 From aebe34202553bea24a5d4e99b9f218b58559c0f0 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Aug 2015 14:03:51 +0200 Subject: improve alpn handling --- libmproxy/protocol2/http.py | 9 ++++++--- libmproxy/protocol2/tls.py | 26 ++++++++++++++++++-------- 2 files changed, 24 insertions(+), 11 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 5a25c317..649e7843 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -78,13 +78,14 @@ class Http2Layer(Layer): body_size_limit=self.config.body_size_limit ) self._stream_id = request.stream_id + return request def read_from_server(self, request_method): return HTTPResponse.from_protocol( self.server_protocol, request_method, body_size_limit=self.config.body_size_limit, - include_body=False, + include_body=True, stream_id=self._stream_id ) @@ -389,9 +390,11 @@ class HttpLayer(Layer): if flow is None or flow == KILL: raise Kill() - if flow.response.stream: + if isinstance(self.ctx, Http2Layer): + pass # streaming is not implemented for http2 yet. + elif flow.response.stream: flow.response.content = CONTENT_MISSING - elif isinstance(self.server_protocol, http1.HTTP1Protocol): + else: flow.response.content = self.server_protocol.read_http_body( flow.response.headers, self.config.body_size_limit, diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 24b8989d..96ee643f 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -87,10 +87,9 @@ class TlsLayer(Layer): self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") self.client_sni = extension.server_names[0].name elif extension.type == 0x10: - self.client_alpn_protocols = extension.alpn_protocols + self.client_alpn_protocols = list(extension.alpn_protocols) - print("sni: %s" % self.client_sni) - print("alpn: %s" % self.client_alpn_protocols) + self.log("Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), "debug") def connect(self): if not self.server_conn: @@ -131,10 +130,13 @@ class TlsLayer(Layer): # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 if self.alpn_for_client_connection in options: - return bytes(self.alpn_for_client_connection) - if default_alpn in options: - return bytes(default_alpn) - return options[0] + choice = bytes(self.alpn_for_client_connection) + elif default_alpn in options: + choice = bytes(default_alpn) + else: + choice = options[0] + self.log("ALPN for client: %s" % choice, "debug") + return choice def _establish_tls_with_client(self): self.log("Establish TLS with client", "debug") @@ -156,6 +158,12 @@ class TlsLayer(Layer): def _establish_tls_with_server(self): self.log("Establish TLS with server", "debug") try: + # We only support http/1.1 and h2. + # If the server only supports spdy (next to http/1.1), it may select that + # and mitmproxy would enter TCP passthrough mode, which we want to avoid. + deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") + alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + self.server_conn.establish_ssl( self.config.clientcerts, self.sni_for_server_connection, @@ -165,7 +173,7 @@ class TlsLayer(Layer): ca_path=self.config.openssl_trusted_cadir_server, ca_pemfile=self.config.openssl_trusted_ca_server, cipher_list=self.config.ciphers_server, - alpn_protos=self.client_alpn_protocols, + alpn_protos=alpn, ) tls_cert_err = self.server_conn.ssl_verification_error if tls_cert_err is not None: @@ -185,6 +193,8 @@ class TlsLayer(Layer): except tcp.NetLibError as e: raise ProtocolException(repr(e), e) + self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") + def _find_cert(self): host = self.server_conn.address.host sans = set() -- cgit v1.2.3 From 778644d4b810e87ce20cf9da1dca55913c2ffd07 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Aug 2015 15:12:04 +0200 Subject: http2: fix bugs, chrome works :tada: --- libmproxy/protocol2/http.py | 2 +- libmproxy/protocol2/tls.py | 6 +++--- libmproxy/proxy/config.py | 5 ++++- 3 files changed, 8 insertions(+), 5 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 649e7843..e3878fa6 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -279,7 +279,7 @@ class HttpLayer(Layer): if isinstance(e, ProtocolException): raise e else: - raise ProtocolException(repr(e), e) + raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) finally: flow.live = False diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 96ee643f..ce684eb9 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -153,7 +153,7 @@ class TlsLayer(Layer): alpn_select_callback=self.__alpn_select_callback, ) except tcp.NetLibError as e: - raise ProtocolException(repr(e), e) + raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) def _establish_tls_with_server(self): self.log("Establish TLS with server", "debug") @@ -189,9 +189,9 @@ class TlsLayer(Layer): (tls_cert_err['depth'], tls_cert_err['errno']), "error") self.log("Aborting connection attempt", "error") - raise ProtocolException(repr(e), e) + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) except tcp.NetLibError as e: - raise ProtocolException(repr(e), e) + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index ec91a6e0..4ca15747 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -14,6 +14,9 @@ TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" +# We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. +# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old +DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" class HostMatcher(object): def __init__(self, patterns=[]): @@ -241,7 +244,7 @@ def ssl_option_group(parser): 'Can be passed multiple times.') group.add_argument( "--ciphers-client", action="store", - type=str, dest="ciphers_client", default=None, + type=str, dest="ciphers_client", default=DEFAULT_CLIENT_CIPHERS, help="Set supported ciphers for client connections. (OpenSSL Syntax)" ) group.add_argument( -- cgit v1.2.3 From 2cfc1b1b4030838f6047f18f8014c91926b414d0 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Aug 2015 20:48:59 +0200 Subject: fix non-alpn clients --- libmproxy/protocol2/tls.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index ce684eb9..7ef0ad8c 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -162,7 +162,10 @@ class TlsLayer(Layer): # If the server only supports spdy (next to http/1.1), it may select that # and mitmproxy would enter TCP passthrough mode, which we want to avoid. deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") - alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + if self.client_alpn_protocols: + alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + else: + alpn = None self.server_conn.establish_ssl( self.config.clientcerts, -- cgit v1.2.3 From 9c6b3eb58a22817daa576063c3626d7a239e7093 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Aug 2015 22:00:50 +0200 Subject: clean up clienthello parsing --- libmproxy/contrib/README | 4 + libmproxy/contrib/tls/_constructs.py | 6 +- libmproxy/contrib/tls/alert_message.py | 64 ------ libmproxy/contrib/tls/ciphersuites.py | 343 --------------------------------- libmproxy/contrib/tls/exceptions.py | 2 - libmproxy/contrib/tls/hello_message.py | 178 ----------------- libmproxy/contrib/tls/message.py | 313 ------------------------------ libmproxy/contrib/tls/record.py | 110 ----------- libmproxy/contrib/tls/utils.py | 26 --- libmproxy/protocol2/tls.py | 14 +- 10 files changed, 19 insertions(+), 1041 deletions(-) delete mode 100644 libmproxy/contrib/tls/alert_message.py delete mode 100644 libmproxy/contrib/tls/ciphersuites.py delete mode 100644 libmproxy/contrib/tls/exceptions.py delete mode 100644 libmproxy/contrib/tls/hello_message.py delete mode 100644 libmproxy/contrib/tls/message.py delete mode 100644 libmproxy/contrib/tls/record.py (limited to 'libmproxy') diff --git a/libmproxy/contrib/README b/libmproxy/contrib/README index 3b0f7512..e339310a 100644 --- a/libmproxy/contrib/README +++ b/libmproxy/contrib/README @@ -8,3 +8,7 @@ jsbeautifier, git checkout 25/03/12, MIT license wbxml - https://github.com/davidpshaw/PyWBXMLDecoder + +tls, BSD license + - https://github.com/mhils/tls/tree/extension-parsing + - limited to required files. \ No newline at end of file diff --git a/libmproxy/contrib/tls/_constructs.py b/libmproxy/contrib/tls/_constructs.py index a5f8b524..9c57a799 100644 --- a/libmproxy/contrib/tls/_constructs.py +++ b/libmproxy/contrib/tls/_constructs.py @@ -4,8 +4,8 @@ from __future__ import absolute_import, division, print_function -from construct import Array, Bytes, Struct, UBInt16, UBInt32, UBInt8, PascalString, Embed, \ - TunnelAdapter, GreedyRange, Switch +from construct import (Array, Bytes, Struct, UBInt16, UBInt32, UBInt8, PascalString, Embed, TunnelAdapter, GreedyRange, + Switch, OptionalGreedyRange) from .utils import UBInt24 @@ -113,7 +113,7 @@ Extension = Struct( extensions = TunnelAdapter( PascalString("extensions", length_field=UBInt16("extensions_length")), - GreedyRange(Extension) + OptionalGreedyRange(Extension) ) ClientHello = Struct( diff --git a/libmproxy/contrib/tls/alert_message.py b/libmproxy/contrib/tls/alert_message.py deleted file mode 100644 index ef02f56d..00000000 --- a/libmproxy/contrib/tls/alert_message.py +++ /dev/null @@ -1,64 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from __future__ import absolute_import, division, print_function - -from enum import Enum - -from characteristic import attributes - -from . import _constructs - - -class AlertLevel(Enum): - WARNING = 1 - FATAL = 2 - - -class AlertDescription(Enum): - CLOSE_NOTIFY = 0 - UNEXPECTED_MESSAGE = 10 - BAD_RECORD_MAC = 20 - DECRYPTION_FAILED_RESERVED = 21 - RECORD_OVERFLOW = 22 - DECOMPRESSION_FAILURE = 30 - HANDSHAKE_FAILURE = 40 - NO_CERTIFICATE_RESERVED = 41 - BAD_CERTIFICATE = 42 - UNSUPPORTED_CERTIFICATE = 43 - CERTIFICATE_REVOKED = 44 - CERTIFICATE_EXPIRED = 45 - CERTIFICATE_UNKNOWN = 46 - ILLEGAL_PARAMETER = 47 - UNKNOWN_CA = 48 - ACCESS_DENIED = 49 - DECODE_ERROR = 50 - DECRYPT_ERROR = 51 - EXPORT_RESTRICTION_RESERVED = 60 - PROTOCOL_VERSION = 70 - INSUFFICIENT_SECURITY = 71 - INTERNAL_ERROR = 80 - USER_CANCELED = 90 - NO_RENEGOTIATION = 100 - UNSUPPORTED_EXTENSION = 110 - - -@attributes(['level', 'description']) -class Alert(object): - """ - An object representing an Alert message. - """ - @classmethod - def from_bytes(cls, bytes): - """ - Parse an ``Alert`` struct. - - :param bytes: the bytes representing the input. - :return: Alert object. - """ - construct = _constructs.Alert.parse(bytes) - return cls( - level=AlertLevel(construct.level), - description=AlertDescription(construct.description) - ) diff --git a/libmproxy/contrib/tls/ciphersuites.py b/libmproxy/contrib/tls/ciphersuites.py deleted file mode 100644 index 86298f80..00000000 --- a/libmproxy/contrib/tls/ciphersuites.py +++ /dev/null @@ -1,343 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from __future__ import absolute_import, division, print_function - -from enum import Enum - -from .exceptions import UnsupportedCipherException - - -class CipherSuites(Enum): - TLS_NULL_WITH_NULL_NULL = 0x0000 - TLS_RSA_WITH_NULL_MD5 = 0x0001 - TLS_RSA_WITH_NULL_SHA = 0x0002 - TLS_RSA_EXPORT_WITH_RC4_40_MD5 = 0x0003 - TLS_RSA_WITH_RC4_128_MD5 = 0x0004 - TLS_RSA_WITH_RC4_128_SHA = 0x0005 - TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5 = 0x0006 - TLS_RSA_WITH_IDEA_CBC_SHA = 0x0007 - TLS_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0008 - TLS_RSA_WITH_DES_CBC_SHA = 0x0009 - TLS_RSA_WITH_3DES_EDE_CBC_SHA = 0x000A - TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x000B - TLS_DH_DSS_WITH_DES_CBC_SHA = 0x000C - TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA = 0x000D - TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x000E - TLS_DH_RSA_WITH_DES_CBC_SHA = 0x000F - TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA = 0x0010 - TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x0011 - TLS_DHE_DSS_WITH_DES_CBC_SHA = 0x0012 - TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA = 0x0013 - TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0014 - TLS_DHE_RSA_WITH_DES_CBC_SHA = 0x0015 - TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA = 0x0016 - TLS_DH_anon_EXPORT_WITH_RC4_40_MD5 = 0x0017 - TLS_DH_anon_WITH_RC4_128_MD5 = 0x0018 - TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA = 0x0019 - TLS_DH_anon_WITH_DES_CBC_SHA = 0x001A - TLS_DH_anon_WITH_3DES_EDE_CBC_SHA = 0x001B - TLS_KRB5_WITH_DES_CBC_SHA = 0x001E - TLS_KRB5_WITH_3DES_EDE_CBC_SHA = 0x001F - TLS_KRB5_WITH_RC4_128_SHA = 0x0020 - TLS_KRB5_WITH_IDEA_CBC_SHA = 0x0021 - TLS_KRB5_WITH_DES_CBC_MD5 = 0x0022 - TLS_KRB5_WITH_3DES_EDE_CBC_MD5 = 0x0023 - TLS_KRB5_WITH_RC4_128_MD5 = 0x0024 - TLS_KRB5_WITH_IDEA_CBC_MD5 = 0x0025 - TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA = 0x0026 - TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA = 0x0027 - TLS_KRB5_EXPORT_WITH_RC4_40_SHA = 0x0028 - TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5 = 0x0029 - TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5 = 0x002A - TLS_KRB5_EXPORT_WITH_RC4_40_MD5 = 0x002B - TLS_PSK_WITH_NULL_SHA = 0x002C - TLS_DHE_PSK_WITH_NULL_SHA = 0x002D - TLS_RSA_PSK_WITH_NULL_SHA = 0x002E - TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F - TLS_DH_DSS_WITH_AES_128_CBC_SHA = 0x0030 - TLS_DH_RSA_WITH_AES_128_CBC_SHA = 0x0031 - TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032 - TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033 - TLS_DH_anon_WITH_AES_128_CBC_SHA = 0x0034 - TLS_RSA_WITH_AES_256_CBC_SHA = 0x0035 - TLS_DH_DSS_WITH_AES_256_CBC_SHA = 0x0036 - TLS_DH_RSA_WITH_AES_256_CBC_SHA = 0x0037 - TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038 - TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039 - TLS_DH_anon_WITH_AES_256_CBC_SHA = 0x003A - TLS_RSA_WITH_NULL_SHA256 = 0x003B - TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x003C - TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D - TLS_DH_DSS_WITH_AES_128_CBC_SHA256 = 0x003E - TLS_DH_RSA_WITH_AES_128_CBC_SHA256 = 0x003F - TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040 - TLS_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0041 - TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA = 0x0042 - TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0043 - TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA = 0x0044 - TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0045 - TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA = 0x0046 - TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067 - TLS_DH_DSS_WITH_AES_256_CBC_SHA256 = 0x0068 - TLS_DH_RSA_WITH_AES_256_CBC_SHA256 = 0x0069 - TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006A - TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B - TLS_DH_anon_WITH_AES_128_CBC_SHA256 = 0x006C - TLS_DH_anon_WITH_AES_256_CBC_SHA256 = 0x006D - TLS_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0084 - TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA = 0x0085 - TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0086 - TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA = 0x0087 - TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0088 - TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA = 0x0089 - TLS_PSK_WITH_RC4_128_SHA = 0x008A - TLS_PSK_WITH_3DES_EDE_CBC_SHA = 0x008B - TLS_PSK_WITH_AES_128_CBC_SHA = 0x008C - TLS_PSK_WITH_AES_256_CBC_SHA = 0x008D - TLS_DHE_PSK_WITH_RC4_128_SHA = 0x008E - TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA = 0x008F - TLS_DHE_PSK_WITH_AES_128_CBC_SHA = 0x0090 - TLS_DHE_PSK_WITH_AES_256_CBC_SHA = 0x0091 - TLS_RSA_PSK_WITH_RC4_128_SHA = 0x0092 - TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA = 0x0093 - TLS_RSA_PSK_WITH_AES_128_CBC_SHA = 0x0094 - TLS_RSA_PSK_WITH_AES_256_CBC_SHA = 0x0095 - TLS_RSA_WITH_SEED_CBC_SHA = 0x0096 - TLS_DH_DSS_WITH_SEED_CBC_SHA = 0x0097 - TLS_DH_RSA_WITH_SEED_CBC_SHA = 0x0098 - TLS_DHE_DSS_WITH_SEED_CBC_SHA = 0x0099 - TLS_DHE_RSA_WITH_SEED_CBC_SHA = 0x009A - TLS_DH_anon_WITH_SEED_CBC_SHA = 0x009B - TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C - TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D - TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E - TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F - TLS_DH_RSA_WITH_AES_128_GCM_SHA256 = 0x00A0 - TLS_DH_RSA_WITH_AES_256_GCM_SHA384 = 0x00A1 - TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 = 0x00A2 - TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 = 0x00A3 - TLS_DH_DSS_WITH_AES_128_GCM_SHA256 = 0x00A4 - TLS_DH_DSS_WITH_AES_256_GCM_SHA384 = 0x00A5 - TLS_DH_anon_WITH_AES_128_GCM_SHA256 = 0x00A6 - TLS_DH_anon_WITH_AES_256_GCM_SHA384 = 0x00A7 - TLS_PSK_WITH_AES_128_GCM_SHA256 = 0x00A8 - TLS_PSK_WITH_AES_256_GCM_SHA384 = 0x00A9 - TLS_DHE_PSK_WITH_AES_128_GCM_SHA256 = 0x00AA - TLS_DHE_PSK_WITH_AES_256_GCM_SHA384 = 0x00AB - TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 = 0x00AC - TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 = 0x00AD - TLS_PSK_WITH_AES_128_CBC_SHA256 = 0x00AE - TLS_PSK_WITH_AES_256_CBC_SHA384 = 0x00AF - TLS_PSK_WITH_NULL_SHA256 = 0x00B0 - TLS_PSK_WITH_NULL_SHA384 = 0x00B1 - TLS_DHE_PSK_WITH_AES_128_CBC_SHA256 = 0x00B2 - TLS_DHE_PSK_WITH_AES_256_CBC_SHA384 = 0x00B3 - TLS_DHE_PSK_WITH_NULL_SHA256 = 0x00B4 - TLS_DHE_PSK_WITH_NULL_SHA384 = 0x00B5 - TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 = 0x00B6 - TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 = 0x00B7 - TLS_RSA_PSK_WITH_NULL_SHA256 = 0x00B8 - TLS_RSA_PSK_WITH_NULL_SHA384 = 0x00B9 - TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BA - TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BB - TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BC - TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BD - TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BE - TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BF - TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C0 - TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C1 - TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C2 - TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C3 - TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C4 - TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C5 - TLS_EMPTY_RENEGOTIATION_INFO_SCSV = 0x00FF - TLS_ECDH_ECDSA_WITH_NULL_SHA = 0xC001 - TLS_ECDH_ECDSA_WITH_RC4_128_SHA = 0xC002 - TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA = 0xC003 - TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA = 0xC004 - TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA = 0xC005 - TLS_ECDHE_ECDSA_WITH_NULL_SHA = 0xC006 - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA = 0xC007 - TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA = 0xC008 - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009 - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A - TLS_ECDH_RSA_WITH_NULL_SHA = 0xC00B - TLS_ECDH_RSA_WITH_RC4_128_SHA = 0xC00C - TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA = 0xC00D - TLS_ECDH_RSA_WITH_AES_128_CBC_SHA = 0xC00E - TLS_ECDH_RSA_WITH_AES_256_CBC_SHA = 0xC00F - TLS_ECDHE_RSA_WITH_NULL_SHA = 0xC010 - TLS_ECDHE_RSA_WITH_RC4_128_SHA = 0xC011 - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA = 0xC012 - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013 - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014 - TLS_ECDH_anon_WITH_NULL_SHA = 0xC015 - TLS_ECDH_anon_WITH_RC4_128_SHA = 0xC016 - TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA = 0xC017 - TLS_ECDH_anon_WITH_AES_128_CBC_SHA = 0xC018 - TLS_ECDH_anon_WITH_AES_256_CBC_SHA = 0xC019 - TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA = 0xC01A - TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA = 0xC01B - TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA = 0xC01C - TLS_SRP_SHA_WITH_AES_128_CBC_SHA = 0xC01D - TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA = 0xC01E - TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA = 0xC01F - TLS_SRP_SHA_WITH_AES_256_CBC_SHA = 0xC020 - TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA = 0xC021 - TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA = 0xC022 - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023 - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024 - TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC025 - TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC026 - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027 - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028 - TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 = 0xC029 - TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 = 0xC02A - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C - TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02D - TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02E - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030 - TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 = 0xC031 - TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 = 0xC032 - TLS_ECDHE_PSK_WITH_RC4_128_SHA = 0xC033 - TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA = 0xC034 - TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA = 0xC035 - TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA = 0xC036 - TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 = 0xC037 - TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 = 0xC038 - TLS_ECDHE_PSK_WITH_NULL_SHA = 0xC039 - TLS_ECDHE_PSK_WITH_NULL_SHA256 = 0xC03A - TLS_ECDHE_PSK_WITH_NULL_SHA384 = 0xC03B - TLS_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC03C - TLS_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC03D - TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256 = 0xC03E - TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384 = 0xC03F - TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC040 - TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC041 - TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256 = 0xC042 - TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384 = 0xC043 - TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC044 - TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC045 - TLS_DH_anon_WITH_ARIA_128_CBC_SHA256 = 0xC046 - TLS_DH_anon_WITH_ARIA_256_CBC_SHA384 = 0xC047 - TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256 = 0xC048 - TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384 = 0xC049 - TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256 = 0xC04A - TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384 = 0xC04B - TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC04C - TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC04D - TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC04E - TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC04F - TLS_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC050 - TLS_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC051 - TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC052 - TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC053 - TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC054 - TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC055 - TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256 = 0xC056 - TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384 = 0xC057 - TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256 = 0xC058 - TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384 = 0xC059 - TLS_DH_anon_WITH_ARIA_128_GCM_SHA256 = 0xC05A - TLS_DH_anon_WITH_ARIA_256_GCM_SHA384 = 0xC05B - TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256 = 0xC05C - TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384 = 0xC05D - TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 = 0xC05E - TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 = 0xC05F - TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC060 - TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC061 - TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC062 - TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC063 - TLS_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC064 - TLS_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC065 - TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC066 - TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC067 - TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC068 - TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC069 - TLS_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06A - TLS_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06B - TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06C - TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06D - TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06E - TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06F - TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC070 - TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC071 - TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC072 - TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC073 - TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC074 - TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC075 - TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC076 - TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC077 - TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC078 - TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC079 - TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07A - TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07B - TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07C - TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07D - TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07E - TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07F - TLS_DHE_DSS_WITH_CAMELLIA_128_GCM_SHA256 = 0xC080 - TLS_DHE_DSS_WITH_CAMELLIA_256_GCM_SHA384 = 0xC081 - TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256 = 0xC082 - TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384 = 0xC083 - TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256 = 0xC084 - TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384 = 0xC085 - TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC086 - TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC087 - TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC088 - TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC089 - TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08A - TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08B - TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08C - TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08D - TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08E - TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08F - TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC090 - TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC091 - TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC092 - TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC093 - TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC094 - TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC095 - TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC096 - TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC097 - TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC098 - TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC099 - TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC09A - TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC09B - TLS_RSA_WITH_AES_128_CCM = 0xC09C - TLS_RSA_WITH_AES_256_CCM = 0xC09D - TLS_DHE_RSA_WITH_AES_128_CCM = 0xC09E - TLS_DHE_RSA_WITH_AES_256_CCM = 0xC09F - TLS_RSA_WITH_AES_128_CCM_8 = 0xC0A0 - TLS_RSA_WITH_AES_256_CCM_8 = 0xC0A1 - TLS_DHE_RSA_WITH_AES_128_CCM_8 = 0xC0A2 - TLS_DHE_RSA_WITH_AES_256_CCM_8 = 0xC0A3 - TLS_PSK_WITH_AES_128_CCM = 0xC0A4 - TLS_PSK_WITH_AES_256_CCM = 0xC0A5 - TLS_DHE_PSK_WITH_AES_128_CCM = 0xC0A6 - TLS_DHE_PSK_WITH_AES_256_CCM = 0xC0A7 - TLS_PSK_WITH_AES_128_CCM_8 = 0xC0A8 - TLS_PSK_WITH_AES_256_CCM_8 = 0xC0A9 - TLS_PSK_DHE_WITH_AES_128_CCM_8 = 0xC0AA - TLS_PSK_DHE_WITH_AES_256_CCM_8 = 0xC0AB - TLS_ECDHE_ECDSA_WITH_AES_128_CCM = 0xC0AC - TLS_ECDHE_ECDSA_WITH_AES_256_CCM = 0xC0AD - TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 = 0xC0AE - TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8 = 0xC0AF - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCC14 - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCC13 - - -def select_preferred_ciphersuite(client_supported, server_supported): - for i in server_supported: - assert isinstance(i, CipherSuites) - if i in client_supported: - return i - - raise UnsupportedCipherException( - "Client supported ciphersuites are not supported on the server." - ) diff --git a/libmproxy/contrib/tls/exceptions.py b/libmproxy/contrib/tls/exceptions.py deleted file mode 100644 index 75b34d11..00000000 --- a/libmproxy/contrib/tls/exceptions.py +++ /dev/null @@ -1,2 +0,0 @@ -class UnsupportedCipherException(Exception): - pass diff --git a/libmproxy/contrib/tls/hello_message.py b/libmproxy/contrib/tls/hello_message.py deleted file mode 100644 index 23cd872b..00000000 --- a/libmproxy/contrib/tls/hello_message.py +++ /dev/null @@ -1,178 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from __future__ import absolute_import, division, print_function - -from enum import Enum - -from characteristic import attributes - -from construct import Container - -from six import BytesIO - -from . import _constructs - - -@attributes(['major', 'minor']) -class ProtocolVersion(object): - """ - An object representing a ProtocolVersion struct. - """ - - -@attributes(['gmt_unix_time', 'random_bytes']) -class Random(object): - """ - An object representing a Random struct. - """ - - -@attributes(['type', 'data']) -class Extension(object): - """ - An object representing an Extension struct. - """ - def as_bytes(self): - return _constructs.Extension.build(Container( - type=self.type.value, length=len(self.data), data=self.data)) - - -@attributes(['client_version', 'random', 'session_id', 'cipher_suites', - 'compression_methods', 'extensions']) -class ClientHello(object): - """ - An object representing a ClientHello message. - """ - def as_bytes(self): - return _constructs.ClientHello.build( - Container( - version=Container(major=self.client_version.major, - minor=self.client_version.minor), - random=Container( - gmt_unix_time=self.random.gmt_unix_time, - random_bytes=self.random.random_bytes - ), - session_id=Container(length=len(self.session_id), - session_id=self.session_id), - cipher_suites=Container(length=len(self.cipher_suites) * 2, - cipher_suites=self.cipher_suites), - compression_methods=Container( - length=len(self.compression_methods), - compression_methods=self.compression_methods - ), - extensions_length=sum([2 + 2 + len(ext.data) - for ext in self.extensions]), - extensions_bytes=b''.join( - [ext.as_bytes() for ext in self.extensions] - ) - ) - ) - - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``ClientHello`` struct. - - :param bytes: the bytes representing the input. - :return: ClientHello object. - """ - construct = _constructs.ClientHello.parse(bytes) - # XXX Is there a better way in Construct to parse an array of - # variable-length structs? - extensions = [] - extensions_io = BytesIO(construct.extensions_bytes) - while extensions_io.tell() < construct.extensions_length: - extension_construct = _constructs.Extension.parse_stream( - extensions_io) - extensions.append( - Extension(type=ExtensionType(extension_construct.type), - data=extension_construct.data)) - return ClientHello( - client_version=ProtocolVersion( - major=construct.version.major, - minor=construct.version.minor, - ), - random=Random( - gmt_unix_time=construct.random.gmt_unix_time, - random_bytes=construct.random.random_bytes, - ), - session_id=construct.session_id.session_id, - # TODO: cipher suites should be enums - cipher_suites=construct.cipher_suites.cipher_suites, - compression_methods=( - construct.compression_methods.compression_methods - ), - extensions=extensions, - ) - - -class ExtensionType(Enum): - SIGNATURE_ALGORITHMS = 13 - # XXX: See http://tools.ietf.org/html/rfc5246#ref-TLSEXT - - -@attributes(['server_version', 'random', 'session_id', 'cipher_suite', - 'compression_method', 'extensions']) -class ServerHello(object): - """ - An object representing a ServerHello message. - """ - def as_bytes(self): - return _constructs.ServerHello.build( - Container( - version=Container(major=self.server_version.major, - minor=self.server_version.minor), - random=Container( - gmt_unix_time=self.random.gmt_unix_time, - random_bytes=self.random.random_bytes - ), - session_id=Container(length=len(self.session_id), - session_id=self.session_id), - cipher_suite=self.cipher_suite, - compression_method=self.compression_method.value, - extensions_length=sum([2 + 2 + len(ext.data) - for ext in self.extensions]), - extensions_bytes=b''.join( - [ext.as_bytes() for ext in self.extensions] - ) - ) - ) - - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``ServerHello`` struct. - - :param bytes: the bytes representing the input. - :return: ServerHello object. - """ - construct = _constructs.ServerHello.parse(bytes) - # XXX: Find a better way to parse extensions - extensions = [] - extensions_io = BytesIO(construct.extensions_bytes) - while extensions_io.tell() < construct.extensions_length: - extension_construct = _constructs.Extension.parse_stream( - extensions_io) - extensions.append( - Extension(type=ExtensionType(extension_construct.type), - data=extension_construct.data)) - return ServerHello( - server_version=ProtocolVersion( - major=construct.version.major, - minor=construct.version.minor, - ), - random=Random( - gmt_unix_time=construct.random.gmt_unix_time, - random_bytes=construct.random.random_bytes, - ), - session_id=construct.session_id.session_id, - cipher_suite=construct.cipher_suite, - compression_method=CompressionMethod(construct.compression_method), - extensions=extensions, - ) - - -class CompressionMethod(Enum): - NULL = 0 diff --git a/libmproxy/contrib/tls/message.py b/libmproxy/contrib/tls/message.py deleted file mode 100644 index b372859f..00000000 --- a/libmproxy/contrib/tls/message.py +++ /dev/null @@ -1,313 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from __future__ import absolute_import, division, print_function - -from enum import Enum - -from characteristic import attributes - -from construct import Container - -from six import BytesIO - -from . import _constructs - -from .hello_message import ( - ClientHello, ProtocolVersion, ServerHello -) - - -class ClientCertificateType(Enum): - RSA_SIGN = 1 - DSS_SIGN = 2 - RSA_FIXED_DH = 3 - DSS_FIXED_DH = 4 - RSA_EPHEMERAL_DH_RESERVED = 5 - DSS_EPHEMERAL_DH_RESERVED = 6 - FORTEZZA_DMS_RESERVED = 20 - - -class HashAlgorithm(Enum): - NONE = 0 - MD5 = 1 - SHA1 = 2 - SHA224 = 3 - SHA256 = 4 - SHA384 = 5 - SHA512 = 6 - - -class SignatureAlgorithm(Enum): - ANONYMOUS = 0 - RSA = 1 - DSA = 2 - ECDSA = 3 - - -class HandshakeType(Enum): - HELLO_REQUEST = 0 - CLIENT_HELLO = 1 - SERVER_HELLO = 2 - CERTIFICATE = 11 - SERVER_KEY_EXCHANGE = 12 - CERTIFICATE_REQUEST = 13 - SERVER_HELLO_DONE = 14 - CERTIFICATE_VERIFY = 15 - CLIENT_KEY_EXCHANGE = 16 - FINISHED = 20 - - -class HelloRequest(object): - """ - An object representing a HelloRequest struct. - """ - def as_bytes(self): - return b'' - - -class ServerHelloDone(object): - """ - An object representing a ServerHelloDone struct. - """ - def as_bytes(self): - return b'' - - -@attributes(['certificate_types', 'supported_signature_algorithms', - 'certificate_authorities']) -class CertificateRequest(object): - """ - An object representing a CertificateRequest struct. - """ - def as_bytes(self): - return _constructs.CertificateRequest.build(Container( - certificate_types=Container( - length=len(self.certificate_types), - certificate_types=[cert_type.value - for cert_type in self.certificate_types] - ), - supported_signature_algorithms=Container( - supported_signature_algorithms_length=2 * len( - self.supported_signature_algorithms - ), - algorithms=[Container( - hash=algorithm.hash.value, - signature=algorithm.signature.value, - ) - for algorithm in self.supported_signature_algorithms - ] - ), - certificate_authorities=Container( - length=len(self.certificate_authorities), - certificate_authorities=self.certificate_authorities - ) - )) - - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``CertificateRequest`` struct. - - :param bytes: the bytes representing the input. - :return: CertificateRequest object. - """ - construct = _constructs.CertificateRequest.parse(bytes) - return cls( - certificate_types=[ - ClientCertificateType(cert_type) - for cert_type in construct.certificate_types.certificate_types - ], - supported_signature_algorithms=[ - SignatureAndHashAlgorithm( - hash=HashAlgorithm(algorithm.hash), - signature=SignatureAlgorithm(algorithm.signature), - ) - for algorithm in ( - construct.supported_signature_algorithms.algorithms - ) - ], - certificate_authorities=( - construct.certificate_authorities.certificate_authorities - ) - ) - - -@attributes(['hash', 'signature']) -class SignatureAndHashAlgorithm(object): - """ - An object representing a SignatureAndHashAlgorithm struct. - """ - - -@attributes(['dh_p', 'dh_g', 'dh_Ys']) -class ServerDHParams(object): - """ - An object representing a ServerDHParams struct. - """ - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``ServerDHParams`` struct. - - :param bytes: the bytes representing the input. - :return: ServerDHParams object. - """ - construct = _constructs.ServerDHParams.parse(bytes) - return cls( - dh_p=construct.dh_p, - dh_g=construct.dh_g, - dh_Ys=construct.dh_Ys - ) - - -@attributes(['client_version', 'random']) -class PreMasterSecret(object): - """ - An object representing a PreMasterSecret struct. - """ - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``PreMasterSecret`` struct. - - :param bytes: the bytes representing the input. - :return: CertificateRequest object. - """ - construct = _constructs.PreMasterSecret.parse(bytes) - return cls( - client_version=ProtocolVersion( - major=construct.version.major, - minor=construct.version.minor, - ), - random=construct.random_bytes, - ) - - -@attributes(['asn1_cert']) -class ASN1Cert(object): - """ - An object representing ASN.1 Certificate - """ - def as_bytes(self): - return _constructs.ASN1Cert.build(Container( - length=len(self.asn1_cert), - asn1_cert=self.asn1_cert - )) - - -@attributes(['certificate_list']) -class Certificate(object): - """ - An object representing a Certificate struct. - """ - def as_bytes(self): - return _constructs.Certificate.build(Container( - certificates_length=sum([4 + len(asn1cert.asn1_cert) - for asn1cert in self.certificate_list]), - certificates_bytes=b''.join( - [asn1cert.as_bytes() for asn1cert in self.certificate_list] - ) - - )) - - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``Certificate`` struct. - - :param bytes: the bytes representing the input. - :return: Certificate object. - """ - construct = _constructs.Certificate.parse(bytes) - # XXX: Find a better way to parse an array of variable-length objects - certificates = [] - certificates_io = BytesIO(construct.certificates_bytes) - - while certificates_io.tell() < construct.certificates_length: - certificate_construct = _constructs.ASN1Cert.parse_stream( - certificates_io - ) - certificates.append( - ASN1Cert(asn1_cert=certificate_construct.asn1_cert) - ) - return cls( - certificate_list=certificates - ) - - -@attributes(['verify_data']) -class Finished(object): - def as_bytes(self): - return self.verify_data - - -@attributes(['msg_type', 'length', 'body']) -class Handshake(object): - """ - An object representing a Handshake struct. - """ - def as_bytes(self): - if self.msg_type in [ - HandshakeType.SERVER_HELLO, HandshakeType.CLIENT_HELLO, - HandshakeType.CERTIFICATE, HandshakeType.CERTIFICATE_REQUEST, - HandshakeType.HELLO_REQUEST, HandshakeType.SERVER_HELLO_DONE, - HandshakeType.FINISHED - ]: - _body_as_bytes = self.body.as_bytes() - else: - _body_as_bytes = b'' - return _constructs.Handshake.build( - Container( - msg_type=self.msg_type.value, - length=self.length, - body=_body_as_bytes - ) - ) - - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``Handshake`` struct. - - :param bytes: the bytes representing the input. - :return: Handshake object. - """ - construct = _constructs.Handshake.parse(bytes) - return cls( - msg_type=HandshakeType(construct.msg_type), - length=construct.length, - body=cls._get_handshake_message( - HandshakeType(construct.msg_type), construct.body - ), - ) - - @staticmethod - def _get_handshake_message(msg_type, body): - _handshake_message_parser = { - HandshakeType.CLIENT_HELLO: ClientHello.from_bytes, - HandshakeType.SERVER_HELLO: ServerHello.from_bytes, - HandshakeType.CERTIFICATE: Certificate.from_bytes, - # 12: parse_server_key_exchange, - HandshakeType.CERTIFICATE_REQUEST: CertificateRequest.from_bytes, - # 15: parse_certificate_verify, - # 16: parse_client_key_exchange, - } - - try: - if msg_type == HandshakeType.HELLO_REQUEST: - return HelloRequest() - elif msg_type == HandshakeType.SERVER_HELLO_DONE: - return ServerHelloDone() - elif msg_type == HandshakeType.FINISHED: - return Finished(verify_data=body) - elif msg_type in [HandshakeType.SERVER_KEY_EXCHANGE, - HandshakeType.CERTIFICATE_VERIFY, - HandshakeType.CLIENT_KEY_EXCHANGE, - ]: - raise NotImplementedError - else: - return _handshake_message_parser[msg_type](body) - except NotImplementedError: - return None # TODO diff --git a/libmproxy/contrib/tls/record.py b/libmproxy/contrib/tls/record.py deleted file mode 100644 index 481c93bc..00000000 --- a/libmproxy/contrib/tls/record.py +++ /dev/null @@ -1,110 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from __future__ import absolute_import, division, print_function - -from enum import Enum - -from characteristic import attributes - -from construct import Container - -from . import _constructs - - -@attributes(['major', 'minor']) -class ProtocolVersion(object): - """ - An object representing a ProtocolVersion struct. - """ - - -@attributes(['type', 'version', 'fragment']) -class TLSPlaintext(object): - """ - An object representing a TLSPlaintext struct. - """ - def as_bytes(self): - return _constructs.TLSPlaintext.build( - Container( - type=self.type.value, - version=Container(major=self.version.major, - minor=self.version.minor), - length=len(self.fragment), - fragment=self.fragment - ) - ) - - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``TLSPlaintext`` struct. - - :param bytes: the bytes representing the input. - :return: TLSPlaintext object. - """ - construct = _constructs.TLSPlaintext.parse(bytes) - return cls( - type=ContentType(construct.type), - version=ProtocolVersion( - major=construct.version.major, - minor=construct.version.minor - ), - fragment=construct.fragment - ) - - -@attributes(['type', 'version', 'fragment']) -class TLSCompressed(object): - """ - An object representing a TLSCompressed struct. - """ - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``TLSCompressed`` struct. - - :param bytes: the bytes representing the input. - :return: TLSCompressed object. - """ - construct = _constructs.TLSCompressed.parse(bytes) - return cls( - type=ContentType(construct.type), - version=ProtocolVersion( - major=construct.version.major, - minor=construct.version.minor - ), - fragment=construct.fragment - ) - - -@attributes(['type', 'version', 'fragment']) -class TLSCiphertext(object): - """ - An object representing a TLSCiphertext struct. - """ - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``TLSCiphertext`` struct. - - :param bytes: the bytes representing the input. - :return: TLSCiphertext object. - """ - construct = _constructs.TLSCiphertext.parse(bytes) - return cls( - type=ContentType(construct.type), - version=ProtocolVersion( - major=construct.version.major, - minor=construct.version.minor - ), - fragment=construct.fragment - ) - - -class ContentType(Enum): - CHANGE_CIPHER_SPEC = 20 - ALERT = 21 - HANDSHAKE = 22 - APPLICATION_DATA = 23 diff --git a/libmproxy/contrib/tls/utils.py b/libmproxy/contrib/tls/utils.py index a971af49..4c917303 100644 --- a/libmproxy/contrib/tls/utils.py +++ b/libmproxy/contrib/tls/utils.py @@ -24,29 +24,3 @@ class _UBInt24(construct.Adapter): def UBInt24(name): # noqa return _UBInt24(construct.Bytes(name, 3)) - - -def LengthPrefixedArray(subcon, length_field=construct.UBInt8("length")): - """ - An array prefixed by a byte length field. - - In contrast to construct.macros.PrefixedArray, - the length field signifies the number of bytes, not the number of elements. - """ - subcon_with_pos = construct.Struct( - subcon.name, - construct.Embed(subcon), - construct.Anchor("__current_pos") - ) - - return construct.Embed( - construct.Struct( - "", - length_field, - construct.Anchor("__start_pos"), - construct.RepeatUntil( - lambda obj, ctx: obj.__current_pos == ctx.__start_pos + getattr(ctx, length_field.name), - subcon_with_pos - ), - ) - ) \ No newline at end of file diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 7ef0ad8c..9c8aeb24 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -61,7 +61,12 @@ class TlsLayer(Layer): layer() def _get_client_hello(self): - # Read all records that contain the initial Client Hello message. + """ + Peek into the socket and read all records that contain the initial client hello message. + + Returns: + The raw handshake packet bytes, without TLS record header(s). + """ client_hello = "" client_hello_size = 1 offset = 0 @@ -75,10 +80,15 @@ class TlsLayer(Layer): return client_hello def _parse_client_hello(self): + """ + Peek into the connection, read the initial client hello and parse it to obtain ALPN values. + """ + raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. try: - client_hello = ClientHello.parse(self._get_client_hello()[4:]) + client_hello = ClientHello.parse(raw_client_hello) except ConstructError as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") + self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") return for extension in client_hello.extensions: -- cgit v1.2.3 From f6dadc2b0de712869d9b8aa928915dbb990bb6af Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 00:07:44 +0200 Subject: no more sni double-connects! --- libmproxy/contrib/README | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/contrib/README b/libmproxy/contrib/README index e339310a..e5ce11da 100644 --- a/libmproxy/contrib/README +++ b/libmproxy/contrib/README @@ -10,5 +10,5 @@ wbxml - https://github.com/davidpshaw/PyWBXMLDecoder tls, BSD license - - https://github.com/mhils/tls/tree/extension-parsing + - https://github.com/mhils/tls/tree/mitmproxy - limited to required files. \ No newline at end of file -- cgit v1.2.3 From 0f97899fbd6cc68e974b1670e9c5188a28b52168 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 15:26:21 +0200 Subject: re-add --ignore and --tcp --- libmproxy/protocol2/root_context.py | 66 ++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 27 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 880bc160..78d48453 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -1,12 +1,12 @@ from __future__ import (absolute_import, print_function, division) -import string -from libmproxy.protocol2.layer import Kill +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol + from .rawtcp import RawTcpLayer from .tls import TlsLayer -from .http import Http1Layer, Http2Layer, HttpLayer +from .http import Http1Layer, Http2Layer -from netlib.http.http2 import HTTP2Protocol class RootContext(object): """ @@ -22,12 +22,19 @@ class RootContext(object): def next_layer(self, top_layer): """ This function determines the next layer in the protocol stack. - :param top_layer: the current top layer - :return: The next layer. + + Arguments: + top_layer: the current top layer. + + Returns: + The next layer """ - # TODO: Handle ignore and tcp passthrough + # 1. Check for --ignore. + if self.config.check_ignore(top_layer.server_conn.address): + return RawTcpLayer(top_layer) + # 2. Check for TLS # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello d = top_layer.client_conn.rfile.peek(3) @@ -37,30 +44,35 @@ class RootContext(object): d[1] == '\x03' and d[2] in ('\x00', '\x01', '\x02', '\x03') ) + if is_tls_client_hello: + return TlsLayer(top_layer, True, True) - d = top_layer.client_conn.rfile.peek(3) - is_ascii = ( - len(d) == 3 and - all(x in string.ascii_letters for x in d) # better be safe here and don't expect uppercase... - ) + # 3. Check for --tcp + if self.config.check_tcp(top_layer.server_conn.address): + return RawTcpLayer(top_layer) - d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) - is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) + # 4. Check for TLS ALPN (HTTP1/HTTP2) + if isinstance(top_layer, TlsLayer): + alpn = top_layer.client_conn.get_alpn_proto_negotiated() + if alpn == HTTP2Protocol.ALPN_PROTO_H2: + return Http2Layer(top_layer, 'transparent') + if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: + return Http1Layer(top_layer, 'transparent') - alpn_proto_negotiated = top_layer.client_conn.get_alpn_proto_negotiated() - is_alpn_h2_negotiated = ( - isinstance(top_layer, TlsLayer) and - alpn_proto_negotiated == HTTP2Protocol.ALPN_PROTO_H2 - ) + # 5. Assume HTTP1 by default + return Http1Layer(top_layer, 'transparent') - if is_tls_client_hello: - return TlsLayer(top_layer, True, True) - elif is_alpn_h2_negotiated or is_http2_magic: - return Http2Layer(top_layer, 'transparent') - elif is_ascii: - return Http1Layer(top_layer, 'transparent') - else: - return RawTcpLayer(top_layer) + # In a future version, we want to implement TCP passthrough as the last fallback, + # but we don't have the UI part ready for that. + # + # d = top_layer.client_conn.rfile.peek(3) + # is_ascii = ( + # len(d) == 3 and + # all(x in string.ascii_letters for x in d) # better be safe here and don't expect uppercase... + # ) + # # TODO: This could block if there are not enough bytes available? + # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) + # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) @property def layers(self): -- cgit v1.2.3 From ecfde4247fcfd8279948b4a22bc4f04c2fb2ba15 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 15:48:41 +0200 Subject: re-add http1 replay --- libmproxy/protocol2/http.py | 90 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index e3878fa6..32c0116b 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -1,14 +1,18 @@ from __future__ import (absolute_import, print_function, division) from .. import version +import threading from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer from libmproxy import utils +from libmproxy.controller import Channel from libmproxy.protocol2.layer import Kill -from libmproxy.protocol import KILL +from libmproxy.protocol import KILL, Error from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest +from libmproxy.proxy import Log +from libmproxy.proxy.connection import ServerConnection from netlib import tcp from netlib.http import status_codes, http1, http2, HttpErrorConnClosed, HttpError from netlib.http.semantics import CONTENT_MISSING @@ -509,3 +513,87 @@ class HttpLayer(Layer): odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()]) )) raise InvalidCredentials("Proxy Authentication Required") + + +class RequestReplayThread(threading.Thread): + name = "RequestReplayThread" + + def __init__(self, config, flow, masterq, should_exit): + """ + masterqueue can be a queue or None, if no scripthooks should be + processed. + """ + self.config, self.flow = config, flow + if masterq: + self.channel = Channel(masterq, should_exit) + else: + self.channel = None + super(RequestReplayThread, self).__init__() + + def run(self): + r = self.flow.request + form_out_backup = r.form_out + try: + self.flow.response = None + + # If we have a channel, run script hooks. + if self.channel: + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == KILL: + raise Kill() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply + + if not self.flow.response: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + # FIXME + server_address = self.config.mode.get_upstream_server( + self.flow.client_conn + )[2:] + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + connect_request = make_connect_request((r.host, r.port)) + server.send(protocol.assemble(connect_request)) + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + + server.send(protocol.assemble(r)) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_protocol( + protocol, + r.method, + body_size_limit=self.config.body_size_limit, + ) + if self.channel: + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == KILL: + raise Kill() + except (HttpError, tcp.NetLibError) as v: + self.flow.error = Error(repr(v)) + if self.channel: + self.channel.ask("error", self.flow) + except Kill: + # KillSignal should only be raised if there's a channel in the + # first place. + self.channel.tell("log", Log("Connection killed", "info")) + finally: + r.form_out = form_out_backup -- cgit v1.2.3 From 515c0244483446350779db59a31b8fd7dc603a5b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 15:59:56 +0200 Subject: handle tls server errors more gracefully --- libmproxy/protocol2/tls.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 9c8aeb24..433dd65d 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -51,9 +51,7 @@ class TlsLayer(Layer): self._parse_client_hello() if client_tls_requires_server_cert: - self.ctx.connect() - self._establish_tls_with_server() - self._establish_tls_with_client() + self._establish_tls_with_client_and_server() elif self._client_tls: self._establish_tls_with_client() @@ -148,6 +146,22 @@ class TlsLayer(Layer): self.log("ALPN for client: %s" % choice, "debug") return choice + def _establish_tls_with_client_and_server(self): + self.ctx.connect() + + # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless + # to send an error message over TLS. + try: + self._establish_tls_with_server() + except Exception as e: + try: + self._establish_tls_with_client() + except: + pass + raise e + + self._establish_tls_with_client() + def _establish_tls_with_client(self): self.log("Establish TLS with client", "debug") cert, key, chain_file = self._find_cert() -- cgit v1.2.3 From 83decd6771c430cca9e99ec0050442249d0aa99a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 17:35:53 +0200 Subject: fix inline script redirects --- libmproxy/flow.py | 3 ++- libmproxy/protocol2/http.py | 19 ++++++++++++------- libmproxy/protocol2/layer.py | 2 +- libmproxy/protocol2/tls.py | 4 ++-- 4 files changed, 17 insertions(+), 11 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 3d9ef722..8605d7a1 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,6 +8,7 @@ import Cookie import cookielib import os import re +from libmproxy.protocol2.http import RequestReplayThread from netlib import odict, wsgi, tcp from netlib.http.semantics import CONTENT_MISSING @@ -934,7 +935,7 @@ class FlowMaster(controller.Master): f.response = None f.error = None self.process_new_request(f) - rt = http.RequestReplayThread( + rt = RequestReplayThread( self.server.config, f, self.masterq if run_scripthooks else False, diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 32c0116b..792cf266 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -212,10 +212,11 @@ class UpstreamConnectLayer(Layer): self.ctx.reconnect() self.send_to_server(self.connect_request) - def set_server(self, address, server_tls, sni, depth=1): + def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: if self.ctx.server_conn: self.ctx.reconnect() + address = Address.wrap(address) self.connect_request.host = address.host self.connect_request.port = address.port self.server_conn.address = address @@ -227,11 +228,16 @@ class HttpLayer(Layer): def __init__(self, ctx, mode): super(HttpLayer, self).__init__(ctx) self.mode = mode + self.__original_server_conn = None + "Contains the original destination in transparent mode, which needs to be restored" + "if an inline script modified the target server for a single http request" def __call__(self): + if self.mode == "transparent": + self.__original_server_conn = self.server_conn while True: try: - flow = HTTPFlow(self.client_conn, self.server_conn, live=True) + flow = HTTPFlow(self.client_conn, self.server_conn, live=self) try: request = self.read_from_client() @@ -288,7 +294,7 @@ class HttpLayer(Layer): flow.live = False def handle_regular_mode_connect(self, request): - self.set_server((request.host, request.port), False, None) + self.set_server((request.host, request.port)) self.send_to_client(make_connect_response(request.httpversion)) layer = self.ctx.next_layer(self) layer() @@ -433,11 +439,10 @@ class HttpLayer(Layer): if flow.request.form_in == "authority": flow.request.scheme = "http" # pseudo value else: - flow.request.host = self.ctx.server_conn.address.host - flow.request.port = self.ctx.server_conn.address.port - flow.request.scheme = "https" if self.server_conn.tls_established else "http" + flow.request.host = self.__original_server_conn.address.host + flow.request.port = self.__original_server_conn.address.port + flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" - # TODO: Expose .set_server functionality to inline scripts request_reply = self.channel.ask("request", flow) if request_reply is None or request_reply == KILL: raise Kill() diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 7cb76591..f72320ff 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -112,7 +112,7 @@ class ServerConnectionMixin(object): self.server_conn.address = address self.connect() - def set_server(self, address, server_tls, sni, depth=1): + def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: if self.server_conn: self._disconnect() diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 433dd65d..b1b80034 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -110,9 +110,9 @@ class TlsLayer(Layer): if self._server_tls and not self.server_conn.tls_established: self._establish_tls_with_server() - def set_server(self, address, server_tls, sni, depth=1): + def set_server(self, address, server_tls=None, sni=None, depth=1): self.ctx.set_server(address, server_tls, sni, depth) - if server_tls is not None: + if depth == 1 and server_tls is not None: self._sni_from_server_change = sni self._server_tls = server_tls -- cgit v1.2.3 From 5b17496c7e5ea3c40a910c4973eeb7bfbcf065bd Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 18:31:15 +0200 Subject: start fixing proxy config --- libmproxy/proxy/config.py | 48 ++++++----------------------------------------- 1 file changed, 6 insertions(+), 42 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 4ca15747..83030235 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -3,14 +3,11 @@ import os import re from OpenSSL import SSL -import netlib -from netlib import http, certutils, tcp +from netlib import certutils, tcp from netlib.http import authentication -from .. import utils, platform, version -from .primitives import RegularProxyMode, SpoofMode, SSLSpoofMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode +from .. import utils, platform -TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" @@ -40,15 +37,12 @@ class ProxyConfig: self, host='', port=8080, - server_version=version.NAMEVERSION, cadir=CA_DIR, clientcerts=None, no_upstream_cert=False, body_size_limit=None, mode=None, upstream_server=None, - http_form_in=None, - http_form_out=None, authenticator=None, ignore_hosts=[], tcp_hosts=[], @@ -57,39 +51,19 @@ class ProxyConfig: certs=[], ssl_version_client=tcp.SSL_DEFAULT_METHOD, ssl_version_server=tcp.SSL_DEFAULT_METHOD, - ssl_ports=TRANSPARENT_SSL_PORTS, - spoofed_ssl_port=None, ssl_verify_upstream_cert=False, ssl_upstream_trusted_cadir=None, ssl_upstream_trusted_ca=None ): self.host = host self.port = port - self.server_version = server_version self.ciphers_client = ciphers_client self.ciphers_server = ciphers_server self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit - - if mode == "transparent": - self.mode = TransparentProxyMode(platform.resolver(), ssl_ports) - elif mode == "socks5": - self.mode = Socks5ProxyMode(ssl_ports) - elif mode == "reverse": - self.mode = ReverseProxyMode(upstream_server) - elif mode == "upstream": - self.mode = UpstreamProxyMode(upstream_server) - elif mode == "spoof": - self.mode = SpoofMode() - elif mode == "sslspoof": - self.mode = SSLSpoofMode(spoofed_ssl_port) - else: - self.mode = RegularProxyMode() - - # Handle manual overrides of the http forms - self.mode.http_form_in = http_form_in or self.mode.http_form_in - self.mode.http_form_out = http_form_out or self.mode.http_form_out + self.mode = mode + self.upstream_server = upstream_server self.check_ignore = HostMatcher(ignore_hosts) self.check_tcp = HostMatcher(tcp_hosts) @@ -97,10 +71,10 @@ class ProxyConfig: self.cadir = os.path.expanduser(cadir) self.certstore = certutils.CertStore.from_store( self.cadir, - CONF_BASENAME) + CONF_BASENAME + ) for spec, cert in certs: self.certstore.add_cert_file(spec, cert) - self.ssl_ports = ssl_ports if isinstance(ssl_version_client, int): self.openssl_method_client = ssl_version_client @@ -279,16 +253,6 @@ def ssl_option_group(parser): dest="ssl_upstream_trusted_ca", help="Path to a PEM formatted trusted CA certificate." ) - group.add_argument( - "--ssl-port", - action="append", - type=int, - dest="ssl_ports", - default=list(TRANSPARENT_SSL_PORTS), - metavar="PORT", - help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " - "Defaults to %s." % - str(TRANSPARENT_SSL_PORTS)) group.add_argument( "--ssl-version-client", dest="ssl_version_client", type=str, default=tcp.SSL_DEFAULT_VERSION, choices=tcp.SSL_VERSIONS.keys(), -- cgit v1.2.3 From a86491eeed13c7889356e5102312f52bd86c3c66 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 18:37:16 +0200 Subject: Revert "unify SSL version/method handling" This reverts commit 14e49f4fc7a38b63099ab0d42afd213b0d567c0f. --- libmproxy/proxy/config.py | 69 ++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 25 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 83030235..f438e9c2 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -49,11 +49,11 @@ class ProxyConfig: ciphers_client=None, ciphers_server=None, certs=[], - ssl_version_client=tcp.SSL_DEFAULT_METHOD, - ssl_version_server=tcp.SSL_DEFAULT_METHOD, + ssl_version_client="secure", + ssl_version_server="secure", ssl_verify_upstream_cert=False, ssl_upstream_trusted_cadir=None, - ssl_upstream_trusted_ca=None + ssl_upstream_trusted_ca=None, ): self.host = host self.port = port @@ -76,14 +76,10 @@ class ProxyConfig: for spec, cert in certs: self.certstore.add_cert_file(spec, cert) - if isinstance(ssl_version_client, int): - self.openssl_method_client = ssl_version_client - else: - self.openssl_method_client = tcp.SSL_VERSIONS[ssl_version_client] - if isinstance(ssl_version_server, int): - self.openssl_method_server = ssl_version_server - else: - self.openssl_method_server = tcp.SSL_VERSIONS[ssl_version_server] + self.openssl_method_client, self.openssl_options_client = version_to_openssl( + ssl_version_client) + self.openssl_method_server, self.openssl_options_server = version_to_openssl( + ssl_version_server) if ssl_verify_upstream_cert: self.openssl_verification_mode_server = SSL.VERIFY_PEER @@ -92,8 +88,33 @@ class ProxyConfig: self.openssl_trusted_cadir_server = ssl_upstream_trusted_cadir self.openssl_trusted_ca_server = ssl_upstream_trusted_ca - self.openssl_options_client = tcp.SSL_DEFAULT_OPTIONS - self.openssl_options_server = tcp.SSL_DEFAULT_OPTIONS + +sslversion_choices = ( + "all", + "secure", + "SSLv2", + "SSLv3", + "TLSv1", + "TLSv1_1", + "TLSv1_2") + + +def version_to_openssl(version): + """ + Convert a reasonable SSL version specification into the format OpenSSL expects. + Don't ask... + https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3 + """ + if version == "all": + return SSL.SSLv23_METHOD, None + elif version == "secure": + # SSLv23_METHOD + NO_SSLv2 + NO_SSLv3 == TLS 1.0+ + # TLSv1_METHOD would be TLS 1.0 only + return SSL.SSLv23_METHOD, (SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) + elif version in sslversion_choices: + return getattr(SSL, "%s_METHOD" % version), None + else: + raise ValueError("Invalid SSL version: %s" % version) def process_proxy_options(parser, options): @@ -254,18 +275,16 @@ def ssl_option_group(parser): help="Path to a PEM formatted trusted CA certificate." ) group.add_argument( - "--ssl-version-client", dest="ssl_version_client", type=str, default=tcp.SSL_DEFAULT_VERSION, - choices=tcp.SSL_VERSIONS.keys(), - help="""" - Use a specified protocol for client connections: - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. - Default to SSLv23.""" + "--ssl-version-client", dest="ssl_version_client", + default="secure", action="store", + choices=sslversion_choices, + help="Set supported SSL/TLS version for client connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure." ) group.add_argument( - "--ssl-version-server", dest="ssl_version_server", type=str, default=tcp.SSL_DEFAULT_VERSION, - choices=tcp.SSL_VERSIONS.keys(), - help="""" - Use a specified protocol for server connections: - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. - Default to SSLv23.""" + "--ssl-version-server", dest="ssl_version_server", + default="secure", action="store", + choices=sslversion_choices, + help="Set supported SSL/TLS version for server connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure." ) -- cgit v1.2.3 From 1cc48345e13917aadc1e0fd93d6011139e78e3d9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 28 Aug 2015 01:51:13 +0200 Subject: clean up config/cmdline, fix bugs, remove cruft --- libmproxy/cmdline.py | 247 +++++++++++++++++++++-------------- libmproxy/flow.py | 6 +- libmproxy/protocol/http.py | 2 +- libmproxy/protocol2/__init__.py | 7 +- libmproxy/protocol2/reverse_proxy.py | 5 +- libmproxy/protocol2/root_context.py | 10 +- libmproxy/protocol2/socks_proxy.py | 2 +- libmproxy/protocol2/tls.py | 38 +++++- libmproxy/proxy/config.py | 196 ++++++++------------------- libmproxy/proxy/server.py | 38 +++++- 10 files changed, 283 insertions(+), 268 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index d033fb76..1d897717 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -2,8 +2,8 @@ from __future__ import absolute_import import os import re import configargparse +from netlib.tcp import Address -from netlib import http import netlib.utils from . import filt, utils, version @@ -102,32 +102,22 @@ def parse_setheader(s): return _parse_hook(s) -def parse_server_spec(url): +def parse_server_spec(url, allowed_schemes=("http", "https")): p = netlib.utils.parse_url(url) - if not p or not p[1] or p[0] not in ("http", "https"): + if not p or not p[1] or p[0] not in allowed_schemes: raise configargparse.ArgumentTypeError( "Invalid server specification: %s" % url ) - - if p[0].lower() == "https": - ssl = [True, True] - else: - ssl = [False, False] - - return ssl + list(p[1:3]) + address = Address(p[1:3]) + scheme = p[0].lower() + return config.ServerSpec(scheme, address) def parse_server_spec_special(url): """ Provides additional support for http2https and https2http schemes. """ - normalized_url = re.sub("^https?2", "", url) - ret = parse_server_spec(normalized_url) - if url.lower().startswith("https2http"): - ret[0] = True - elif url.lower().startswith("http2https"): - ret[0] = False - return ret + return parse_server_spec(url, allowed_schemes=("http", "https", "http2https", "https2http")) def get_common_options(options): @@ -192,24 +182,24 @@ def get_common_options(options): outfile=options.outfile, verbosity=options.verbose, nopop=options.nopop, - replay_ignore_content = options.replay_ignore_content, - replay_ignore_params = options.replay_ignore_params, - replay_ignore_payload_params = options.replay_ignore_payload_params, - replay_ignore_host = options.replay_ignore_host + replay_ignore_content=options.replay_ignore_content, + replay_ignore_params=options.replay_ignore_params, + replay_ignore_payload_params=options.replay_ignore_payload_params, + replay_ignore_host=options.replay_ignore_host ) -def common_options(parser): +def basic_options(parser): parser.add_argument( '--version', - action= 'version', - version= "%(prog)s" + " " + version.VERSION + action='version', + version="%(prog)s" + " " + version.VERSION ) parser.add_argument( '--shortversion', - action= 'version', - help = "show program's short version number and exit", - version = version.VERSION + action='version', + help="show program's short version number and exit", + version=version.VERSION ) parser.add_argument( "--anticache", @@ -301,11 +291,42 @@ def common_options(parser): """ ) + +def proxy_modes(parser): + group = parser.add_argument_group("Proxy Modes").add_mutually_exclusive_group() + group.add_argument( + "-R", "--reverse", + action="store", + type=parse_server_spec_special, + dest="reverse_proxy", + default=None, + help=""" + Forward all requests to upstream HTTP server: + http[s][2http[s]]://host[:port] + """ + ) + group.add_argument( + "--socks", + action="store_true", dest="socks_proxy", default=False, + help="Set SOCKS5 proxy mode." + ) + group.add_argument( + "-T", "--transparent", + action="store_true", dest="transparent_proxy", default=False, + help="Set transparent proxy mode." + ) + group.add_argument( + "-U", "--upstream", + action="store", + type=parse_server_spec, + dest="upstream_proxy", + default=None, + help="Forward all requests to upstream proxy server: http://host[:port]" + ) + + +def proxy_options(parser): group = parser.add_argument_group("Proxy Options") - # We could make a mutually exclusive group out of -R, -U, -T, but we don't - # do that because - --upstream-server should be in that group as well, but - # it's already in a different group. - our own error messages are more - # helpful group.add_argument( "-b", "--bind-address", action="store", type=str, dest="addr", default='', @@ -344,70 +365,78 @@ def common_options(parser): action="store", type=int, dest="port", default=8080, help="Proxy service port." ) + + +def proxy_ssl_options(parser): + # TODO: Agree to consistently either use "upstream" or "server". + group = parser.add_argument_group("SSL") group.add_argument( - "-R", "--reverse", - action="store", - type=parse_server_spec_special, - dest="reverse_proxy", - default=None, - help=""" - Forward all requests to upstream HTTP server: - http[s][2http[s]]://host[:port] - """ - ) + "--cert", + dest='certs', + default=[], + type=str, + metavar="SPEC", + action="append", + help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' + 'The domain may include a wildcard, and is equal to "*" if not specified. ' + 'The file at path is a certificate in PEM format. If a private key is included ' + 'in the PEM, it is used, else the default key in the conf dir is used. ' + 'The PEM file should contain the full certificate chain, with the leaf certificate ' + 'as the first entry. Can be passed multiple times.') group.add_argument( - "--socks", - action="store_true", dest="socks_proxy", default=False, - help="Set SOCKS5 proxy mode." + "--ciphers-client", action="store", + type=str, dest="ciphers_client", default=config.DEFAULT_CLIENT_CIPHERS, + help="Set supported ciphers for client connections. (OpenSSL Syntax)" ) group.add_argument( - "-T", "--transparent", - action="store_true", dest="transparent_proxy", default=False, - help="Set transparent proxy mode." + "--ciphers-server", action="store", + type=str, dest="ciphers_server", default=None, + help="Set supported ciphers for server connections. (OpenSSL Syntax)" ) group.add_argument( - "-U", "--upstream", - action="store", - type=parse_server_spec, - dest="upstream_proxy", - default=None, - help="Forward all requests to upstream proxy server: http://host[:port]" + "--client-certs", action="store", + type=str, dest="clientcerts", default=None, + help="Client certificate directory." ) group.add_argument( - "--spoof", - action="store_true", dest="spoof_mode", default=False, - help="Use Host header to connect to HTTP servers." + "--no-upstream-cert", default=False, + action="store_true", dest="no_upstream_cert", + help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--ssl-spoof", - action="store_true", dest="ssl_spoof_mode", default=False, - help="Use TLS SNI to connect to HTTPS servers." + "--verify-upstream-cert", default=False, + action="store_true", dest="ssl_verify_upstream_cert", + help="Verify upstream server SSL/TLS certificates and fail if invalid " + "or not present." ) group.add_argument( - "--spoofed-port", - action="store", dest="spoofed_ssl_port", type=int, default=443, - help="Port number of upstream HTTPS servers in SSL spoof mode." + "--upstream-trusted-cadir", default=None, action="store", + dest="ssl_verify_upstream_trusted_cadir", + help="Path to a directory of trusted CA certificates for upstream " + "server verification prepared using the c_rehash tool." ) - - group = parser.add_argument_group( - "Advanced Proxy Options", - """ - The following options allow a custom adjustment of the proxy - behavior. Normally, you don't want to use these options directly and - use the provided wrappers instead (-R, -U, -T). - """ + group.add_argument( + "--upstream-trusted-ca", default=None, action="store", + dest="ssl_verify_upstream_trusted_ca", + help="Path to a PEM formatted trusted CA certificate." ) group.add_argument( - "--http-form-in", dest="http_form_in", default=None, - action="store", choices=("relative", "absolute"), - help="Override the HTTP request form accepted by the proxy" + "--ssl-version-client", dest="ssl_version_client", + default="secure", action="store", + choices=config.sslversion_choices.keys(), + help="Set supported SSL/TLS version for client connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) group.add_argument( - "--http-form-out", dest="http_form_out", default=None, - action="store", choices=("relative", "absolute"), - help="Override the HTTP request form sent upstream by the proxy" + "--ssl-version-server", dest="ssl_version_server", + default="secure", action="store", + choices=config.sslversion_choices.keys(), + help="Set supported SSL/TLS version for server connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) + +def onboarding_app(parser): group = parser.add_argument_group("Onboarding App") group.add_argument( "--noapp", @@ -433,6 +462,8 @@ def common_options(parser): help="Port to serve the onboarding app from." ) + +def client_replay(parser): group = parser.add_argument_group("Client Replay") group.add_argument( "-c", "--client-replay", @@ -440,6 +471,8 @@ def common_options(parser): help="Replay client requests from a saved file." ) + +def server_replay(parser): group = parser.add_argument_group("Server Replay") group.add_argument( "-S", "--server-replay", @@ -504,6 +537,8 @@ def common_options(parser): default=False, help="Ignore request's destination host while searching for a saved flow to replay") + +def replacements(parser): group = parser.add_argument_group( "Replacements", """ @@ -520,14 +555,16 @@ def common_options(parser): ) group.add_argument( "--replace-from-file", - action = "append", type=str, dest="replace_file", default=[], - metavar = "PATH", - help = """ + action="append", type=str, dest="replace_file", default=[], + metavar="PATH", + help=""" Replacement pattern, where the replacement clause is a path to a file. """ ) + +def set_headers(parser): group = parser.add_argument_group( "Set Headers", """ @@ -543,21 +580,22 @@ def common_options(parser): help="Header set pattern." ) + +def proxy_authentication(parser): group = parser.add_argument_group( "Proxy Authentication", """ Specify which users are allowed to access the proxy and the method used for authenticating them. """ - ) - user_specification_group = group.add_mutually_exclusive_group() - user_specification_group.add_argument( + ).add_mutually_exclusive_group() + group.add_argument( "--nonanonymous", action="store_true", dest="auth_nonanonymous", help="Allow access to any user long as a credentials are specified." ) - user_specification_group.add_argument( + group.add_argument( "--singleuser", action="store", dest="auth_singleuser", type=str, metavar="USER", @@ -566,14 +604,25 @@ def common_options(parser): username:password. """ ) - user_specification_group.add_argument( + group.add_argument( "--htpasswd", action="store", dest="auth_htpasswd", type=str, metavar="PATH", help="Allow access to users specified in an Apache htpasswd file." ) - config.ssl_option_group(parser) + +def common_options(parser): + basic_options(parser) + proxy_modes(parser) + proxy_options(parser) + proxy_ssl_options(parser) + onboarding_app(parser) + client_replay(parser) + server_replay(parser) + replacements(parser) + set_headers(parser) + proxy_authentication(parser) def mitmproxy(): @@ -583,13 +632,13 @@ def mitmproxy(): parser = configargparse.ArgumentParser( usage="%(prog)s [options]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmproxy.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) common_options(parser) parser.add_argument( @@ -628,20 +677,20 @@ def mitmproxy(): def mitmdump(): parser = configargparse.ArgumentParser( usage="%(prog)s [options] [filter]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmdump.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) common_options(parser) parser.add_argument( "--keepserving", - action= "store_true", dest="keepserving", default=False, - help= """ + action="store_true", dest="keepserving", default=False, + help=""" Continue serving after client playback or file read. We exit by default. """ @@ -658,13 +707,13 @@ def mitmdump(): def mitmweb(): parser = configargparse.ArgumentParser( usage="%(prog)s [options]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmweb.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) group = parser.add_argument_group("Mitmweb") diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 8605d7a1..a2b807ba 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -860,9 +860,9 @@ class FlowMaster(controller.Master): """ if self.server and self.server.config.mode == "reverse": - f.request.host, f.request.port = self.server.config.mode.dst[2:] - f.request.scheme = "https" if self.server.config.mode.dst[ - 1] else "http" + f.request.host = self.server.config.upstream_server.address.host + f.request.port = self.server.config.upstream_server.address.port + f.request.scheme = re.sub("^https?2", "", self.server.config.upstream_server.scheme) f.reply = controller.DummyReply() if f.request: diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4472cb2a..56d7d57f 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -10,6 +10,7 @@ from email.utils import parsedate_tz, formatdate, mktime_tz import netlib from netlib import http, tcp, odict, utils, encoding from netlib.http import cookies, http1, http2 +from netlib.http.http1 import HTTP1Protocol from netlib.http.semantics import CONTENT_MISSING from .tcp import TCPHandler @@ -757,7 +758,6 @@ class RequestReplayThread(threading.Thread): server.send(self.flow.server_conn.protocol.assemble(r)) self.flow.server_conn = server - self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) self.flow.response = HTTPResponse.from_protocol( self.flow.server_conn.protocol, r.method, diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index d5dafaae..61b9a77e 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -3,8 +3,11 @@ from .root_context import RootContext from .socks_proxy import Socks5Proxy from .reverse_proxy import ReverseProxy from .http_proxy import HttpProxy, HttpUpstreamProxy -from .rawtcp import RawTcpLayer +from .transparent_proxy import TransparentProxy +from .http import make_error_response __all__ = [ - "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" + "RootContext", + "Socks5Proxy", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "TransparentProxy", + "make_error_response" ] diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index 9d5a4beb..76163c71 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -12,10 +12,7 @@ class ReverseProxy(Layer, ServerConnectionMixin): self._server_tls = server_tls def __call__(self): - if self._client_tls or self._server_tls: - layer = TlsLayer(self, self._client_tls, self._server_tls) - else: - layer = self.ctx.next_layer(self) + layer = TlsLayer(self, self._client_tls, self._server_tls) try: layer() diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 78d48453..af0e7a37 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -4,7 +4,7 @@ from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol from .rawtcp import RawTcpLayer -from .tls import TlsLayer +from .tls import TlsLayer, is_tls_record_magic from .http import Http1Layer, Http2Layer @@ -38,13 +38,7 @@ class RootContext(object): # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello d = top_layer.client_conn.rfile.peek(3) - is_tls_client_hello = ( - len(d) == 3 and - d[0] == '\x16' and - d[1] == '\x03' and - d[2] in ('\x00', '\x01', '\x02', '\x03') - ) - if is_tls_client_hello: + if is_tls_record_magic(d): return TlsLayer(top_layer, True, True) # 3. Check for --tcp diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py index 18b363d5..91935d24 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/protocol2/socks_proxy.py @@ -8,7 +8,7 @@ from .layer import Layer, ServerConnectionMixin class Socks5Proxy(Layer, ServerConnectionMixin): def __call__(self): try: - s5mode = Socks5ProxyMode(self.config.ssl_ports) + s5mode = Socks5ProxyMode([]) address = s5mode.get_upstream_server(self.client_conn)[2:] except ProxyError as e: # TODO: Unmonkeypatch diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index b1b80034..850bf5dc 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -11,6 +11,21 @@ from ..exceptions import ProtocolException from .layer import Layer +def is_tls_record_magic(d): + """ + Returns: + True, if the passed bytes start with the TLS record magic bytes. + False, otherwise. + """ + d = d[:3] + return ( + len(d) == 3 and + d[0] == '\x16' and + d[1] == '\x03' and + d[2] in ('\x00', '\x01', '\x02', '\x03') + ) + + class TlsLayer(Layer): def __init__(self, ctx, client_tls, server_tls): self.client_sni = None @@ -69,9 +84,13 @@ class TlsLayer(Layer): client_hello_size = 1 offset = 0 while len(client_hello) < client_hello_size: - record_header = self.client_conn.rfile.peek(offset+5)[offset:] + record_header = self.client_conn.rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) != 5: + raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) record_size = struct.unpack("!H", record_header[3:])[0] + 5 - record_body = self.client_conn.rfile.peek(offset+record_size)[offset+5:] + record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) client_hello += record_body offset += record_size client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 @@ -81,7 +100,12 @@ class TlsLayer(Layer): """ Peek into the connection, read the initial client hello and parse it to obtain ALPN values. """ - raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + try: + raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + except ProtocolException as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + return + try: client_hello = ClientHello.parse(raw_client_hello) except ConstructError as e: @@ -97,7 +121,10 @@ class TlsLayer(Layer): elif extension.type == 0x10: self.client_alpn_protocols = list(extension.alpn_protocols) - self.log("Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), "debug") + self.log( + "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), + "debug" + ) def connect(self): if not self.server_conn: @@ -226,7 +253,8 @@ class TlsLayer(Layer): host = self.server_conn.address.host sans = set() # Incorporate upstream certificate - if self.server_conn and self.server_conn.tls_established and (not self.config.no_upstream_cert): + if self.server_conn and self.server_conn.tls_established and ( + not self.config.no_upstream_cert): upstream_cert = self.server_conn.cert sans.update(upstream_cert.altnames) if upstream_cert.cn: diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index f438e9c2..8ab5a216 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +import collections import os import re from OpenSSL import SSL @@ -7,6 +8,7 @@ from netlib import certutils, tcp from netlib.http import authentication from .. import utils, platform +from netlib.tcp import Address CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" @@ -15,8 +17,9 @@ CA_DIR = "~/.mitmproxy" # https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" + class HostMatcher(object): - def __init__(self, patterns=[]): + def __init__(self, patterns=tuple()): self.patterns = list(patterns) self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] @@ -32,6 +35,9 @@ class HostMatcher(object): return bool(self.patterns) +ServerSpec = collections.namedtuple("ServerSpec", "scheme address") + + class ProxyConfig: def __init__( self, @@ -41,19 +47,19 @@ class ProxyConfig: clientcerts=None, no_upstream_cert=False, body_size_limit=None, - mode=None, + mode="regular", upstream_server=None, authenticator=None, - ignore_hosts=[], - tcp_hosts=[], + ignore_hosts=tuple(), + tcp_hosts=tuple(), ciphers_client=None, ciphers_server=None, - certs=[], + certs=tuple(), ssl_version_client="secure", ssl_version_server="secure", ssl_verify_upstream_cert=False, - ssl_upstream_trusted_cadir=None, - ssl_upstream_trusted_ca=None, + ssl_verify_upstream_trusted_cadir=None, + ssl_verify_upstream_trusted_ca=None, ): self.host = host self.port = port @@ -63,7 +69,10 @@ class ProxyConfig: self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit self.mode = mode - self.upstream_server = upstream_server + if upstream_server: + self.upstream_server = ServerSpec(upstream_server[0], Address.wrap(upstream_server[1])) + else: + self.upstream_server = None self.check_ignore = HostMatcher(ignore_hosts) self.check_tcp = HostMatcher(tcp_hosts) @@ -76,57 +85,46 @@ class ProxyConfig: for spec, cert in certs: self.certstore.add_cert_file(spec, cert) - self.openssl_method_client, self.openssl_options_client = version_to_openssl( - ssl_version_client) - self.openssl_method_server, self.openssl_options_server = version_to_openssl( - ssl_version_server) + self.openssl_method_client, self.openssl_options_client = \ + sslversion_choices[ssl_version_client] + self.openssl_method_server, self.openssl_options_server = \ + sslversion_choices[ssl_version_server] if ssl_verify_upstream_cert: self.openssl_verification_mode_server = SSL.VERIFY_PEER else: self.openssl_verification_mode_server = SSL.VERIFY_NONE - self.openssl_trusted_cadir_server = ssl_upstream_trusted_cadir - self.openssl_trusted_ca_server = ssl_upstream_trusted_ca - - -sslversion_choices = ( - "all", - "secure", - "SSLv2", - "SSLv3", - "TLSv1", - "TLSv1_1", - "TLSv1_2") - - -def version_to_openssl(version): - """ - Convert a reasonable SSL version specification into the format OpenSSL expects. - Don't ask... - https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3 - """ - if version == "all": - return SSL.SSLv23_METHOD, None - elif version == "secure": - # SSLv23_METHOD + NO_SSLv2 + NO_SSLv3 == TLS 1.0+ - # TLSv1_METHOD would be TLS 1.0 only - return SSL.SSLv23_METHOD, (SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) - elif version in sslversion_choices: - return getattr(SSL, "%s_METHOD" % version), None - else: - raise ValueError("Invalid SSL version: %s" % version) + self.openssl_trusted_cadir_server = ssl_verify_upstream_trusted_cadir + self.openssl_trusted_ca_server = ssl_verify_upstream_trusted_ca + + +""" +Map a reasonable SSL version specification into the format OpenSSL expects. +Don't ask... +https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3 +""" +sslversion_choices = { + "all": (SSL.SSLv23_METHOD, 0), + # SSLv23_METHOD + NO_SSLv2 + NO_SSLv3 == TLS 1.0+ + # TLSv1_METHOD would be TLS 1.0 only + "secure": (SSL.SSLv23_METHOD, (SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)), + "SSLv2": (SSL.SSLv2_METHOD, 0), + "SSLv3": (SSL.SSLv3_METHOD, 0), + "TLSv1": (SSL.TLSv1_METHOD, 0), + "TLSv1_1": (SSL.TLSv1_1_METHOD, 0), + "TLSv1_2": (SSL.TLSv1_2_METHOD, 0), +} def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) c = 0 - mode, upstream_server, spoofed_ssl_port = None, None, None + mode, upstream_server = "regular", None if options.transparent_proxy: c += 1 if not platform.resolver: - return parser.error( - "Transparent mode not supported on this platform.") + return parser.error("Transparent mode not supported on this platform.") mode = "transparent" if options.socks_proxy: c += 1 @@ -139,32 +137,26 @@ def process_proxy_options(parser, options): c += 1 mode = "upstream" upstream_server = options.upstream_proxy - if options.spoof_mode: - c += 1 - mode = "spoof" - if options.ssl_spoof_mode: - c += 1 - mode = "sslspoof" - spoofed_ssl_port = options.spoofed_ssl_port if c > 1: return parser.error( "Transparent, SOCKS5, reverse and upstream proxy mode " - "are mutually exclusive.") + "are mutually exclusive. Read the docs on proxy modes to understand why." + ) if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) - if not os.path.exists( - options.clientcerts) or not os.path.isdir( - options.clientcerts): + if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): return parser.error( "Client certificate directory does not exist or is not a directory: %s" % - options.clientcerts) + options.clientcerts + ) - if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd): + if options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd: if options.auth_singleuser: if len(options.auth_singleuser.split(':')) != 2: return parser.error( - "Invalid single-user specification. Please use the format username:password") + "Invalid single-user specification. Please use the format username:password" + ) username, password = options.auth_singleuser.split(':') password_manager = authentication.PassManSingleUser(username, password) elif options.auth_nonanonymous: @@ -189,12 +181,6 @@ def process_proxy_options(parser, options): parser.error("Certificate file does not exist: %s" % parts[1]) certs.append(parts) - ssl_ports = options.ssl_ports - if options.ssl_ports != TRANSPARENT_SSL_PORTS: - # arparse appends to default value by default, strip that off. - # see http://bugs.python.org/issue16399 - ssl_ports = ssl_ports[len(TRANSPARENT_SSL_PORTS):] - return ProxyConfig( host=options.addr, port=options.port, @@ -204,87 +190,15 @@ def process_proxy_options(parser, options): body_size_limit=body_size_limit, mode=mode, upstream_server=upstream_server, - http_form_in=options.http_form_in, - http_form_out=options.http_form_out, ignore_hosts=options.ignore_hosts, tcp_hosts=options.tcp_hosts, authenticator=authenticator, ciphers_client=options.ciphers_client, ciphers_server=options.ciphers_server, - certs=certs, + certs=tuple(certs), ssl_version_client=options.ssl_version_client, ssl_version_server=options.ssl_version_server, - ssl_ports=ssl_ports, - spoofed_ssl_port=spoofed_ssl_port, ssl_verify_upstream_cert=options.ssl_verify_upstream_cert, - ssl_upstream_trusted_cadir=options.ssl_upstream_trusted_cadir, - ssl_upstream_trusted_ca=options.ssl_upstream_trusted_ca - ) - - -def ssl_option_group(parser): - group = parser.add_argument_group("SSL") - group.add_argument( - "--cert", - dest='certs', - default=[], - type=str, - metavar="SPEC", - action="append", - help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' - 'The domain may include a wildcard, and is equal to "*" if not specified. ' - 'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' - 'it is used, else the default key in the conf dir is used. ' - 'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. ' - 'Can be passed multiple times.') - group.add_argument( - "--ciphers-client", action="store", - type=str, dest="ciphers_client", default=DEFAULT_CLIENT_CIPHERS, - help="Set supported ciphers for client connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--ciphers-server", action="store", - type=str, dest="ciphers_server", default=None, - help="Set supported ciphers for server connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--client-certs", action="store", - type=str, dest="clientcerts", default=None, - help="Client certificate directory." - ) - group.add_argument( - "--no-upstream-cert", default=False, - action="store_true", dest="no_upstream_cert", - help="Don't connect to upstream server to look up certificate details." - ) - group.add_argument( - "--verify-upstream-cert", default=False, - action="store_true", dest="ssl_verify_upstream_cert", - help="Verify upstream server SSL/TLS certificates and fail if invalid " - "or not present." - ) - group.add_argument( - "--upstream-trusted-cadir", default=None, action="store", - dest="ssl_upstream_trusted_cadir", - help="Path to a directory of trusted CA certificates for upstream " - "server verification prepared using the c_rehash tool." - ) - group.add_argument( - "--upstream-trusted-ca", default=None, action="store", - dest="ssl_upstream_trusted_ca", - help="Path to a PEM formatted trusted CA certificate." - ) - group.add_argument( - "--ssl-version-client", dest="ssl_version_client", - default="secure", action="store", - choices=sslversion_choices, - help="Set supported SSL/TLS version for client connections. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure." - ) - group.add_argument( - "--ssl-version-server", dest="ssl_version_server", - default="secure", action="store", - choices=sslversion_choices, - help="Set supported SSL/TLS version for server connections. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure." - ) + ssl_verify_upstream_trusted_cadir=options.ssl_verify_upstream_trusted_cadir, + ssl_verify_upstream_trusted_ca=options.ssl_verify_upstream_trusted_ca + ) \ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 19ddb930..1fc4cbda 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -5,6 +5,8 @@ import sys import socket from libmproxy.protocol2.layer import Kill from netlib import tcp +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError from ..protocol.handle import protocol_handler from .. import protocol2 @@ -82,11 +84,31 @@ class ConnectionHandler2: self.channel ) - # FIXME: properly parse config - if self.config.mode == "upstream": - root_layer = protocol2.HttpUpstreamProxy(root_context, ("localhost", 8081)) - else: + mode = self.config.mode + if mode == "upstream": + root_layer = protocol2.HttpUpstreamProxy( + root_context, + self.config.upstream_server.address + ) + elif mode == "transparent": + root_layer = protocol2.TransparentProxy(root_context) + elif mode == "reverse": + client_tls = self.config.upstream_server.scheme.startswith("https") + server_tls = self.config.upstream_server.scheme.endswith("https") + root_layer = protocol2.ReverseProxy( + root_context, + self.config.upstream_server.address, + client_tls, + server_tls + ) + elif mode == "socks5": + root_layer = protocol2.Socks5Proxy(root_context) + elif mode == "regular": root_layer = protocol2.HttpProxy(root_context) + elif callable(mode): # pragma: nocover + root_layer = mode(root_context) + else: # pragma: nocover + raise ValueError("Unknown proxy mode: %s" % mode) try: root_layer() @@ -94,6 +116,14 @@ class ConnectionHandler2: self.log("Connection killed", "info") except ProtocolException as e: self.log(e, "info") + # If an error propagates to the topmost level, + # we send an HTTP error response, which is both + # understandable by HTTP clients and humans. + try: + error_response = protocol2.make_error_response(502, repr(e)) + self.client_conn.send(HTTP1Protocol().assemble(error_response)) + except NetLibError: + pass except Exception: self.log(traceback.format_exc(), "error") print(traceback.format_exc(), file=sys.stderr) -- cgit v1.2.3 From 2dfba2105b4b5ad094ee364124c0552d2e4a4947 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 29 Aug 2015 12:34:01 +0200 Subject: move sslversion mapping to netlib --- libmproxy/cmdline.py | 10 +++++----- libmproxy/proxy/config.py | 20 +------------------- 2 files changed, 6 insertions(+), 24 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 1d897717..591e87ed 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import os import re import configargparse -from netlib.tcp import Address +from netlib.tcp import Address, sslversion_choices import netlib.utils @@ -423,15 +423,15 @@ def proxy_ssl_options(parser): group.add_argument( "--ssl-version-client", dest="ssl_version_client", default="secure", action="store", - choices=config.sslversion_choices.keys(), - help="Set supported SSL/TLS version for client connections. " + choices=sslversion_choices.keys(), + help="Set supported SSL/TLS versions for client connections. " "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) group.add_argument( "--ssl-version-server", dest="ssl_version_server", default="secure", action="store", - choices=config.sslversion_choices.keys(), - help="Set supported SSL/TLS version for server connections. " + choices=sslversion_choices.keys(), + help="Set supported SSL/TLS versions for server connections. " "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 8ab5a216..415ee215 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -8,7 +8,7 @@ from netlib import certutils, tcp from netlib.http import authentication from .. import utils, platform -from netlib.tcp import Address +from netlib.tcp import Address, sslversion_choices CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" @@ -98,24 +98,6 @@ class ProxyConfig: self.openssl_trusted_ca_server = ssl_verify_upstream_trusted_ca -""" -Map a reasonable SSL version specification into the format OpenSSL expects. -Don't ask... -https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3 -""" -sslversion_choices = { - "all": (SSL.SSLv23_METHOD, 0), - # SSLv23_METHOD + NO_SSLv2 + NO_SSLv3 == TLS 1.0+ - # TLSv1_METHOD would be TLS 1.0 only - "secure": (SSL.SSLv23_METHOD, (SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)), - "SSLv2": (SSL.SSLv2_METHOD, 0), - "SSLv3": (SSL.SSLv3_METHOD, 0), - "TLSv1": (SSL.TLSv1_METHOD, 0), - "TLSv1_1": (SSL.TLSv1_1_METHOD, 0), - "TLSv1_2": (SSL.TLSv1_2_METHOD, 0), -} - - def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) -- cgit v1.2.3 From 63844df34367bf7147c2d43a9e4061515f6430c9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 29 Aug 2015 14:28:11 +0200 Subject: fix streaming --- libmproxy/protocol2/http.py | 192 ++++++++++++++++++++++++++++---------------- 1 file changed, 122 insertions(+), 70 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 792cf266..0fde9fb1 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -25,32 +25,101 @@ from netlib.http.http2 import HTTP2Protocol # TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. -class Http1Layer(Layer): +class _HttpLayer(Layer): + supports_streaming = False + + def read_request(self): + raise NotImplementedError() + + def send_request(self, request): + raise NotImplementedError() + + def read_response(self, request_method): + raise NotImplementedError() + + def send_response(self, response): + raise NotImplementedError() + +class _StreamingHttpLayer(_HttpLayer): + supports_streaming = True + + def read_response_headers(self): + raise NotImplementedError + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + raise NotImplementedError() + yield "this is a generator" + + def send_response_headers(self, response): + raise NotImplementedError + + def send_response_body(self, response, chunks): + raise NotImplementedError() + + +class Http1Layer(_StreamingHttpLayer): + def __init__(self, ctx, mode): super(Http1Layer, self).__init__(ctx) self.mode = mode self.client_protocol = HTTP1Protocol(self.client_conn) self.server_protocol = HTTP1Protocol(self.server_conn) - def read_from_client(self): + def read_request(self): return HTTPRequest.from_protocol( self.client_protocol, body_size_limit=self.config.body_size_limit ) - def read_from_server(self, request_method): + def send_request(self, request): + self.server_conn.send(self.server_protocol.assemble(request)) + + def read_response(self, request_method): return HTTPResponse.from_protocol( self.server_protocol, - request_method, + request_method=request_method, body_size_limit=self.config.body_size_limit, - include_body=False, + include_body=True ) - def send_to_client(self, message): - self.client_conn.send(self.client_protocol.assemble(message)) + def send_response(self, response): + self.client_conn.send(self.client_protocol.assemble(response)) - def send_to_server(self, message): - self.server_conn.send(self.server_protocol.assemble(message)) + def read_response_headers(self): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=None, # does not matter if we don't read the body. + body_size_limit=self.config.body_size_limit, + include_body=False + ) + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + return self.server_protocol.read_http_body_chunked( + headers, + self.config.body_size_limit, + request_method, + response_code, + False, + max_chunk_size + ) + + def send_response_headers(self, response): + h = self.client_protocol._assemble_response_first_line(response) + self.client_conn.wfile.write(h+"\r\n") + h = self.client_protocol._assemble_response_headers( + response, + preserve_transfer_encoding=True + ) + self.client_conn.send(h+"\r\n") + + def send_response_body(self, response, chunks): + if self.client_protocol.has_chunked_encoding(response.headers): + chunks = ( + "%d\r\n%s\r\n" % (len(chunk), chunk) + for chunk in chunks + ) + for chunk in chunks: + self.client_conn.send(chunk) def connect(self): self.ctx.connect() @@ -69,14 +138,14 @@ class Http1Layer(Layer): layer() -class Http2Layer(Layer): +class Http2Layer(_HttpLayer): def __init__(self, ctx, mode): super(Http2Layer, self).__init__(ctx) self.mode = mode self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) - def read_from_client(self): + def read_request(self): request = HTTPRequest.from_protocol( self.client_protocol, body_size_limit=self.config.body_size_limit @@ -84,23 +153,23 @@ class Http2Layer(Layer): self._stream_id = request.stream_id return request - def read_from_server(self, request_method): + def send_request(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.server_conn.send(self.server_protocol.assemble(message)) + + def read_response(self, request_method): return HTTPResponse.from_protocol( self.server_protocol, - request_method, + request_method=request_method, body_size_limit=self.config.body_size_limit, include_body=True, stream_id=self._stream_id ) - def send_to_client(self, message): + def send_response(self, message): # TODO: implement flow control and WINDOW_UPDATE frames self.client_conn.send(self.client_protocol.assemble(message)) - def send_to_server(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames - self.server_conn.send(self.server_protocol.assemble(message)) - def connect(self): self.ctx.connect() self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) @@ -122,7 +191,7 @@ class Http2Layer(Layer): layer() def handle_unexpected_frame(self, frm): - print(frm.human_readable()) + self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") def make_error_response(status_code, message, headers=None): @@ -204,13 +273,13 @@ class UpstreamConnectLayer(Layer): def connect(self): if not self.server_conn: self.ctx.connect() - self.send_to_server(self.connect_request) + self.send_request(self.connect_request) else: pass # swallow the message def reconnect(self): self.ctx.reconnect() - self.send_to_server(self.connect_request) + self.send_request(self.connect_request) def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: @@ -240,7 +309,7 @@ class HttpLayer(Layer): flow = HTTPFlow(self.client_conn, self.server_conn, live=self) try: - request = self.read_from_client() + request = self.read_request() except tcp.NetLibError: # don't throw an error for disconnects that happen # before/between requests. @@ -280,7 +349,7 @@ class HttpLayer(Layer): except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: try: - self.send_to_client(make_error_response( + self.send_response(make_error_response( getattr(e, "code", 502), repr(e) )) @@ -295,7 +364,7 @@ class HttpLayer(Layer): def handle_regular_mode_connect(self, request): self.set_server((request.host, request.port)) - self.send_to_client(make_connect_response(request.httpversion)) + self.send_response(make_connect_response(request.httpversion)) layer = self.ctx.next_layer(self) layer() @@ -334,44 +403,33 @@ class HttpLayer(Layer): return close_connection def send_response_to_client(self, flow): - if not flow.response.stream: + if not (self.supports_streaming and flow.response.stream): # no streaming: # we already received the full response from the server and can # send it to the client straight away. - self.send_to_client(flow.response) + self.send_response(flow.response) else: # streaming: - # First send the headers and then transfer the response - # incrementally: - h = self.client_protocol._assemble_response_first_line(flow.response) - self.send_to_client(h + "\r\n") - h = self.client_protocol._assemble_response_headers(flow.response, preserve_transfer_encoding=True) - self.send_to_client(h + "\r\n") - - chunks = self.client_protocol.read_http_body_chunked( - flow.response.headers, - self.config.body_size_limit, - flow.request.method, - flow.response.code, - False, - 4096 + # First send the headers and then transfer the response incrementally + self.send_response_headers(flow.response) + chunks = self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code, + max_chunk_size=4096 ) - if callable(flow.response.stream): chunks = flow.response.stream(chunks) - - for chunk in chunks: - for part in chunk: - # TODO: That's going to fail. - self.send_to_client(part) - self.client_conn.wfile.flush() - + self.send_response_body(flow.response, chunks) flow.response.timestamp_end = utils.timestamp() def get_response_from_server(self, flow): def get_response(): - self.send_to_server(flow.request) - flow.response = self.read_from_server(flow.request.method) + self.send_request(flow.request) + if self.supports_streaming: + flow.response = self.read_response_headers() + else: + flow.response = self.read_response() try: get_response() @@ -400,18 +458,15 @@ class HttpLayer(Layer): if flow is None or flow == KILL: raise Kill() - if isinstance(self.ctx, Http2Layer): - pass # streaming is not implemented for http2 yet. - elif flow.response.stream: - flow.response.content = CONTENT_MISSING - else: - flow.response.content = self.server_protocol.read_http_body( - flow.response.headers, - self.config.body_size_limit, - flow.request.method, - flow.response.code, - False - ) + if self.supports_streaming: + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = "".join(self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code + )) flow.response.timestamp_end = utils.timestamp() # no further manipulation of self.server_conn beyond this point @@ -480,14 +535,14 @@ class HttpLayer(Layer): if self.server_conn.tls_established: self.reconnect() - self.send_to_server(make_connect_request(address)) + self.send_request(make_connect_request(address)) tls_layer = TlsLayer(self, False, True) tls_layer._establish_tls_with_server() """ def validate_request(self, request): if request.form_in == "absolute" and request.scheme != "http": - self.send_to_client(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + self.send_response(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { @@ -501,7 +556,7 @@ class HttpLayer(Layer): err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( " or ".join(allowed_request_forms), request.form_in ) - self.send_to_client(make_error_response(400, err_message)) + self.send_response(make_error_response(400, err_message)) raise HttpException(err_message) if self.mode == "regular": @@ -512,7 +567,7 @@ class HttpLayer(Layer): if self.config.authenticator.authenticate(request.headers): self.config.authenticator.clean(request.headers) else: - self.send_to_client(make_error_response( + self.send_response(make_error_response( 407, "Proxy Authentication Required", odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()]) @@ -552,10 +607,7 @@ class RequestReplayThread(threading.Thread): if not self.flow.response: # In all modes, we directly connect to the server displayed if self.config.mode == "upstream": - # FIXME - server_address = self.config.mode.get_upstream_server( - self.flow.client_conn - )[2:] + server_address = self.config.upstream_server.address server = ServerConnection(server_address) server.connect() protocol = HTTP1Protocol(server) -- cgit v1.2.3 From a7058e2a3c59cc2b13aaea3d7c767a3ca4a4bc40 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 29 Aug 2015 20:53:25 +0200 Subject: fix bugs, fix tests --- libmproxy/console/statusbar.py | 11 +++++---- libmproxy/protocol2/http.py | 54 +++++++++++++++++++++++++++++------------- 2 files changed, 43 insertions(+), 22 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/console/statusbar.py b/libmproxy/console/statusbar.py index 7eb2131b..ea2dbfa8 100644 --- a/libmproxy/console/statusbar.py +++ b/libmproxy/console/statusbar.py @@ -199,11 +199,12 @@ class StatusBar(urwid.WidgetWrap): r.append("[%s]" % (":".join(opts))) if self.master.server.config.mode in ["reverse", "upstream"]: - dst = self.master.server.config.mode.dst - scheme = "https" if dst[0] else "http" - if dst[1] != dst[0]: - scheme += "2https" if dst[1] else "http" - r.append("[dest:%s]" % utils.unparse_url(scheme, *dst[2:])) + dst = self.master.server.config.upstream_server + r.append("[dest:%s]" % netlib.utils.unparse_url( + dst.scheme, + dst.address.host, + dst.address.port + )) if self.master.scripts: r.append("[") r.append(("heading_key", "s")) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 0fde9fb1..a3f32926 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -40,6 +40,7 @@ class _HttpLayer(Layer): def send_response(self, response): raise NotImplementedError() + class _StreamingHttpLayer(_HttpLayer): supports_streaming = True @@ -58,7 +59,6 @@ class _StreamingHttpLayer(_HttpLayer): class Http1Layer(_StreamingHttpLayer): - def __init__(self, ctx, mode): super(Http1Layer, self).__init__(ctx) self.mode = mode @@ -105,12 +105,12 @@ class Http1Layer(_StreamingHttpLayer): def send_response_headers(self, response): h = self.client_protocol._assemble_response_first_line(response) - self.client_conn.wfile.write(h+"\r\n") + self.client_conn.wfile.write(h + "\r\n") h = self.client_protocol._assemble_response_headers( response, preserve_transfer_encoding=True ) - self.client_conn.send(h+"\r\n") + self.client_conn.send(h + "\r\n") def send_response_body(self, response, chunks): if self.client_protocol.has_chunked_encoding(response.headers): @@ -142,8 +142,10 @@ class Http2Layer(_HttpLayer): def __init__(self, ctx, mode): super(Http2Layer, self).__init__(ctx) self.mode = mode - self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) def read_request(self): request = HTTPRequest.from_protocol( @@ -172,17 +174,20 @@ class Http2Layer(_HttpLayer): def connect(self): self.ctx.connect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol.perform_connection_preface() def reconnect(self): self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol.perform_connection_preface() def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol.perform_connection_preface() def __call__(self): @@ -264,7 +269,10 @@ class UpstreamConnectLayer(Layer): def __init__(self, ctx, connect_request): super(UpstreamConnectLayer, self).__init__(ctx) self.connect_request = connect_request - self.server_conn = ConnectServerConnection((connect_request.host, connect_request.port), self.ctx) + self.server_conn = ConnectServerConnection( + (connect_request.host, connect_request.port), + self.ctx + ) def __call__(self): layer = self.ctx.next_layer(self) @@ -280,6 +288,9 @@ class UpstreamConnectLayer(Layer): def reconnect(self): self.ctx.reconnect() self.send_request(self.connect_request) + resp = self.read_response("CONNECT") + if resp.code != 200: + raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: @@ -290,7 +301,7 @@ class UpstreamConnectLayer(Layer): self.connect_request.port = address.port self.server_conn.address = address else: - self.ctx.set_server(address, server_tls, sni, depth-1) + self.ctx.set_server(address, server_tls, sni, depth - 1) class HttpLayer(Layer): @@ -413,10 +424,10 @@ class HttpLayer(Layer): # First send the headers and then transfer the response incrementally self.send_response_headers(flow.response) chunks = self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code, - max_chunk_size=4096 + flow.response.headers, + flow.request.method, + flow.response.code, + max_chunk_size=4096 ) if callable(flow.response.stream): chunks = flow.response.stream(chunks) @@ -521,7 +532,8 @@ class HttpLayer(Layer): # If there's not TlsLayer below which could catch the exception, # TLS will not be established. if tls and not self.server_conn.tls_established: - raise ProtocolException("Cannot upgrade to SSL, no TLS layer on the protocol stack.") + raise ProtocolException( + "Cannot upgrade to SSL, no TLS layer on the protocol stack.") else: if not self.server_conn: self.connect() @@ -542,7 +554,8 @@ class HttpLayer(Layer): def validate_request(self, request): if request.form_in == "absolute" and request.scheme != "http": - self.send_response(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + self.send_response( + make_error_response(400, "Invalid request scheme: %s" % request.scheme)) raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { @@ -570,7 +583,11 @@ class HttpLayer(Layer): self.send_response(make_error_response( 407, "Proxy Authentication Required", - odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()]) + odict.ODictCaseless( + [ + [k, v] for k, v in + self.config.authenticator.auth_challenge_headers().items() + ]) )) raise InvalidCredentials("Proxy Authentication Required") @@ -614,6 +631,9 @@ class RequestReplayThread(threading.Thread): if r.scheme == "https": connect_request = make_connect_request((r.host, r.port)) server.send(protocol.assemble(connect_request)) + resp = protocol.read_response("CONNECT") + if resp.code != 200: + raise HttpError(502, "Upstream server refuses CONNECT request") server.establish_ssl( self.config.clientcerts, sni=self.flow.server_conn.sni -- cgit v1.2.3 From 100ea27c30d89b895a02a1b128edc5472ab84b3e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 29 Aug 2015 23:08:16 +0200 Subject: simplify raw tcp protocol --- libmproxy/protocol2/rawtcp.py | 62 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 8 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index 6819ad6e..e8e3cf65 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -1,21 +1,67 @@ from __future__ import (absolute_import, print_function, division) +import socket +import select + +from OpenSSL import SSL -import OpenSSL from ..exceptions import ProtocolException +from netlib.tcp import NetLibError +from netlib.utils import cleanBin from ..protocol.tcp import TCPHandler from .layer import Layer class RawTcpLayer(Layer): + chunk_size = 4096 + + def __init__(self, ctx, logging=True): + self.logging = logging + super(RawTcpLayer, self).__init__(ctx) + def __call__(self): self.connect() - tcp_handler = TCPHandler(self) + + buf = memoryview(bytearray(self.chunk_size)) + + client = self.client_conn.connection + server = self.server_conn.connection + conns = [client, server] + try: - tcp_handler.handle_messages() - except OpenSSL.SSL.Error as e: - raise ProtocolException("SSL error: %s" % repr(e), e) + while True: + r, _, _ = select.select(conns, [], [], 10) + for conn in r: + + size = conn.recv_into(buf, self.chunk_size) + if not size: + conns.remove(conn) + # Shutdown connection to the other peer + if isinstance(conn, SSL.Connection): + # We can't half-close a connection, so we just close everything here. + # Sockets will be cleaned up on a higher level. + return + else: + conn.shutdown(socket.SHUT_WR) + + if len(conns) == 0: + return + continue + + dst = server if conn == client else client + dst.sendall(buf[:size]) + if self.logging: + # log messages are prepended with the client address, + # hence the "weird" direction string. + if dst == server: + direction = "-> tcp -> {!r}".format(self.server_conn.address) + else: + direction = "<- tcp <- {!r}".format(self.server_conn.address) + data = cleanBin(buf[:size].tobytes()) + self.log( + "{}\r\n{}".format(direction, data), + "info" + ) - def establish_server_connection(self): - pass - # FIXME: Remove method, currently just here to mock TCPHandler's call to it. + except (socket.error, NetLibError, SSL.Error) as e: + raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) \ No newline at end of file -- cgit v1.2.3 From dd7f50d64bef38fa67b4cace91913d03691dde26 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 01:21:58 +0200 Subject: restructure code, remove cruft --- libmproxy/exceptions.py | 4 + libmproxy/flow.py | 2 +- libmproxy/protocol/http.py | 83 +------ libmproxy/protocol2/http.py | 112 +-------- libmproxy/protocol2/http_proxy.py | 3 +- libmproxy/protocol2/http_replay.py | 95 ++++++++ libmproxy/protocol2/layer.py | 19 +- libmproxy/protocol2/messages.py | 46 ---- libmproxy/protocol2/rawtcp.py | 13 +- libmproxy/protocol2/reverse_proxy.py | 5 +- libmproxy/protocol2/root_context.py | 5 +- libmproxy/protocol2/socks_proxy.py | 54 ++++- libmproxy/protocol2/tls.py | 22 +- libmproxy/protocol2/transparent_proxy.py | 3 +- libmproxy/proxy/__init__.py | 13 +- libmproxy/proxy/connection.py | 6 +- libmproxy/proxy/primitives.py | 179 +------------- libmproxy/proxy/server.py | 401 +++---------------------------- libmproxy/utils.py | 4 - 19 files changed, 240 insertions(+), 829 deletions(-) create mode 100644 libmproxy/protocol2/http_replay.py delete mode 100644 libmproxy/protocol2/messages.py (limited to 'libmproxy') diff --git a/libmproxy/exceptions.py b/libmproxy/exceptions.py index 3825c409..f34d9707 100644 --- a/libmproxy/exceptions.py +++ b/libmproxy/exceptions.py @@ -18,6 +18,10 @@ class ProtocolException(ProxyException): pass +class Socks5Exception(ProtocolException): + pass + + class HttpException(ProtocolException): pass diff --git a/libmproxy/flow.py b/libmproxy/flow.py index a2b807ba..dac607a0 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,7 +8,7 @@ import Cookie import cookielib import os import re -from libmproxy.protocol2.http import RequestReplayThread +from libmproxy.protocol2.http_replay import RequestReplayThread from netlib import odict, wsgi, tcp from netlib.http.semantics import CONTENT_MISSING diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 56d7d57f..a30437d1 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -695,85 +695,4 @@ class HTTPHandler(ProtocolHandler): else: raise http.HttpAuthenticationError( self.c.config.authenticator.auth_challenge_headers()) - return request.headers - - -class RequestReplayThread(threading.Thread): - name = "RequestReplayThread" - - def __init__(self, config, flow, masterq, should_exit): - """ - masterqueue can be a queue or None, if no scripthooks should be - processed. - """ - self.config, self.flow = config, flow - if masterq: - self.channel = controller.Channel(masterq, should_exit) - else: - self.channel = None - super(RequestReplayThread, self).__init__() - - def run(self): - r = self.flow.request - form_out_backup = r.form_out - try: - self.flow.response = None - - # If we have a channel, run script hooks. - if self.channel: - request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == KILL: - raise KillSignal() - elif isinstance(request_reply, HTTPResponse): - self.flow.response = request_reply - - if not self.flow.response: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - # FIXME - server_address = self.config.mode.get_upstream_server( - self.flow.client_conn - )[2:] - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - send_connect_request(server, r.host, r.port) - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - else: - r.form_out = "absolute" - else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - - server.send(self.flow.server_conn.protocol.assemble(r)) - self.flow.server_conn = server - self.flow.response = HTTPResponse.from_protocol( - self.flow.server_conn.protocol, - r.method, - body_size_limit=self.config.body_size_limit, - ) - if self.channel: - response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == KILL: - raise KillSignal() - except (proxy.ProxyError, http.HttpError, tcp.NetLibError) as v: - self.flow.error = Error(repr(v)) - if self.channel: - self.channel.ask("error", self.flow) - except KillSignal: - # KillSignal should only be raised if there's a channel in the - # first place. - self.channel.tell("log", proxy.Log("Connection killed", "info")) - finally: - r.form_out = form_out_backup + return request.headers \ No newline at end of file diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index a3f32926..a508ae8b 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -1,28 +1,20 @@ from __future__ import (absolute_import, print_function, division) -from .. import version -import threading -from ..exceptions import InvalidCredentials, HttpException, ProtocolException -from .layer import Layer -from libmproxy import utils -from libmproxy.controller import Channel -from libmproxy.protocol2.layer import Kill -from libmproxy.protocol import KILL, Error - -from libmproxy.protocol.http import HTTPFlow -from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest -from libmproxy.proxy import Log -from libmproxy.proxy.connection import ServerConnection from netlib import tcp -from netlib.http import status_codes, http1, http2, HttpErrorConnClosed, HttpError +from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError from netlib.http.semantics import CONTENT_MISSING from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol - -# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. +from .. import version, utils +from ..exceptions import InvalidCredentials, HttpException, ProtocolException +from .layer import Layer +from ..proxy import Kill +from libmproxy.protocol import KILL, Error +from libmproxy.protocol.http import HTTPFlow +from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest class _HttpLayer(Layer): @@ -138,6 +130,7 @@ class Http1Layer(_StreamingHttpLayer): layer() +# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. class Http2Layer(_HttpLayer): def __init__(self, ctx, mode): super(Http2Layer, self).__init__(ctx) @@ -359,6 +352,9 @@ class HttpLayer(Layer): return except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: + if flow.request and not flow.response: + flow.error = Error(repr(e)) + self.channel.ask("error", flow) try: self.send_response(make_error_response( getattr(e, "code", 502), @@ -590,87 +586,3 @@ class HttpLayer(Layer): ]) )) raise InvalidCredentials("Proxy Authentication Required") - - -class RequestReplayThread(threading.Thread): - name = "RequestReplayThread" - - def __init__(self, config, flow, masterq, should_exit): - """ - masterqueue can be a queue or None, if no scripthooks should be - processed. - """ - self.config, self.flow = config, flow - if masterq: - self.channel = Channel(masterq, should_exit) - else: - self.channel = None - super(RequestReplayThread, self).__init__() - - def run(self): - r = self.flow.request - form_out_backup = r.form_out - try: - self.flow.response = None - - # If we have a channel, run script hooks. - if self.channel: - request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == KILL: - raise Kill() - elif isinstance(request_reply, HTTPResponse): - self.flow.response = request_reply - - if not self.flow.response: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - server_address = self.config.upstream_server.address - server = ServerConnection(server_address) - server.connect() - protocol = HTTP1Protocol(server) - if r.scheme == "https": - connect_request = make_connect_request((r.host, r.port)) - server.send(protocol.assemble(connect_request)) - resp = protocol.read_response("CONNECT") - if resp.code != 200: - raise HttpError(502, "Upstream server refuses CONNECT request") - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - else: - r.form_out = "absolute" - else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - protocol = HTTP1Protocol(server) - if r.scheme == "https": - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - - server.send(protocol.assemble(r)) - self.flow.server_conn = server - self.flow.response = HTTPResponse.from_protocol( - protocol, - r.method, - body_size_limit=self.config.body_size_limit, - ) - if self.channel: - response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == KILL: - raise Kill() - except (HttpError, tcp.NetLibError) as v: - self.flow.error = Error(repr(v)) - if self.channel: - self.channel.ask("error", self.flow) - except Kill: - # KillSignal should only be raised if there's a channel in the - # first place. - self.channel.tell("log", Log("Connection killed", "info")) - finally: - r.form_out = form_out_backup diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index c24af6cf..b3389eb7 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -13,6 +13,7 @@ class HttpProxy(Layer, ServerConnectionMixin): if self.server_conn: self._disconnect() + class HttpUpstreamProxy(Layer, ServerConnectionMixin): def __init__(self, ctx, server_address): super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) @@ -23,4 +24,4 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): layer() finally: if self.server_conn: - self._disconnect() \ No newline at end of file + self._disconnect() diff --git a/libmproxy/protocol2/http_replay.py b/libmproxy/protocol2/http_replay.py new file mode 100644 index 00000000..872ef9cd --- /dev/null +++ b/libmproxy/protocol2/http_replay.py @@ -0,0 +1,95 @@ +import threading +from netlib.http import HttpError +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError + +from ..controller import Channel +from ..protocol import KILL, Error +from ..protocol.http_wrappers import HTTPResponse +from ..proxy import Log, Kill +from ..proxy.connection import ServerConnection +from .http import make_connect_request + + +class RequestReplayThread(threading.Thread): + name = "RequestReplayThread" + + def __init__(self, config, flow, masterq, should_exit): + """ + masterqueue can be a queue or None, if no scripthooks should be + processed. + """ + self.config, self.flow = config, flow + if masterq: + self.channel = Channel(masterq, should_exit) + else: + self.channel = None + super(RequestReplayThread, self).__init__() + + def run(self): + r = self.flow.request + form_out_backup = r.form_out + try: + self.flow.response = None + + # If we have a channel, run script hooks. + if self.channel: + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == KILL: + raise Kill() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply + + if not self.flow.response: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + server_address = self.config.upstream_server.address + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + connect_request = make_connect_request((r.host, r.port)) + server.send(protocol.assemble(connect_request)) + resp = protocol.read_response("CONNECT") + if resp.code != 200: + raise HttpError(502, "Upstream server refuses CONNECT request") + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + + server.send(protocol.assemble(r)) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_protocol( + protocol, + r.method, + body_size_limit=self.config.body_size_limit, + ) + if self.channel: + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == KILL: + raise Kill() + except (HttpError, NetLibError) as v: + self.flow.error = Error(repr(v)) + if self.channel: + self.channel.ask("error", self.flow) + except Kill: + # KillSignal should only be raised if there's a channel in the + # first place. + self.channel.tell("log", Log("Connection killed", "info")) + finally: + r.form_out = form_out_backup diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index f72320ff..2b47cc26 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -30,8 +30,6 @@ Further goals: inline scripts shall have a chance to handle everything locally. """ from __future__ import (absolute_import, print_function, division) -import Queue -import threading from netlib import tcp from ..proxy import Log from ..proxy.connection import ServerConnection @@ -80,10 +78,8 @@ class Layer(_LayerCodeCompletion): def log(self, msg, level, subs=()): full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] + "{}: {}".format(repr(self.client_conn.address), msg) + ] for i in subs: full_msg.append(" -> " + i) full_msg = "\n".join(full_msg) @@ -119,7 +115,7 @@ class ServerConnectionMixin(object): self.log("Set new server address: " + repr(address), "debug") self.server_conn.address = address else: - self.ctx.set_server(address, server_tls, sni, depth-1) + self.ctx.set_server(address, server_tls, sni, depth - 1) def _disconnect(self): """ @@ -138,10 +134,5 @@ class ServerConnectionMixin(object): try: self.server_conn.connect() except tcp.NetLibError as e: - raise ProtocolException("Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) - - -class Kill(Exception): - """ - Kill a connection. - """ \ No newline at end of file + raise ProtocolException( + "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py deleted file mode 100644 index de049486..00000000 --- a/libmproxy/protocol2/messages.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -This module contains all valid messages layers can send to the underlying layers. -""" -from __future__ import (absolute_import, print_function, division) -from netlib.tcp import Address - - -class _Message(object): - def __eq__(self, other): - # Allow message == Connect checks. - if isinstance(self, other): - return True - return self is other - - def __ne__(self, other): - return not self.__eq__(other) - - -class Connect(_Message): - """ - Connect to the server - """ - - -class Reconnect(_Message): - """ - Re-establish the server connection - """ - - -class SetServer(_Message): - """ - Change the upstream server. - """ - - def __init__(self, address, server_tls, sni, depth=1): - self.address = Address.wrap(address) - self.server_tls = server_tls - self.sni = sni - - # upstream proxy scenario: you may want to change either the final target or the upstream proxy. - # We can express this neatly as the "nth-server-providing-layer" - # ServerConnection could get a `via` attribute. - self.depth = depth - - diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index e8e3cf65..b10217f1 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -4,10 +4,9 @@ import select from OpenSSL import SSL -from ..exceptions import ProtocolException from netlib.tcp import NetLibError from netlib.utils import cleanBin -from ..protocol.tcp import TCPHandler +from ..exceptions import ProtocolException from .layer import Layer @@ -31,6 +30,7 @@ class RawTcpLayer(Layer): while True: r, _, _ = select.select(conns, [], [], 10) for conn in r: + dst = server if conn == client else client size = conn.recv_into(buf, self.chunk_size) if not size: @@ -41,22 +41,21 @@ class RawTcpLayer(Layer): # Sockets will be cleaned up on a higher level. return else: - conn.shutdown(socket.SHUT_WR) + dst.shutdown(socket.SHUT_WR) if len(conns) == 0: return continue - dst = server if conn == client else client dst.sendall(buf[:size]) if self.logging: # log messages are prepended with the client address, # hence the "weird" direction string. if dst == server: - direction = "-> tcp -> {!r}".format(self.server_conn.address) + direction = "-> tcp -> {}".format(repr(self.server_conn.address)) else: - direction = "<- tcp <- {!r}".format(self.server_conn.address) + direction = "<- tcp <- {}".format(repr(self.server_conn.address)) data = cleanBin(buf[:size].tobytes()) self.log( "{}\r\n{}".format(direction, data), @@ -64,4 +63,4 @@ class RawTcpLayer(Layer): ) except (socket.error, NetLibError, SSL.Error) as e: - raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) \ No newline at end of file + raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index 76163c71..e959db86 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -5,17 +5,18 @@ from .tls import TlsLayer class ReverseProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx, server_address, client_tls, server_tls): super(ReverseProxy, self).__init__(ctx, server_address=server_address) self._client_tls = client_tls self._server_tls = server_tls def __call__(self): + # Always use a TLS layer here; if someone changes the scheme, there needs to be a + # TLS layer underneath. layer = TlsLayer(self, self._client_tls, self._server_tls) try: layer() finally: if self.server_conn: - self._disconnect() \ No newline at end of file + self._disconnect() diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index af0e7a37..4d69204f 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -32,7 +32,7 @@ class RootContext(object): # 1. Check for --ignore. if self.config.check_ignore(top_layer.server_conn.address): - return RawTcpLayer(top_layer) + return RawTcpLayer(top_layer, logging=False) # 2. Check for TLS # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 @@ -62,7 +62,8 @@ class RootContext(object): # d = top_layer.client_conn.rfile.peek(3) # is_ascii = ( # len(d) == 3 and - # all(x in string.ascii_letters for x in d) # better be safe here and don't expect uppercase... + # # better be safe here and don't expect uppercase... + # all(x in string.ascii_letters for x in d) # ) # # TODO: This could block if there are not enough bytes available? # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py index 91935d24..525520e8 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/protocol2/socks_proxy.py @@ -1,27 +1,59 @@ from __future__ import (absolute_import, print_function, division) -from ..exceptions import ProtocolException -from ..proxy import ProxyError, Socks5ProxyMode +from netlib import socks +from netlib.tcp import NetLibError +from ..exceptions import Socks5Exception from .layer import Layer, ServerConnectionMixin class Socks5Proxy(Layer, ServerConnectionMixin): def __call__(self): try: - s5mode = Socks5ProxyMode([]) - address = s5mode.get_upstream_server(self.client_conn)[2:] - except ProxyError as e: - # TODO: Unmonkeypatch - raise ProtocolException(str(e), e) + # Parse Client Greeting + client_greet = socks.ClientGreeting.from_file(self.client_conn.rfile, fail_early=True) + client_greet.assert_socks5() + if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: + raise socks.SocksError( + socks.METHOD.NO_ACCEPTABLE_METHODS, + "mitmproxy only supports SOCKS without authentication" + ) - self.server_conn.address = address + # Send Server Greeting + server_greet = socks.ServerGreeting( + socks.VERSION.SOCKS5, + socks.METHOD.NO_AUTHENTICATION_REQUIRED + ) + server_greet.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() - # TODO: Kill event + # Parse Connect Request + connect_request = socks.Message.from_file(self.client_conn.rfile) + connect_request.assert_socks5() + if connect_request.msg != socks.CMD.CONNECT: + raise socks.SocksError( + socks.REP.COMMAND_NOT_SUPPORTED, + "mitmproxy only supports SOCKS5 CONNECT." + ) - layer = self.ctx.next_layer(self) + # We always connect lazily, but we need to pretend to the client that we connected. + connect_reply = socks.Message( + socks.VERSION.SOCKS5, + socks.REP.SUCCEEDED, + connect_request.atyp, + # dummy value, we don't have an upstream connection yet. + connect_request.addr + ) + connect_reply.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + except (socks.SocksError, NetLibError) as e: + raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e) + self.server_conn.address = connect_request.addr + + layer = self.ctx.next_layer(self) try: layer() finally: if self.server_conn: - self._disconnect() \ No newline at end of file + self._disconnect() diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 850bf5dc..0c02b0ea 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -1,11 +1,11 @@ from __future__ import (absolute_import, print_function, division) import struct -from construct import ConstructError -from netlib import tcp -import netlib.http.http2 +from construct import ConstructError +from netlib.tcp import NetLibError, NetLibInvalidCertificateError +from netlib.http.http1 import HTTP1Protocol from ..contrib.tls._constructs import ClientHello from ..exceptions import ProtocolException from .layer import Layer @@ -161,7 +161,7 @@ class TlsLayer(Layer): """ # This gets triggered if we haven't established an upstream connection yet. - default_alpn = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 + default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 if self.alpn_for_client_connection in options: @@ -203,7 +203,7 @@ class TlsLayer(Layer): chain_file=chain_file, alpn_select_callback=self.__alpn_select_callback, ) - except tcp.NetLibError as e: + except NetLibError as e: raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) def _establish_tls_with_server(self): @@ -236,7 +236,7 @@ class TlsLayer(Layer): (tls_cert_err['depth'], tls_cert_err['errno']), "error") self.log("Ignoring server verification error, continuing with connection", "error") - except tcp.NetLibInvalidCertificateError as e: + except NetLibInvalidCertificateError as e: tls_cert_err = self.server_conn.ssl_verification_error self.log( "TLS verification failed for upstream server at depth %s with error: %s" % @@ -244,7 +244,7 @@ class TlsLayer(Layer): "error") self.log("Aborting connection attempt", "error") raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) - except tcp.NetLibError as e: + except NetLibError as e: raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") @@ -253,8 +253,12 @@ class TlsLayer(Layer): host = self.server_conn.address.host sans = set() # Incorporate upstream certificate - if self.server_conn and self.server_conn.tls_established and ( - not self.config.no_upstream_cert): + use_upstream_cert = ( + self.server_conn and + self.server_conn.tls_established and + (not self.config.no_upstream_cert) + ) + if use_upstream_cert: upstream_cert = self.server_conn.cert sans.update(upstream_cert.altnames) if upstream_cert.cn: diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py index 9263dbde..e6ebf115 100644 --- a/libmproxy/protocol2/transparent_proxy.py +++ b/libmproxy/protocol2/transparent_proxy.py @@ -6,7 +6,6 @@ from .layer import Layer, ServerConnectionMixin class TransparentProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx): super(TransparentProxy, self).__init__(ctx) self.resolver = platform.resolver() @@ -22,4 +21,4 @@ class TransparentProxy(Layer, ServerConnectionMixin): layer() finally: if self.server_conn: - self._disconnect() \ No newline at end of file + self._disconnect() diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index 7d664707..709654cb 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1,2 +1,11 @@ -from .primitives import * -from .config import ProxyConfig, process_proxy_options \ No newline at end of file +from __future__ import (absolute_import, print_function, division) + +from .primitives import Log, Kill +from .config import ProxyConfig +from .connection import ClientConnection, ServerConnection + +__all__ = [ + "Log", "Kill", + "ProxyConfig", + "ClientConnection", "ServerConnection" +] \ No newline at end of file diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index c329ed64..94f318f6 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -12,7 +12,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): # Eventually, this object is restored from state. We don't have a # connection then. if client_connection: - tcp.BaseHandler.__init__(self, client_connection, address, server) + super(ClientConnection, self).__init__(client_connection, address, server) else: self.connection = None self.server = None @@ -80,11 +80,11 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): return f def convert_to_ssl(self, *args, **kwargs): - tcp.BaseHandler.convert_to_ssl(self, *args, **kwargs) + super(ClientConnection, self).convert_to_ssl(*args, **kwargs) self.timestamp_ssl_setup = utils.timestamp() def finish(self): - tcp.BaseHandler.finish(self) + super(ClientConnection, self).finish() self.timestamp_end = utils.timestamp() diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index a9f31181..2e440fe8 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -1,178 +1,15 @@ from __future__ import absolute_import +import collections from netlib import socks, tcp -class ProxyError(Exception): - def __init__(self, code, message, headers=None): - super(ProxyError, self).__init__(message) - self.code, self.headers = code, headers - - -class ProxyServerError(Exception): - pass - - -class ProxyMode(object): - http_form_in = None - http_form_out = None - - def get_upstream_server(self, client_conn): - """ - Returns the address of the server to connect to. - Returns None if the address needs to be determined on the protocol level (regular proxy mode) - """ - raise NotImplementedError() # pragma: nocover - - @property - def name(self): - return self.__class__.__name__.replace("ProxyMode", "").lower() - - def __str__(self): - return self.name - - def __eq__(self, other): - """ - Allow comparisions with "regular" etc. - """ - if isinstance(other, ProxyMode): - return self is other - else: - return self.name == other - - def __ne__(self, other): - return not self.__eq__(other) - - -class RegularProxyMode(ProxyMode): - http_form_in = "absolute" - http_form_out = "relative" - - def get_upstream_server(self, client_conn): - return None - - -class SpoofMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def get_upstream_server(self, client_conn): - return None - - @property - def name(self): - return "spoof" - - -class SSLSpoofMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, sslport): - self.sslport = sslport - - def get_upstream_server(self, client_conn): - return None - - @property - def name(self): - return "sslspoof" - - -class TransparentProxyMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, resolver, sslports): - self.resolver = resolver - self.sslports = sslports - - def get_upstream_server(self, client_conn): - try: - dst = self.resolver.original_addr(client_conn.connection) - except Exception as e: - raise ProxyError(502, "Transparent mode failure: %s" % str(e)) - - if dst[1] in self.sslports: - ssl = True - else: - ssl = False - return [ssl, ssl] + list(dst) - - -class Socks5ProxyMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, sslports): - self.sslports = sslports - - def get_upstream_server(self, client_conn): - try: - # Parse Client Greeting - client_greet = socks.ClientGreeting.from_file(client_conn.rfile, fail_early=True) - client_greet.assert_socks5() - if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: - raise socks.SocksError( - socks.METHOD.NO_ACCEPTABLE_METHODS, - "mitmproxy only supports SOCKS without authentication" - ) - - # Send Server Greeting - server_greet = socks.ServerGreeting( - socks.VERSION.SOCKS5, - socks.METHOD.NO_AUTHENTICATION_REQUIRED - ) - server_greet.to_file(client_conn.wfile) - client_conn.wfile.flush() - - # Parse Connect Request - connect_request = socks.Message.from_file(client_conn.rfile) - connect_request.assert_socks5() - if connect_request.msg != socks.CMD.CONNECT: - raise socks.SocksError( - socks.REP.COMMAND_NOT_SUPPORTED, - "mitmproxy only supports SOCKS5 CONNECT." - ) - - # We do not connect here yet, as the clientconnect event has not - # been handled yet. - - connect_reply = socks.Message( - socks.VERSION.SOCKS5, - socks.REP.SUCCEEDED, - connect_request.atyp, - # dummy value, we don't have an upstream connection yet. - connect_request.addr - ) - connect_reply.to_file(client_conn.wfile) - client_conn.wfile.flush() - - ssl = bool(connect_request.addr.port in self.sslports) - return ssl, ssl, connect_request.addr.host, connect_request.addr.port - - except (socks.SocksError, tcp.NetLibError) as e: - raise ProxyError(502, "SOCKS5 mode failure: %s" % repr(e)) - - -class _ConstDestinationProxyMode(ProxyMode): - def __init__(self, dst): - self.dst = dst - - def get_upstream_server(self, client_conn): - return self.dst - - -class ReverseProxyMode(_ConstDestinationProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - -class UpstreamProxyMode(_ConstDestinationProxyMode): - http_form_in = "absolute" - http_form_out = "absolute" - - -class Log: +class Log(object): def __init__(self, msg, level="info"): self.msg = msg self.level = level + + +class Kill(Exception): + """ + Kill a connection. + """ \ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 1fc4cbda..69784014 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -3,16 +3,14 @@ from __future__ import absolute_import, print_function import traceback import sys import socket -from libmproxy.protocol2.layer import Kill from netlib import tcp from netlib.http.http1 import HTTP1Protocol from netlib.tcp import NetLibError -from ..protocol.handle import protocol_handler from .. import protocol2 -from ..exceptions import ProtocolException -from .primitives import ProxyServerError, Log, ProxyError -from .connection import ClientConnection, ServerConnection +from ..exceptions import ProtocolException, ServerException +from .primitives import Log, Kill +from .connection import ClientConnection class DummyServer: @@ -38,9 +36,9 @@ class ProxyServer(tcp.TCPServer): """ self.config = config try: - tcp.TCPServer.__init__(self, (config.host, config.port)) - except socket.error as v: - raise ProxyServerError('Error starting proxy server: ' + repr(v)) + super(ProxyServer, self).__init__((config.host, config.port)) + except socket.error as e: + raise ServerException('Error starting proxy server: ' + repr(e), e) self.channel = None def start_slave(self, klass, channel): @@ -51,33 +49,28 @@ class ProxyServer(tcp.TCPServer): self.channel = channel def handle_client_connection(self, conn, client_address): - h = ConnectionHandler2( - self.config, + h = ConnectionHandler( conn, client_address, - self, - self.channel) + self.config, + self.channel + ) h.handle() - h.finish() -class ConnectionHandler2: - # FIXME: parameter ordering - # FIXME: remove server attribute - def __init__(self, config, client_conn, client_address, server, channel): +class ConnectionHandler(object): + def __init__(self, client_conn, client_address, config, channel): self.config = config """@type: libmproxy.proxy.config.ProxyConfig""" self.client_conn = ClientConnection( client_conn, client_address, - server) + None) """@type: libmproxy.proxy.connection.ClientConnection""" self.channel = channel """@type: libmproxy.controller.Channel""" - def handle(self): - self.log("clientconnect", "info") - + def _create_root_layer(self): root_context = protocol2.RootContext( self.client_conn, self.config, @@ -86,33 +79,38 @@ class ConnectionHandler2: mode = self.config.mode if mode == "upstream": - root_layer = protocol2.HttpUpstreamProxy( + return protocol2.HttpUpstreamProxy( root_context, self.config.upstream_server.address ) elif mode == "transparent": - root_layer = protocol2.TransparentProxy(root_context) + return protocol2.TransparentProxy(root_context) elif mode == "reverse": client_tls = self.config.upstream_server.scheme.startswith("https") server_tls = self.config.upstream_server.scheme.endswith("https") - root_layer = protocol2.ReverseProxy( + return protocol2.ReverseProxy( root_context, self.config.upstream_server.address, client_tls, server_tls ) elif mode == "socks5": - root_layer = protocol2.Socks5Proxy(root_context) + return protocol2.Socks5Proxy(root_context) elif mode == "regular": - root_layer = protocol2.HttpProxy(root_context) + return protocol2.HttpProxy(root_context) elif callable(mode): # pragma: nocover - root_layer = mode(root_context) + return mode(root_context) else: # pragma: nocover raise ValueError("Unknown proxy mode: %s" % mode) + def handle(self): + self.log("clientconnect", "info") + + root_layer = self._create_root_layer() + try: root_layer() - except Kill as e: + except Kill: self.log("Connection killed", "info") except ProtocolException as e: self.log(e, "info") @@ -131,349 +129,8 @@ class ConnectionHandler2: print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) self.log("clientdisconnect", "info") - - def finish(self): - self.client_conn.finish() - - def log(self, msg, level, subs=()): - # FIXME: Duplicate code - full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - -class ConnectionHandler: - def __init__( - self, - config, - client_connection, - client_address, - server, - channel): - self.config = config - """@type: libmproxy.proxy.config.ProxyConfig""" - self.client_conn = ClientConnection( - client_connection, - client_address, - server) - """@type: libmproxy.proxy.connection.ClientConnection""" - self.server_conn = None - """@type: libmproxy.proxy.connection.ServerConnection""" - self.channel = channel - """@type: libmproxy.controller.Channel""" - - self.conntype = "http" - - def handle(self): - try: - self.log("clientconnect", "info") - - # Can we already identify the target server and connect to it? - client_ssl, server_ssl = False, False - conn_kwargs = dict() - upstream_info = self.config.mode.get_upstream_server( - self.client_conn) - if upstream_info: - self.set_server_address(upstream_info[2:]) - client_ssl, server_ssl = upstream_info[:2] - if self.config.check_ignore(self.server_conn.address): - self.log( - "Ignore host: %s:%s" % - self.server_conn.address(), - "info") - self.conntype = "tcp" - conn_kwargs["log"] = False - client_ssl, server_ssl = False, False - else: - # No upstream info from the metadata: upstream info in the - # protocol (e.g. HTTP absolute-form) - pass - - self.channel.ask("clientconnect", self) - - # Check for existing connection: If an inline script already established a - # connection, do not apply client_ssl or server_ssl. - if self.server_conn and not self.server_conn.connection: - self.establish_server_connection() - if client_ssl or server_ssl: - self.establish_ssl(client=client_ssl, server=server_ssl) - - if self.config.check_tcp(self.server_conn.address): - self.log( - "Generic TCP mode for host: %s:%s" % - self.server_conn.address(), - "info") - self.conntype = "tcp" - - elif not self.server_conn and self.config.mode == "sslspoof": - port = self.config.mode.sslport - self.set_server_address(("-", port)) - self.establish_ssl(client=True) - host = self.client_conn.connection.get_servername() - if host: - self.set_server_address((host, port)) - self.establish_server_connection() - self.establish_ssl(server=True, sni=host) - - # Delegate handling to the protocol handler - protocol_handler( - self.conntype)( - self, - **conn_kwargs).handle_messages() - - self.log("clientdisconnect", "info") - self.channel.tell("clientdisconnect", self) - - except ProxyError as e: - protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e) - except Exception: - import traceback - import sys - - self.log(traceback.format_exc(), "error") - print(traceback.format_exc(), file=sys.stderr) - print("mitmproxy has crashed!", file=sys.stderr) - print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) - finally: - # Make sure that we close the server connection in any case. - # The client connection is closed by the ProxyServer and does not - # have be handled here. - self.del_server_connection() - - def del_server_connection(self): - """ - Deletes (and closes) an existing server connection. - """ - if self.server_conn and self.server_conn.connection: - self.server_conn.finish() - self.server_conn.close() - self.log( - "serverdisconnect", "debug", [ - "%s:%s" % - (self.server_conn.address.host, self.server_conn.address.port)]) - self.channel.tell("serverdisconnect", self) - self.server_conn = None - - def set_server_address(self, addr): - """ - Sets a new server address with the given priority. - Does not re-establish either connection or SSL handshake. - """ - address = tcp.Address.wrap(addr) - - # Don't reconnect to the same destination. - if self.server_conn and self.server_conn.address == address: - return - - if self.server_conn: - self.del_server_connection() - - self.log( - "Set new server address: %s:%s" % - (address.host, address.port), "debug") - self.server_conn = ServerConnection(address) - - def establish_server_connection(self, ask=True): - """ - Establishes a new server connection. - If there is already an existing server connection, the function returns immediately. - - By default, this function ".ask"s the proxy master. This is deadly if this function is already called from the - master (e.g. via change_server), because this navigates us in a simple deadlock (the master is single-threaded). - In these scenarios, ask=False can be passed to suppress the call to the master. - """ - if self.server_conn.connection: - return - self.log( - "serverconnect", "debug", [ - "%s:%s" % - self.server_conn.address()[ - :2]]) - if ask: - self.channel.ask("serverconnect", self) - try: - self.server_conn.connect() - except tcp.NetLibError as v: - raise ProxyError(502, v) - - def establish_ssl(self, client=False, server=False, sni=None): - """ - Establishes SSL on the existing connection(s) to the server or the client, - as specified by the parameters. - """ - - # Logging - if client or server: - subs = [] - if client: - subs.append("with client") - if server: - subs.append("with server (sni: %s)" % sni) - self.log("Establish SSL", "debug", subs) - - if server: - if not self.server_conn or not self.server_conn.connection: - raise ProxyError(502, "No server connection.") - if self.server_conn.ssl_established: - raise ProxyError(502, "SSL to Server already established.") - try: - self.server_conn.establish_ssl( - self.config.clientcerts, - sni, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify_options=self.config.openssl_verification_mode_server, - ca_path=self.config.openssl_trusted_cadir_server, - ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, - ) - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") - except tcp.NetLibError as v: - e = ProxyError(502, repr(v)) - # Workaround for https://github.com/mitmproxy/mitmproxy/issues/427 - # The upstream server may reject connections without SNI, which means we need to - # establish SSL with the client first, hope for a SNI (which triggers a reconnect which replaces the - # ServerConnection object) and see whether that worked. - if client and "handshake failure" in e.message: - self.server_conn.may_require_sni = e - else: - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") - raise e - if client: - if self.client_conn.ssl_established: - raise ProxyError(502, "SSL to Client already established.") - cert, key, chain_file = self.find_cert() - try: - self.client_conn.convert_to_ssl( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - handle_sni=self.handle_sni, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - except tcp.NetLibError as v: - raise ProxyError(400, repr(v)) - - # Workaround for #427 part 2 - if server and hasattr(self.server_conn, "may_require_sni"): - raise self.server_conn.may_require_sni - - def server_reconnect(self, new_sni=False): - address = self.server_conn.address - had_ssl = self.server_conn.ssl_established - state = self.server_conn.state - sni = new_sni or self.server_conn.sni - self.log("(server reconnect follows)", "debug") - self.del_server_connection() - self.set_server_address(address) - self.establish_server_connection() - - for s in state: - protocol_handler(s[0])(self).handle_server_reconnect(s[1]) - self.server_conn.state = state - - # Receiving new_sni where had_ssl is False is a weird case that happens when the workaround for - # https://github.com/mitmproxy/mitmproxy/issues/427 is active. In this - # case, we want to establish SSL as well. - if had_ssl or new_sni: - self.establish_ssl(server=True, sni=sni) - - def finish(self): self.client_conn.finish() - def log(self, msg, level, subs=()): - full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - def find_cert(self): - host = self.server_conn.address.host - sans = [] - if self.server_conn.ssl_established and ( - not self.config.no_upstream_cert): - upstream_cert = self.server_conn.cert - sans.extend(upstream_cert.altnames) - if upstream_cert.cn: - sans.append(host) - host = upstream_cert.cn.decode("utf8").encode("idna") - if self.server_conn.sni: - sans.append(self.server_conn.sni) - # for ssl spoof mode - if hasattr(self.client_conn, "sni"): - sans.append(self.client_conn.sni) - - ret = self.config.certstore.get_cert(host, sans) - if not ret: - raise ProxyError(502, "Unable to generate dummy cert.") - return ret - - def handle_sni(self, connection): - """ - This callback gets called during the SSL handshake with the client. - The client has just sent the Sever Name Indication (SNI). We now connect upstream to - figure out which certificate needs to be served. - """ - try: - sn = connection.get_servername() - if not sn: - return - sni = sn.decode("utf8").encode("idna") - # for ssl spoof mode - self.client_conn.sni = sni - - if sni != self.server_conn.sni: - self.log("SNI received: %s" % sni, "debug") - # We should only re-establish upstream SSL if one of the following conditions is true: - # - We established SSL with the server previously - # - We initially wanted to establish SSL with the server, - # but the server refused to negotiate without SNI. - if self.server_conn.ssl_established or hasattr( - self.server_conn, - "may_require_sni"): - # reconnect to upstream server with SNI - self.server_reconnect(sni) - # Now, change client context to reflect changed certificate: - cert, key, chain_file = self.find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - connection.set_context(new_context) - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except: # pragma: no cover - import traceback - self.log( - "Error in handle_sni:\r\n" + - traceback.format_exc(), - "error") + def log(self, msg, level): + msg = "{}: {}".format(repr(self.client_conn.address), msg) + self.channel.tell("log", Log(msg, level)) \ No newline at end of file diff --git a/libmproxy/utils.py b/libmproxy/utils.py index 3ac3cc01..a6ca55f7 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -1,14 +1,10 @@ from __future__ import absolute_import import os import datetime -import urllib import re import time -import functools -import cgi import json -import netlib.utils def timestamp(): """ -- cgit v1.2.3 From 1dd09a5509219e7390abbb8c0b6818c7e792daa1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 02:27:38 +0200 Subject: always insert tls layer for inline script upgrades --- libmproxy/cmdline.py | 13 +++---------- libmproxy/protocol2/http_proxy.py | 5 ++--- libmproxy/protocol2/reverse_proxy.py | 10 +++------- libmproxy/protocol2/root_context.py | 33 +++++++++++++++++++++++++-------- libmproxy/protocol2/tls.py | 13 +++++++++++++ libmproxy/proxy/config.py | 2 ++ libmproxy/proxy/server.py | 4 +--- 7 files changed, 49 insertions(+), 31 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 591e87ed..55377af2 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -102,9 +102,9 @@ def parse_setheader(s): return _parse_hook(s) -def parse_server_spec(url, allowed_schemes=("http", "https")): +def parse_server_spec(url): p = netlib.utils.parse_url(url) - if not p or not p[1] or p[0] not in allowed_schemes: + if not p or not p[1] or p[0] not in ("http", "https"): raise configargparse.ArgumentTypeError( "Invalid server specification: %s" % url ) @@ -113,13 +113,6 @@ def parse_server_spec(url, allowed_schemes=("http", "https")): return config.ServerSpec(scheme, address) -def parse_server_spec_special(url): - """ - Provides additional support for http2https and https2http schemes. - """ - return parse_server_spec(url, allowed_schemes=("http", "https", "http2https", "https2http")) - - def get_common_options(options): stickycookie, stickyauth = None, None if options.stickycookie_filt: @@ -297,7 +290,7 @@ def proxy_modes(parser): group.add_argument( "-R", "--reverse", action="store", - type=parse_server_spec_special, + type=parse_server_spec, dest="reverse_proxy", default=None, help=""" diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index b3389eb7..2876c022 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -1,12 +1,11 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin -from .http import Http1Layer class HttpProxy(Layer, ServerConnectionMixin): def __call__(self): - layer = Http1Layer(self, "regular") + layer = self.ctx.next_layer(self) try: layer() finally: @@ -19,7 +18,7 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) def __call__(self): - layer = Http1Layer(self, "upstream") + layer = self.ctx.next_layer(self) try: layer() finally: diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index e959db86..c4cabccc 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -5,16 +5,12 @@ from .tls import TlsLayer class ReverseProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx, server_address, client_tls, server_tls): + def __init__(self, ctx, server_address, server_tls): super(ReverseProxy, self).__init__(ctx, server_address=server_address) - self._client_tls = client_tls - self._server_tls = server_tls + self.server_tls = server_tls def __call__(self): - # Always use a TLS layer here; if someone changes the scheme, there needs to be a - # TLS layer underneath. - layer = TlsLayer(self, self._client_tls, self._server_tls) - + layer = self.ctx.next_layer(self) try: layer() finally: diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 4d69204f..210ba6ab 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -6,7 +6,9 @@ from netlib.http.http2 import HTTP2Protocol from .rawtcp import RawTcpLayer from .tls import TlsLayer, is_tls_record_magic from .http import Http1Layer, Http2Layer - +from .layer import ServerConnectionMixin +from .http_proxy import HttpProxy, HttpUpstreamProxy +from .reverse_proxy import ReverseProxy class RootContext(object): """ @@ -34,18 +36,33 @@ class RootContext(object): if self.config.check_ignore(top_layer.server_conn.address): return RawTcpLayer(top_layer, logging=False) - # 2. Check for TLS - # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 - # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello d = top_layer.client_conn.rfile.peek(3) - if is_tls_record_magic(d): + client_tls = is_tls_record_magic(d) + + # 2. Always insert a TLS layer, even if there's neither client nor server tls. + # An inline script may upgrade from http to https, + # in which case we need some form of TLS layer. + if isinstance(top_layer, ReverseProxy): + return TlsLayer(top_layer, client_tls, top_layer.server_tls) + if isinstance(top_layer, ServerConnectionMixin): + return TlsLayer(top_layer, client_tls, client_tls) + + # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. + if isinstance(top_layer, TlsLayer): + if isinstance(top_layer.ctx, HttpProxy): + return Http1Layer(top_layer, "regular") + if isinstance(top_layer.ctx, HttpUpstreamProxy): + return Http1Layer(top_layer, "upstream") + + # 4. Check for other TLS cases (e.g. after CONNECT). + if client_tls: return TlsLayer(top_layer, True, True) - # 3. Check for --tcp + # 4. Check for --tcp if self.config.check_tcp(top_layer.server_conn.address): return RawTcpLayer(top_layer) - # 4. Check for TLS ALPN (HTTP1/HTTP2) + # 5. Check for TLS ALPN (HTTP1/HTTP2) if isinstance(top_layer, TlsLayer): alpn = top_layer.client_conn.get_alpn_proto_negotiated() if alpn == HTTP2Protocol.ALPN_PROTO_H2: @@ -53,7 +70,7 @@ class RootContext(object): if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: return Http1Layer(top_layer, 'transparent') - # 5. Assume HTTP1 by default + # 6. Assume HTTP1 by default return Http1Layer(top_layer, 'transparent') # In a future version, we want to implement TCP passthrough as the last fallback, diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 0c02b0ea..041adaaa 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -18,6 +18,9 @@ def is_tls_record_magic(d): False, otherwise. """ d = d[:3] + + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello return ( len(d) == 3 and d[0] == '\x16' and @@ -73,6 +76,16 @@ class TlsLayer(Layer): layer = self.ctx.next_layer(self) layer() + def __repr__(self): + if self._client_tls and self._server_tls: + return "TlsLayer(client and server)" + elif self._client_tls: + return "TlsLayer(client)" + elif self._server_tls: + return "TlsLayer(server)" + else: + return "TlsLayer(inactive)" + def _get_client_hello(self): """ Peek into the socket and read all records that contain the initial client hello message. diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 415ee215..b360abbd 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -24,6 +24,8 @@ class HostMatcher(object): self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] def __call__(self, address): + if not address: + return False address = tcp.Address.wrap(address) host = "%s:%s" % (address.host, address.port) if any(rex.search(host) for rex in self.regexes): diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 69784014..5abd0877 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -86,12 +86,10 @@ class ConnectionHandler(object): elif mode == "transparent": return protocol2.TransparentProxy(root_context) elif mode == "reverse": - client_tls = self.config.upstream_server.scheme.startswith("https") - server_tls = self.config.upstream_server.scheme.endswith("https") + server_tls = self.config.upstream_server.scheme == "https" return protocol2.ReverseProxy( root_context, self.config.upstream_server.address, - client_tls, server_tls ) elif mode == "socks5": -- cgit v1.2.3 From 21e7f420d2870d89ebc05181c1fca674d80e4e7c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 03:23:57 +0200 Subject: minor fixes --- libmproxy/main.py | 6 +++--- libmproxy/protocol2/tls.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/main.py b/libmproxy/main.py index 4dd6fdb1..faef8c82 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -2,11 +2,11 @@ from __future__ import print_function, absolute_import import os import signal import sys -import netlib.version from netlib.version_check import check_pyopenssl_version, check_mitmproxy_version from . import version, cmdline -from .proxy import process_proxy_options, ProxyServerError +from .exceptions import ServerException from .proxy.server import DummyServer, ProxyServer +from .proxy.config import process_proxy_options def assert_utf8_env(): @@ -31,7 +31,7 @@ def get_server(dummy_server, options): else: try: return ProxyServer(options) - except ProxyServerError as v: + except ServerException as v: print(str(v), file=sys.stderr) sys.exit(1) diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 041adaaa..73bb12f3 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -66,7 +66,8 @@ class TlsLayer(Layer): self._client_tls and self._server_tls and not self.config.no_upstream_cert ) - self._parse_client_hello() + if self._client_tls: + self._parse_client_hello() if client_tls_requires_server_cert: self._establish_tls_with_client_and_server() -- cgit v1.2.3 From 3873e08339fd701738a1522af32e37363fcec14b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 03:42:11 +0200 Subject: remove old code --- libmproxy/flow.py | 7 +- libmproxy/protocol/handle.py | 20 -- libmproxy/protocol/http.py | 608 +----------------------------------- libmproxy/protocol/http_wrappers.py | 40 +-- libmproxy/protocol/primitives.py | 130 +------- libmproxy/protocol/tcp.py | 97 ------ 6 files changed, 22 insertions(+), 880 deletions(-) delete mode 100644 libmproxy/protocol/handle.py delete mode 100644 libmproxy/protocol/tcp.py (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index dac607a0..a2f57512 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,15 +8,16 @@ import Cookie import cookielib import os import re +from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol2.http_replay import RequestReplayThread -from netlib import odict, wsgi, tcp +from netlib import odict, wsgi from netlib.http.semantics import CONTENT_MISSING import netlib.http from . import controller, protocol, tnetstring, filt, script, version from .onboarding import app -from .protocol import http, handle +from .protocol import http from .proxy.config import HostMatcher from .proxy.connection import ClientConnection, ServerConnection import urlparse @@ -1090,7 +1091,7 @@ class FlowReader: "Incompatible serialized data version: %s" % v ) off = self.fo.tell() - yield handle.protocols[data["type"]]["flow"].from_state(data) + yield HTTPFlow.from_state(data) except ValueError as v: # Error is due to EOF if self.fo.tell() == off and self.fo.read() == '': diff --git a/libmproxy/protocol/handle.py b/libmproxy/protocol/handle.py deleted file mode 100644 index 49cb3c1b..00000000 --- a/libmproxy/protocol/handle.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import absolute_import -from . import http, tcp - -protocols = { - 'http': dict(handler=http.HTTPHandler, flow=http.HTTPFlow), - 'tcp': dict(handler=tcp.TCPHandler) -} - - -def protocol_handler(protocol): - """ - @type protocol: str - @returns: libmproxy.protocol.primitives.ProtocolHandler - """ - if protocol in protocols: - return protocols[protocol]["handler"] - - raise NotImplementedError( - "Unknown Protocol: %s" % - protocol) # pragma: nocover diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index a30437d1..bde7b088 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,62 +1,9 @@ from __future__ import absolute_import -import Cookie -import copy -import threading -import time -import urllib -import urlparse -from email.utils import parsedate_tz, formatdate, mktime_tz -import netlib -from netlib import http, tcp, odict, utils, encoding -from netlib.http import cookies, http1, http2 -from netlib.http.http1 import HTTP1Protocol -from netlib.http.semantics import CONTENT_MISSING - -from .tcp import TCPHandler -from .primitives import KILL, ProtocolHandler, Flow, Error -from ..proxy.connection import ServerConnection -from .. import utils, controller, stateobject, proxy +from .primitives import Flow from .http_wrappers import decoded, HTTPRequest, HTTPResponse - -class KillSignal(Exception): - pass - - -def send_connect_request(conn, host, port, update_state=True): - upstream_request = HTTPRequest( - "authority", - "CONNECT", - None, - host, - port, - None, - (1, 1), - odict.ODictCaseless(), - "" - ) - - # we currently only support HTTP/1 CONNECT requests - protocol = http1.HTTP1Protocol(conn) - - conn.send(protocol.assemble(upstream_request)) - resp = HTTPResponse.from_protocol(protocol, upstream_request.method) - if resp.status_code != 200: - raise proxy.ProxyError(resp.status_code, - "Cannot establish SSL " + - "connection with upstream proxy: \r\n" + - repr(resp)) - if update_state: - conn.state.append(("http", { - "state": "connect", - "host": host, - "port": port} - )) - return resp - - class HTTPFlow(Flow): """ A HTTPFlow is a collection of objects representing a single HTTP @@ -143,556 +90,3 @@ class HTTPFlow(Flow): if self.response: c += self.response.replace(pattern, repl, *args, **kwargs) return c - - -class HTTPHandler(ProtocolHandler): - """ - HTTPHandler implements mitmproxys understanding of the HTTP protocol. - - """ - - def __init__(self, c): - super(HTTPHandler, self).__init__(c) - self.expected_form_in = c.config.mode.http_form_in - self.expected_form_out = c.config.mode.http_form_out - self.skip_authentication = False - - def handle_messages(self): - while self.handle_flow(): - pass - - def get_response_from_server(self, flow): - self.c.establish_server_connection() - - for attempt in (0, 1): - try: - if not self.c.server_conn.protocol: - # instantiate new protocol if connection does not have one yet - # TODO: select correct protocol based on ALPN (?) - self.c.server_conn.protocol = http1.HTTP1Protocol(self.c.server_conn) - # self.c.server_conn.protocol = http2.HTTP2Protocol(self.c.server_conn) - # self.c.server_conn.protocol.perform_connection_preface() - - self.c.server_conn.send(self.c.server_conn.protocol.assemble(flow.request)) - - # Only get the headers at first... - flow.response = HTTPResponse.from_protocol( - self.c.server_conn.protocol, - flow.request.method, - body_size_limit=self.c.config.body_size_limit, - include_body=False, - ) - break - except (tcp.NetLibError, http.HttpErrorConnClosed) as v: - self.c.log( - "error in server communication: %s" % repr(v), - level="debug" - ) - if attempt == 0: - # In any case, we try to reconnect at least once. This is - # necessary because it might be possible that we already - # initiated an upstream connection after clientconnect that - # has already been expired, e.g consider the following event - # log: - # > clientconnect (transparent mode destination known) - # > serverconnect - # > read n% of large request - # > server detects timeout, disconnects - # > read (100-n)% of large request - # > send large request upstream - self.c.server_reconnect() - else: - raise - - # call the appropriate script hook - this is an opportunity for an - # inline script to set flow.stream = True - flow = self.c.channel.ask("responseheaders", flow) - if flow is None or flow == KILL: - raise KillSignal() - else: - # now get the rest of the request body, if body still needs to be - # read but not streaming this response - if flow.response.stream: - flow.response.content = CONTENT_MISSING - else: - if isinstance(self.c.server_conn.protocol, http1.HTTP1Protocol): - # streaming is only supported with HTTP/1 at the moment - flow.response.content = self.c.server_conn.protocol.read_http_body( - flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, - flow.response.code, - False - ) - flow.response.timestamp_end = utils.timestamp() - - def handle_flow(self): - flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live) - - try: - try: - if not flow.client_conn.protocol: - # instantiate new protocol if connection does not have one yet - # the first request might be a CONNECT - which is currently only supported with HTTP/1 - flow.client_conn.protocol = http1.HTTP1Protocol(self.c.client_conn) - - req = HTTPRequest.from_protocol( - flow.client_conn.protocol, - body_size_limit=self.c.config.body_size_limit - ) - except tcp.NetLibError: - # don't throw an error for disconnects that happen - # before/between requests. - return False - - self.c.log( - "request", - "debug", - [repr(req)] - ) - ret = self.process_request(flow, req) - if ret: - # instantiate new protocol if connection does not have one yet - # TODO: select correct protocol based on ALPN (?) - flow.client_conn.protocol = http1.HTTP1Protocol(self.c.client_conn) - # flow.client_conn.protocol = http2.HTTP2Protocol(self.c.client_conn, is_server=True) - if ret is not None: - return ret - - # Be careful NOT to assign the request to the flow before - # process_request completes. This is because the call can raise an - # exception. If the request object is already attached, this results - # in an Error object that has an attached request that has not been - # sent through to the Master. - flow.request = req - request_reply = self.c.channel.ask("request", flow) - if request_reply is None or request_reply == KILL: - raise KillSignal() - - # The inline script may have changed request.host - self.process_server_address(flow) - - if isinstance(request_reply, HTTPResponse): - flow.response = request_reply - else: - self.get_response_from_server(flow) - - # no further manipulation of self.c.server_conn beyond this point - # we can safely set it as the final attribute value here. - flow.server_conn = self.c.server_conn - - self.c.log( - "response", - "debug", - [repr(flow.response)] - ) - response_reply = self.c.channel.ask("response", flow) - if response_reply is None or response_reply == KILL: - raise KillSignal() - - self.send_response_to_client(flow) - - if self.check_close_connection(flow): - return False - - # We sent a CONNECT request to an upstream proxy. - if flow.request.form_in == "authority" and flow.response.code == 200: - # TODO: Possibly add headers (memory consumption/usefulness - # tradeoff) Make sure to add state info before the actual - # processing of the CONNECT request happens. During an SSL - # upgrade, we may receive an SNI indication from the client, - # which resets the upstream connection. If this is the case, we - # must already re-issue the CONNECT request at this point. - self.c.server_conn.state.append( - ( - "http", { - "state": "connect", - "host": flow.request.host, - "port": flow.request.port - } - ) - ) - if not self.process_connect_request( - (flow.request.host, flow.request.port)): - return False - - # If the user has changed the target server on this connection, - # restore the original target server - flow.live.restore_server() - - return True # Next flow please. - except ( - http.HttpAuthenticationError, - http.HttpError, - proxy.ProxyError, - tcp.NetLibError, - ) as e: - self.handle_error(e, flow) - except KillSignal: - self.c.log("Connection killed", "info") - finally: - flow.live = None # Connection is not live anymore. - return False - - def handle_server_reconnect(self, state): - if state["state"] == "connect": - send_connect_request( - self.c.server_conn, - state["host"], - state["port"], - update_state=False - ) - else: # pragma: nocover - raise RuntimeError("Unknown State: %s" % state["state"]) - - def handle_error(self, error, flow=None): - message = repr(error) - message_debug = None - - if isinstance(error, tcp.NetLibError): - message = None - message_debug = "TCP connection closed unexpectedly." - elif "tlsv1 alert unknown ca" in message: - message = "TLSv1 Alert Unknown CA: The client does not trust the proxy's certificate." - elif "handshake error" in message: - message_debug = message - message = "SSL handshake error: The client may not trust the proxy's certificate." - - if message: - self.c.log(message, level="info") - if message_debug: - self.c.log(message_debug, level="debug") - - if flow: - # TODO: no flows without request or with both request and response - # at the moment. - if flow.request and not flow.response: - flow.error = Error(message or message_debug) - self.c.channel.ask("error", flow) - try: - status_code = getattr(error, "code", 502) - headers = getattr(error, "headers", None) - - html_message = message or "" - if message_debug: - html_message += "
%s
" % message_debug - self.send_error(status_code, html_message, headers) - except: - pass - - def send_error(self, status_code, message, headers): - response = http.status_codes.RESPONSES.get(status_code, "Unknown") - body = """ - - - %d %s - - %s - - """ % (status_code, response, message) - - if not headers: - headers = odict.ODictCaseless() - assert isinstance(headers, odict.ODictCaseless) - - headers["Server"] = [self.c.config.server_version] - headers["Connection"] = ["close"] - headers["Content-Length"] = [len(body)] - headers["Content-Type"] = ["text/html"] - - resp = HTTPResponse( - (1, 1), # if HTTP/2 is used, this value is ignored anyway - status_code, - response, - headers, - body, - ) - - # if no protocol is assigned yet - just assume HTTP/1 - # TODO: maybe check ALPN and use HTTP/2 if required? - protocol = self.c.client_conn.protocol or http1.HTTP1Protocol(self.c.client_conn) - self.c.client_conn.send(protocol.assemble(resp)) - - def process_request(self, flow, request): - """ - @returns: - True, if the request should not be sent upstream - False, if the connection should be aborted - None, if the request should be sent upstream - (a status code != None should be returned directly by handle_flow) - """ - - if not self.skip_authentication: - self.authenticate(request) - - # Determine .scheme, .host and .port attributes - # For absolute-form requests, they are directly given in the request. - # For authority-form requests, we only need to determine the request scheme. - # For relative-form requests, we need to determine host and port as - # well. - if not request.scheme: - request.scheme = "https" if flow.server_conn and flow.server_conn.ssl_established else "http" - if not request.host: - # Host/Port Complication: In upstream mode, use the server we CONNECTed to, - # not the upstream proxy. - if flow.server_conn: - for s in flow.server_conn.state: - if s[0] == "http" and s[1]["state"] == "connect": - request.host, request.port = s[1]["host"], s[1]["port"] - if not request.host and flow.server_conn: - request.host, request.port = flow.server_conn.address.host, flow.server_conn.address.port - - - # Now we can process the request. - if request.form_in == "authority": - if self.c.client_conn.ssl_established: - raise http.HttpError( - 400, - "Must not CONNECT on already encrypted connection" - ) - - if self.c.config.mode == "regular": - self.c.set_server_address((request.host, request.port)) - # Update server_conn attribute on the flow - flow.server_conn = self.c.server_conn - - # since we currently only support HTTP/1 CONNECT requests - # the response must be HTTP/1 as well - self.c.client_conn.send( - ('HTTP/%s.%s 200 ' % (request.httpversion[0], request.httpversion[1])) + - 'Connection established\r\n' + - 'Content-Length: 0\r\n' + - ('Proxy-agent: %s\r\n' % self.c.config.server_version) + - '\r\n' - ) - return self.process_connect_request(self.c.server_conn.address) - elif self.c.config.mode == "upstream": - return None - else: - # CONNECT should never occur if we don't expect absolute-form - # requests - pass - - elif request.form_in == self.expected_form_in: - request.form_out = self.expected_form_out - if request.form_in == "absolute": - if request.scheme != "http": - raise http.HttpError( - 400, - "Invalid request scheme: %s" % request.scheme - ) - if self.c.config.mode == "regular": - # Update info so that an inline script sees the correct - # value at flow.server_conn - self.c.set_server_address((request.host, request.port)) - flow.server_conn = self.c.server_conn - - elif request.form_in == "relative": - if self.c.config.mode == "spoof": - # Host header - h = request.pretty_host(hostheader=True) - if h is None: - raise http.HttpError( - 400, - "Invalid request: No host information" - ) - p = netlib.utils.parse_url("http://" + h) - request.scheme = p[0] - request.host = p[1] - request.port = p[2] - self.c.set_server_address((request.host, request.port)) - flow.server_conn = self.c.server_conn - - if self.c.config.mode == "sslspoof": - # SNI is processed in server.py - if not (flow.server_conn and flow.server_conn.ssl_established): - raise http.HttpError( - 400, - "Invalid request: No host information" - ) - - return None - - raise http.HttpError( - 400, "Invalid HTTP request form (expected: %s, got: %s)" % ( - self.expected_form_in, request.form_in - ) - ) - - def process_server_address(self, flow): - # Depending on the proxy mode, server handling is entirely different - # We provide a mostly unified API to the user, which needs to be - # unfiddled here - # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 ) - address = tcp.Address((flow.request.host, flow.request.port)) - - ssl = (flow.request.scheme == "https") - - if self.c.config.mode == "upstream": - # The connection to the upstream proxy may have a state we may need - # to take into account. - connected_to = None - for s in flow.server_conn.state: - if s[0] == "http" and s[1]["state"] == "connect": - connected_to = tcp.Address((s[1]["host"], s[1]["port"])) - - # We need to reconnect if the current flow either requires a - # (possibly impossible) change to the connection state, e.g. the - # host has changed but we already CONNECTed somewhere else. - needs_server_change = ( - ssl != self.c.server_conn.ssl_established - or - # HTTP proxying is "stateless", CONNECT isn't. - (connected_to and address != connected_to) - ) - - if needs_server_change: - # force create new connection to the proxy server to reset - # state - self.live.change_server(self.c.server_conn.address, force=True) - if ssl: - send_connect_request( - self.c.server_conn, - address.host, - address.port - ) - self.c.establish_ssl(server=True) - else: - # If we're not in upstream mode, we just want to update the host - # and possibly establish TLS. This is a no op if the addresses - # match. - self.live.change_server(address, ssl=ssl) - - flow.server_conn = self.c.server_conn - - def send_response_to_client(self, flow): - if not flow.response.stream: - # no streaming: - # we already received the full response from the server and can - # send it to the client straight away. - self.c.client_conn.send(self.c.client_conn.protocol.assemble(flow.response)) - else: - if isinstance(self.c.client_conn.protocol, http2.HTTP2Protocol): - raise NotImplementedError("HTTP streaming with HTTP/2 is currently not supported.") - - - # streaming: - # First send the headers and then transfer the response - # incrementally: - h = self.c.client_conn.protocol._assemble_response_first_line(flow.response) - self.c.client_conn.send(h + "\r\n") - h = self.c.client_conn.protocol._assemble_response_headers(flow.response, preserve_transfer_encoding=True) - self.c.client_conn.send(h + "\r\n") - - chunks = self.c.server_conn.protocol.read_http_body_chunked( - flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, - flow.response.code, - False, - 4096 - ) - - if callable(flow.response.stream): - chunks = flow.response.stream(chunks) - - for chunk in chunks: - for part in chunk: - self.c.client_conn.wfile.write(part) - self.c.client_conn.wfile.flush() - - flow.response.timestamp_end = utils.timestamp() - - def check_close_connection(self, flow): - """ - Checks if the connection should be closed depending on the HTTP - semantics. Returns True, if so. - """ - - # TODO: add logic for HTTP/2 - - close_connection = ( - http1.HTTP1Protocol.connection_close( - flow.request.httpversion, - flow.request.headers - ) or http1.HTTP1Protocol.connection_close( - flow.response.httpversion, - flow.response.headers - ) or http1.HTTP1Protocol.expected_http_body_size( - flow.response.headers, - False, - flow.request.method, - flow.response.code) == -1 - ) - if close_connection: - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 - # and no Content-Length header - pass - else: - return True - return False - - def process_connect_request(self, address): - """ - Process a CONNECT request. - Returns True if the CONNECT request has been processed successfully. - Returns False, if the connection should be closed immediately. - """ - address = tcp.Address.wrap(address) - if self.c.config.check_ignore(address): - self.c.log("Ignore host: %s:%s" % address(), "info") - TCPHandler(self.c, log=False).handle_messages() - return False - else: - self.expected_form_in = "relative" - self.expected_form_out = "relative" - self.skip_authentication = True - - # In practice, nobody issues a CONNECT request to send unencrypted - # HTTP requests afterwards. If we don't delegate to TCP mode, we - # should always negotiate a SSL connection. - # - # FIXME: Turns out the previous statement isn't entirely true. - # Chrome on Windows CONNECTs to :80 if an explicit proxy is - # configured and a websocket connection should be established. We - # don't support websocket at the moment, so it fails anyway, but we - # should come up with a better solution to this if we start to - # support WebSockets. - should_establish_ssl = ( - address.port in self.c.config.ssl_ports - or - not self.c.config.check_tcp(address) - ) - - if should_establish_ssl: - self.c.log( - "Received CONNECT request to SSL port. " - "Upgrading to SSL...", "debug" - ) - server_ssl = not self.c.config.no_upstream_cert - if server_ssl: - self.c.establish_server_connection() - self.c.establish_ssl(server=server_ssl, client=True) - self.c.log("Upgrade to SSL completed.", "debug") - - if self.c.config.check_tcp(address): - self.c.log( - "Generic TCP mode for host: %s:%s" % address(), - "info" - ) - TCPHandler(self.c).handle_messages() - return False - - return True - - def authenticate(self, request): - if self.c.config.authenticator: - if self.c.config.authenticator.authenticate(request.headers): - self.c.config.authenticator.clean(request.headers) - else: - raise http.HttpAuthenticationError( - self.c.config.authenticator.auth_challenge_headers()) - return request.headers \ No newline at end of file diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index b1000a79..a26ddbb4 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -1,20 +1,12 @@ from __future__ import absolute_import import Cookie import copy -import threading import time -import urllib -import urlparse from email.utils import parsedate_tz, formatdate, mktime_tz -import netlib -from netlib import http, tcp, odict, utils, encoding -from netlib.http import cookies, semantics, http1 - -from .tcp import TCPHandler -from .primitives import KILL, ProtocolHandler, Flow, Error -from ..proxy.connection import ServerConnection -from .. import utils, controller, stateobject, proxy +from netlib import odict, encoding +from netlib.http import semantics, CONTENT_MISSING +from .. import utils, stateobject class decoded(object): @@ -170,19 +162,19 @@ class HTTPRequest(MessageMixin, semantics.Request): """ def __init__( - self, - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - body, - timestamp_start=None, - timestamp_end=None, - form_out=None, + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + body, + timestamp_start=None, + timestamp_end=None, + form_out=None, ): semantics.Request.__init__( self, diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py index 92fc95e5..c663f0c5 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/protocol/primitives.py @@ -1,11 +1,10 @@ from __future__ import absolute_import import copy import uuid -import netlib.tcp + from .. import stateobject, utils, version from ..proxy.connection import ClientConnection, ServerConnection - KILL = 0 # const for killed requests @@ -165,130 +164,3 @@ class Flow(stateobject.StateObject): self.intercepted = False self.reply() master.handle_accept_intercept(self) - - - -class ProtocolHandler(object): - """ - A ProtocolHandler implements an application-layer protocol, e.g. HTTP. - See: libmproxy.protocol.http.HTTPHandler - """ - - def __init__(self, c): - self.c = c - """@type: libmproxy.proxy.server.ConnectionHandler""" - self.live = LiveConnection(c) - """@type: LiveConnection""" - - def handle_messages(self): - """ - This method gets called if a client connection has been made. Depending - on the proxy settings, a server connection might already exist as well. - """ - raise NotImplementedError # pragma: nocover - - def handle_server_reconnect(self, state): - """ - This method gets called if a server connection needs to reconnect and - there's a state associated with the server connection (e.g. a - previously-sent CONNECT request or a SOCKS proxy request). This method - gets called after the connection has been restablished but before SSL is - established. - """ - raise NotImplementedError # pragma: nocover - - def handle_error(self, error): - """ - This method gets called should there be an uncaught exception during the - connection. This might happen outside of handle_messages, e.g. if the - initial SSL handshake fails in transparent mode. - """ - raise error # pragma: nocover - - -class LiveConnection(object): - """ - This facade allows interested parties (FlowMaster, inline scripts) to - interface with a live connection, without exposing the internals - of the ConnectionHandler. - """ - - def __init__(self, c): - self.c = c - """@type: libmproxy.proxy.server.ConnectionHandler""" - self._backup_server_conn = None - """@type: libmproxy.proxy.connection.ServerConnection""" - - def change_server( - self, - address, - ssl=None, - sni=None, - force=False, - persistent_change=False): - """ - Change the server connection to the specified address. - @returns: - True, if a new connection has been established, - False, if an existing connection has been used - """ - address = netlib.tcp.Address.wrap(address) - - ssl_mismatch = ( - ssl is not None and - ( - (self.c.server_conn.connection and ssl != self.c.server_conn.ssl_established) - or - (sni is not None and sni != self.c.server_conn.sni) - ) - ) - address_mismatch = (address != self.c.server_conn.address) - - if persistent_change: - self._backup_server_conn = None - - if ssl_mismatch or address_mismatch or force: - - self.c.log( - "Change server connection: %s:%s -> %s:%s [persistent: %s]" % ( - self.c.server_conn.address.host, - self.c.server_conn.address.port, - address.host, - address.port, - persistent_change - ), - "debug" - ) - - if not self._backup_server_conn and not persistent_change: - self._backup_server_conn = self.c.server_conn - self.c.server_conn = None - else: - # This is at least the second temporary change. We can kill the - # current connection. - self.c.del_server_connection() - - self.c.set_server_address(address) - self.c.establish_server_connection(ask=False) - if ssl: - self.c.establish_ssl(server=True, sni=sni) - return True - return False - - def restore_server(self): - # TODO: Similar to _backup_server_conn, introduce _cache_server_conn, - # which keeps the changed connection open This may be beneficial if a - # user is rewriting all requests from http to https or similar. - if not self._backup_server_conn: - return - - self.c.log("Restore original server connection: %s:%s -> %s:%s" % ( - self.c.server_conn.address.host, - self.c.server_conn.address.port, - self._backup_server_conn.address.host, - self._backup_server_conn.address.port - ), "debug") - - self.c.del_server_connection() - self.c.server_conn = self._backup_server_conn - self._backup_server_conn = None diff --git a/libmproxy/protocol/tcp.py b/libmproxy/protocol/tcp.py deleted file mode 100644 index 0feb77c6..00000000 --- a/libmproxy/protocol/tcp.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import absolute_import -import select -import socket -from .primitives import ProtocolHandler -from netlib.utils import cleanBin -from netlib.tcp import NetLibError - - -class TCPHandler(ProtocolHandler): - """ - TCPHandler acts as a generic TCP forwarder. - Data will be .log()ed, but not stored any further. - """ - - chunk_size = 4096 - - def __init__(self, c, log=True): - super(TCPHandler, self).__init__(c) - self.log = log - - def handle_messages(self): - self.c.establish_server_connection() - - server = "%s:%s" % self.c.server_conn.address()[:2] - buf = memoryview(bytearray(self.chunk_size)) - conns = [self.c.client_conn.rfile, self.c.server_conn.rfile] - - try: - while True: - r, _, _ = select.select(conns, [], [], 10) - for rfile in r: - if self.c.client_conn.rfile == rfile: - src, dst = self.c.client_conn, self.c.server_conn - direction = "-> tcp ->" - src_str, dst_str = "client", server - else: - dst, src = self.c.client_conn, self.c.server_conn - direction = "<- tcp <-" - dst_str, src_str = "client", server - - closed = False - if src.ssl_established: - # Unfortunately, pyOpenSSL lacks a recv_into function. - # We need to read a single byte before .pending() - # becomes usable - contents = src.rfile.read(1) - contents += src.rfile.read(src.connection.pending()) - if not contents: - closed = True - else: - size = src.connection.recv_into(buf) - if not size: - closed = True - - if closed: - conns.remove(src.rfile) - # Shutdown connection to the other peer - if dst.ssl_established: - # We can't half-close a connection, so we just close everything here. - # Sockets will be cleaned up on a higher level. - return - else: - dst.connection.shutdown(socket.SHUT_WR) - - if len(conns) == 0: - return - continue - - if src.ssl_established or dst.ssl_established: - # if one of the peers is over SSL, we need to send - # bytes/strings - if not src.ssl_established: - # we revc'd into buf but need bytes/string now. - contents = buf[:size].tobytes() - if self.log: - self.c.log( - "%s %s\r\n%s" % ( - direction, dst_str, cleanBin(contents) - ), - "info" - ) - # Do not use dst.connection.send here, which may raise - # OpenSSL-specific errors. - dst.send(contents) - else: - # socket.socket.send supports raw bytearrays/memoryviews - if self.log: - self.c.log( - "%s %s\r\n%s" % ( - direction, dst_str, cleanBin(buf.tobytes()) - ), - "info" - ) - dst.connection.send(buf[:size]) - except (socket.error, NetLibError) as e: - self.c.log("TCP connection closed unexpectedly.", "debug") - return -- cgit v1.2.3 From 421b241ff010ae979cff8df504b6744e4c291aeb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 13:40:23 +0200 Subject: remove http2http references --- libmproxy/protocol2/reverse_proxy.py | 1 - libmproxy/protocol2/root_context.py | 1 + 2 files changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index c4cabccc..3ca998d5 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -1,7 +1,6 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin -from .tls import TlsLayer class ReverseProxy(Layer, ServerConnectionMixin): diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 210ba6ab..daea54bd 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -10,6 +10,7 @@ from .layer import ServerConnectionMixin from .http_proxy import HttpProxy, HttpUpstreamProxy from .reverse_proxy import ReverseProxy + class RootContext(object): """ The outmost context provided to the root layer. -- cgit v1.2.3 From a86ec56012136664688fa4a8efcd866b5e3e17a8 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 15:27:29 +0200 Subject: move files around --- libmproxy/console/common.py | 2 +- libmproxy/console/flowview.py | 2 +- libmproxy/filt.py | 2 +- libmproxy/flow.py | 23 +- libmproxy/models/__init__.py | 16 + libmproxy/models/connections.py | 194 ++++++++++ libmproxy/models/flow.py | 166 ++++++++ libmproxy/models/http.py | 554 ++++++++++++++++++++++++++ libmproxy/protocol/__init__.py | 13 +- libmproxy/protocol/base.py | 152 ++++++++ libmproxy/protocol/http.py | 602 +++++++++++++++++++++++++---- libmproxy/protocol/http_replay.py | 95 +++++ libmproxy/protocol/http_wrappers.py | 413 -------------------- libmproxy/protocol/primitives.py | 166 -------- libmproxy/protocol/rawtcp.py | 66 ++++ libmproxy/protocol/tls.py | 288 ++++++++++++++ libmproxy/protocol2/__init__.py | 13 - libmproxy/protocol2/http.py | 588 ---------------------------- libmproxy/protocol2/http_proxy.py | 26 -- libmproxy/protocol2/http_replay.py | 95 ----- libmproxy/protocol2/layer.py | 138 ------- libmproxy/protocol2/rawtcp.py | 66 ---- libmproxy/protocol2/reverse_proxy.py | 17 - libmproxy/protocol2/root_context.py | 95 ----- libmproxy/protocol2/socks_proxy.py | 59 --- libmproxy/protocol2/tls.py | 288 -------------- libmproxy/protocol2/transparent_proxy.py | 24 -- libmproxy/proxy/__init__.py | 8 +- libmproxy/proxy/config.py | 2 +- libmproxy/proxy/connection.py | 193 --------- libmproxy/proxy/modes/__init__.py | 12 + libmproxy/proxy/modes/http_proxy.py | 26 ++ libmproxy/proxy/modes/reverse_proxy.py | 17 + libmproxy/proxy/modes/socks_proxy.py | 60 +++ libmproxy/proxy/modes/transparent_proxy.py | 24 ++ libmproxy/proxy/primitives.py | 15 - libmproxy/proxy/root_context.py | 93 +++++ libmproxy/proxy/server.py | 23 +- 38 files changed, 2329 insertions(+), 2307 deletions(-) create mode 100644 libmproxy/models/__init__.py create mode 100644 libmproxy/models/connections.py create mode 100644 libmproxy/models/flow.py create mode 100644 libmproxy/models/http.py create mode 100644 libmproxy/protocol/base.py create mode 100644 libmproxy/protocol/http_replay.py delete mode 100644 libmproxy/protocol/http_wrappers.py delete mode 100644 libmproxy/protocol/primitives.py create mode 100644 libmproxy/protocol/rawtcp.py create mode 100644 libmproxy/protocol/tls.py delete mode 100644 libmproxy/protocol2/__init__.py delete mode 100644 libmproxy/protocol2/http.py delete mode 100644 libmproxy/protocol2/http_proxy.py delete mode 100644 libmproxy/protocol2/http_replay.py delete mode 100644 libmproxy/protocol2/layer.py delete mode 100644 libmproxy/protocol2/rawtcp.py delete mode 100644 libmproxy/protocol2/reverse_proxy.py delete mode 100644 libmproxy/protocol2/root_context.py delete mode 100644 libmproxy/protocol2/socks_proxy.py delete mode 100644 libmproxy/protocol2/tls.py delete mode 100644 libmproxy/protocol2/transparent_proxy.py delete mode 100644 libmproxy/proxy/connection.py create mode 100644 libmproxy/proxy/modes/__init__.py create mode 100644 libmproxy/proxy/modes/http_proxy.py create mode 100644 libmproxy/proxy/modes/reverse_proxy.py create mode 100644 libmproxy/proxy/modes/socks_proxy.py create mode 100644 libmproxy/proxy/modes/transparent_proxy.py delete mode 100644 libmproxy/proxy/primitives.py create mode 100644 libmproxy/proxy/root_context.py (limited to 'libmproxy') diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 1940e390..c25f7267 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -8,7 +8,7 @@ from netlib.http.semantics import CONTENT_MISSING import netlib.utils from .. import utils -from ..protocol.http import decoded +from ..models import decoded from . import signals diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 1e0f0c17..8b828653 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -9,7 +9,7 @@ from netlib.http.semantics import CONTENT_MISSING from . import common, grideditor, contentview, signals, searchable, tabs from . import flowdetailview from .. import utils, controller -from ..protocol.http import HTTPRequest, HTTPResponse, decoded +from ..models import HTTPRequest, HTTPResponse, decoded class SearchError(Exception): diff --git a/libmproxy/filt.py b/libmproxy/filt.py index 25747bc6..cfd3a1bc 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -35,7 +35,7 @@ from __future__ import absolute_import import re import sys import pyparsing as pp -from .protocol.http import decoded +from .models import decoded class _Token: diff --git a/libmproxy/flow.py b/libmproxy/flow.py index a2f57512..00ec83d2 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,19 +8,18 @@ import Cookie import cookielib import os import re -from libmproxy.protocol.http import HTTPFlow -from libmproxy.protocol2.http_replay import RequestReplayThread +import urlparse + from netlib import odict, wsgi from netlib.http.semantics import CONTENT_MISSING import netlib.http - -from . import controller, protocol, tnetstring, filt, script, version +from . import controller, tnetstring, filt, script, version from .onboarding import app -from .protocol import http from .proxy.config import HostMatcher -from .proxy.connection import ClientConnection, ServerConnection -import urlparse +from .protocol.http_replay import RequestReplayThread +from .protocol import Kill +from .models import ClientConnection, ServerConnection, HTTPResponse, HTTPFlow, HTTPRequest class AppRegistry: @@ -790,7 +789,7 @@ class FlowMaster(controller.Master): rflow = self.server_playback.next_flow(flow) if not rflow: return None - response = http.HTTPResponse.from_state(rflow.response.get_state()) + response = HTTPResponse.from_state(rflow.response.get_state()) response.is_replay = True if self.refresh_server_playback: response.refresh() @@ -836,10 +835,10 @@ class FlowMaster(controller.Master): sni=host, ssl_established=True )) - f = http.HTTPFlow(c, s) + f = HTTPFlow(c, s) headers = odict.ODictCaseless() - req = http.HTTPRequest( + req = HTTPRequest( "absolute", method, scheme, @@ -981,7 +980,7 @@ class FlowMaster(controller.Master): ) if err: self.add_event("Error in wsgi app. %s" % err, "error") - f.reply(protocol.KILL) + f.reply(Kill) return if f not in self.state.flows: # don't add again on replay self.state.add_flow(f) @@ -998,7 +997,7 @@ class FlowMaster(controller.Master): if self.stream_large_bodies: self.stream_large_bodies.run(f, False) except netlib.http.HttpError: - f.reply(protocol.KILL) + f.reply(Kill) return f.reply() diff --git a/libmproxy/models/__init__.py b/libmproxy/models/__init__.py new file mode 100644 index 00000000..3947847c --- /dev/null +++ b/libmproxy/models/__init__.py @@ -0,0 +1,16 @@ +from __future__ import (absolute_import, print_function, division) + +from .http import ( + HTTPFlow, HTTPRequest, HTTPResponse, decoded, + make_error_response, make_connect_request, make_connect_response +) +from .connections import ClientConnection, ServerConnection +from .flow import Flow, Error + +__all__ = [ + "HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded" + "make_error_response", "make_connect_request", + "make_connect_response", + "ClientConnection", "ServerConnection", + "Flow", "Error", +] diff --git a/libmproxy/models/connections.py b/libmproxy/models/connections.py new file mode 100644 index 00000000..98bae3cc --- /dev/null +++ b/libmproxy/models/connections.py @@ -0,0 +1,194 @@ +from __future__ import absolute_import + +import copy +import os + +from netlib import tcp, certutils +from .. import stateobject, utils + + +class ClientConnection(tcp.BaseHandler, stateobject.StateObject): + def __init__(self, client_connection, address, server): + # Eventually, this object is restored from state. We don't have a + # connection then. + if client_connection: + super(ClientConnection, self).__init__(client_connection, address, server) + else: + self.connection = None + self.server = None + self.wfile = None + self.rfile = None + self.address = None + self.clientcert = None + self.ssl_established = None + + self.timestamp_start = utils.timestamp() + self.timestamp_end = None + self.timestamp_ssl_setup = None + self.protocol = None + + def __nonzero__(self): + return bool(self.connection) and not self.finished + + def __repr__(self): + return "".format( + ssl="[ssl] " if self.ssl_established else "", + host=self.address.host, + port=self.address.port + ) + + @property + def tls_established(self): + return self.ssl_established + + _stateobject_attributes = dict( + ssl_established=bool, + timestamp_start=float, + timestamp_end=float, + timestamp_ssl_setup=float + ) + + def get_state(self, short=False): + d = super(ClientConnection, self).get_state(short) + d.update( + address={ + "address": self.address(), + "use_ipv6": self.address.use_ipv6}, + clientcert=self.cert.to_pem() if self.clientcert else None) + return d + + def load_state(self, state): + super(ClientConnection, self).load_state(state) + self.address = tcp.Address( + **state["address"]) if state["address"] else None + self.clientcert = certutils.SSLCert.from_pem( + state["clientcert"]) if state["clientcert"] else None + + def copy(self): + return copy.copy(self) + + def send(self, message): + if isinstance(message, list): + message = b''.join(message) + self.wfile.write(message) + self.wfile.flush() + + @classmethod + def from_state(cls, state): + f = cls(None, tuple(), None) + f.load_state(state) + return f + + def convert_to_ssl(self, *args, **kwargs): + super(ClientConnection, self).convert_to_ssl(*args, **kwargs) + self.timestamp_ssl_setup = utils.timestamp() + + def finish(self): + super(ClientConnection, self).finish() + self.timestamp_end = utils.timestamp() + + +class ServerConnection(tcp.TCPClient, stateobject.StateObject): + def __init__(self, address): + tcp.TCPClient.__init__(self, address) + + self.via = None + self.timestamp_start = None + self.timestamp_end = None + self.timestamp_tcp_setup = None + self.timestamp_ssl_setup = None + self.protocol = None + + def __nonzero__(self): + return bool(self.connection) and not self.finished + + def __repr__(self): + if self.ssl_established and self.sni: + ssl = "[ssl: {0}] ".format(self.sni) + elif self.ssl_established: + ssl = "[ssl] " + else: + ssl = "" + return "".format( + ssl=ssl, + host=self.address.host, + port=self.address.port + ) + + @property + def tls_established(self): + return self.ssl_established + + _stateobject_attributes = dict( + timestamp_start=float, + timestamp_end=float, + timestamp_tcp_setup=float, + timestamp_ssl_setup=float, + address=tcp.Address, + source_address=tcp.Address, + cert=certutils.SSLCert, + ssl_established=bool, + sni=str + ) + _stateobject_long_attributes = {"cert"} + + def get_state(self, short=False): + d = super(ServerConnection, self).get_state(short) + d.update( + address={"address": self.address(), + "use_ipv6": self.address.use_ipv6}, + source_address=({"address": self.source_address(), + "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), + cert=self.cert.to_pem() if self.cert else None + ) + return d + + def load_state(self, state): + super(ServerConnection, self).load_state(state) + + self.address = tcp.Address( + **state["address"]) if state["address"] else None + self.source_address = tcp.Address( + **state["source_address"]) if state["source_address"] else None + self.cert = certutils.SSLCert.from_pem( + state["cert"]) if state["cert"] else None + + @classmethod + def from_state(cls, state): + f = cls(tuple()) + f.load_state(state) + return f + + def copy(self): + return copy.copy(self) + + def connect(self): + self.timestamp_start = utils.timestamp() + tcp.TCPClient.connect(self) + self.timestamp_tcp_setup = utils.timestamp() + + def send(self, message): + if isinstance(message, list): + message = b''.join(message) + self.wfile.write(message) + self.wfile.flush() + + def establish_ssl(self, clientcerts, sni, **kwargs): + clientcert = None + if clientcerts: + path = os.path.join( + clientcerts, + self.address.host.encode("idna")) + ".pem" + if os.path.exists(path): + clientcert = path + + self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) + self.sni = sni + self.timestamp_ssl_setup = utils.timestamp() + + def finish(self): + tcp.TCPClient.finish(self) + self.timestamp_end = utils.timestamp() + + +ServerConnection._stateobject_attributes["via"] = ServerConnection diff --git a/libmproxy/models/flow.py b/libmproxy/models/flow.py new file mode 100644 index 00000000..58287e5b --- /dev/null +++ b/libmproxy/models/flow.py @@ -0,0 +1,166 @@ +from __future__ import absolute_import +import copy +import uuid + +from .. import stateobject, utils, version +from .connections import ClientConnection, ServerConnection + + +class Error(stateobject.StateObject): + """ + An Error. + + This is distinct from an protocol error response (say, a HTTP code 500), + which is represented by a normal HTTPResponse object. This class is + responsible for indicating errors that fall outside of normal protocol + communications, like interrupted connections, timeouts, protocol errors. + + Exposes the following attributes: + + flow: Flow object + msg: Message describing the error + timestamp: Seconds since the epoch + """ + + def __init__(self, msg, timestamp=None): + """ + @type msg: str + @type timestamp: float + """ + self.flow = None # will usually be set by the flow backref mixin + self.msg = msg + self.timestamp = timestamp or utils.timestamp() + + _stateobject_attributes = dict( + msg=str, + timestamp=float + ) + + def __str__(self): + return self.msg + + @classmethod + def from_state(cls, state): + # the default implementation assumes an empty constructor. Override + # accordingly. + f = cls(None) + f.load_state(state) + return f + + def copy(self): + c = copy.copy(self) + return c + + +class Flow(stateobject.StateObject): + """ + A Flow is a collection of objects representing a single transaction. + This class is usually subclassed for each protocol, e.g. HTTPFlow. + """ + + def __init__(self, type, client_conn, server_conn, live=None): + self.type = type + self.id = str(uuid.uuid4()) + self.client_conn = client_conn + """@type: ClientConnection""" + self.server_conn = server_conn + """@type: ServerConnection""" + self.live = live + """@type: LiveConnection""" + + self.error = None + """@type: Error""" + self.intercepted = False + """@type: bool""" + self._backup = None + self.reply = None + + _stateobject_attributes = dict( + id=str, + error=Error, + client_conn=ClientConnection, + server_conn=ServerConnection, + type=str, + intercepted=bool + ) + + def get_state(self, short=False): + d = super(Flow, self).get_state(short) + d.update(version=version.IVERSION) + if self._backup and self._backup != d: + if short: + d.update(modified=True) + else: + d.update(backup=self._backup) + return d + + def __eq__(self, other): + return self is other + + def copy(self): + f = copy.copy(self) + + f.id = str(uuid.uuid4()) + f.live = False + f.client_conn = self.client_conn.copy() + f.server_conn = self.server_conn.copy() + + if self.error: + f.error = self.error.copy() + return f + + def modified(self): + """ + Has this Flow been modified? + """ + if self._backup: + return self._backup != self.get_state() + else: + return False + + def backup(self, force=False): + """ + Save a backup of this Flow, which can be reverted to using a + call to .revert(). + """ + if not self._backup: + self._backup = self.get_state() + + def revert(self): + """ + Revert to the last backed up state. + """ + if self._backup: + self.load_state(self._backup) + self._backup = None + + def kill(self, master): + """ + Kill this request. + """ + from ..protocol import Kill + + self.error = Error("Connection killed") + self.intercepted = False + self.reply(Kill) + master.handle_error(self) + + def intercept(self, master): + """ + Intercept this Flow. Processing will stop until accept_intercept is + called. + """ + if self.intercepted: + return + self.intercepted = True + master.handle_intercept(self) + + def accept_intercept(self, master): + """ + Continue with the flow - called after an intercept(). + """ + if not self.intercepted: + return + self.intercepted = False + self.reply() + master.handle_accept_intercept(self) diff --git a/libmproxy/models/http.py b/libmproxy/models/http.py new file mode 100644 index 00000000..fb2f305b --- /dev/null +++ b/libmproxy/models/http.py @@ -0,0 +1,554 @@ +from __future__ import (absolute_import, print_function, division) +import Cookie +import copy +from email.utils import parsedate_tz, formatdate, mktime_tz +import time + +from libmproxy import utils +from netlib import odict, encoding +from netlib.http import status_codes +from netlib.tcp import Address +from netlib.http.semantics import Request, Response, CONTENT_MISSING +from .. import version, stateobject +from .flow import Flow + + +class MessageMixin(stateobject.StateObject): + _stateobject_attributes = dict( + httpversion=tuple, + headers=odict.ODictCaseless, + body=str, + timestamp_start=float, + timestamp_end=float + ) + _stateobject_long_attributes = {"body"} + + def get_state(self, short=False): + ret = super(MessageMixin, self).get_state(short) + if short: + if self.body: + ret["contentLength"] = len(self.body) + elif self.body == CONTENT_MISSING: + ret["contentLength"] = None + else: + ret["contentLength"] = 0 + return ret + + def get_decoded_content(self): + """ + Returns the decoded content based on the current Content-Encoding + header. + Doesn't change the message iteself or its headers. + """ + ce = self.headers.get_first("content-encoding") + if not self.body or ce not in encoding.ENCODINGS: + return self.body + return encoding.decode(ce, self.body) + + def decode(self): + """ + Decodes body based on the current Content-Encoding header, then + removes the header. If there is no Content-Encoding header, no + action is taken. + + Returns True if decoding succeeded, False otherwise. + """ + ce = self.headers.get_first("content-encoding") + if not self.body or ce not in encoding.ENCODINGS: + return False + data = encoding.decode(ce, self.body) + if data is None: + return False + self.body = data + del self.headers["content-encoding"] + return True + + def encode(self, e): + """ + Encodes body with the encoding e, where e is "gzip", "deflate" + or "identity". + """ + # FIXME: Error if there's an existing encoding header? + self.body = encoding.encode(e, self.body) + self.headers["content-encoding"] = [e] + + def copy(self): + c = copy.copy(self) + c.headers = self.headers.copy() + return c + + def replace(self, pattern, repl, *args, **kwargs): + """ + Replaces a regular expression pattern with repl in both the headers + and the body of the message. Encoded body will be decoded + before replacement, and re-encoded afterwards. + + Returns the number of replacements made. + """ + with decoded(self): + self.body, c = utils.safe_subn( + pattern, repl, self.body, *args, **kwargs + ) + c += self.headers.replace(pattern, repl, *args, **kwargs) + return c + + +class HTTPRequest(MessageMixin, Request): + """ + An HTTP request. + + Exposes the following attributes: + + method: HTTP method + + scheme: URL scheme (http/https) + + host: Target hostname of the request. This is not neccessarily the + directy upstream server (which could be another proxy), but it's always + the target server we want to reach at the end. This attribute is either + inferred from the request itself (absolute-form, authority-form) or from + the connection metadata (e.g. the host in reverse proxy mode). + + port: Destination port + + path: Path portion of the URL (not present in authority-form) + + httpversion: HTTP version tuple, e.g. (1,1) + + headers: odict.ODictCaseless object + + content: Content of the request, None, or CONTENT_MISSING if there + is content associated, but not present. CONTENT_MISSING evaluates + to False to make checking for the presence of content natural. + + form_in: The request form which mitmproxy has received. The following + values are possible: + + - relative (GET /index.html, OPTIONS *) (covers origin form and + asterisk form) + - absolute (GET http://example.com:80/index.html) + - authority-form (CONNECT example.com:443) + Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 + + form_out: The request form which mitmproxy will send out to the + destination + + timestamp_start: Timestamp indicating when request transmission started + + timestamp_end: Timestamp indicating when request transmission ended + """ + + def __init__( + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + body, + timestamp_start=None, + timestamp_end=None, + form_out=None, + ): + Request.__init__( + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + body, + timestamp_start, + timestamp_end, + ) + self.form_out = form_out or form_in + + # Have this request's cookies been modified by sticky cookies or auth? + self.stickycookie = False + self.stickyauth = False + + # Is this request replayed? + self.is_replay = False + + _stateobject_attributes = MessageMixin._stateobject_attributes.copy() + _stateobject_attributes.update( + form_in=str, + method=str, + scheme=str, + host=str, + port=int, + path=str, + form_out=str, + is_replay=bool + ) + + @classmethod + def from_state(cls, state): + f = cls( + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None) + f.load_state(state) + return f + + @classmethod + def from_protocol( + self, + protocol, + *args, + **kwargs + ): + req = protocol.read_request(*args, **kwargs) + return self.wrap(req) + + @classmethod + def wrap(self, request): + req = HTTPRequest( + form_in=request.form_in, + method=request.method, + scheme=request.scheme, + host=request.host, + port=request.port, + path=request.path, + httpversion=request.httpversion, + headers=request.headers, + body=request.body, + timestamp_start=request.timestamp_start, + timestamp_end=request.timestamp_end, + form_out=(request.form_out if hasattr(request, 'form_out') else None), + ) + if hasattr(request, 'stream_id'): + req.stream_id = request.stream_id + return req + + def __hash__(self): + return id(self) + + def replace(self, pattern, repl, *args, **kwargs): + """ + Replaces a regular expression pattern with repl in the headers, the + request path and the body of the request. Encoded content will be + decoded before replacement, and re-encoded afterwards. + + Returns the number of replacements made. + """ + c = MessageMixin.replace(self, pattern, repl, *args, **kwargs) + self.path, pc = utils.safe_subn( + pattern, repl, self.path, *args, **kwargs + ) + c += pc + return c + + +class HTTPResponse(MessageMixin, Response): + """ + An HTTP response. + + Exposes the following attributes: + + httpversion: HTTP version tuple, e.g. (1, 0), (1, 1), or (2, 0) + + status_code: HTTP response status code + + msg: HTTP response message + + headers: ODict Caseless object + + content: Content of the request, None, or CONTENT_MISSING if there + is content associated, but not present. CONTENT_MISSING evaluates + to False to make checking for the presence of content natural. + + timestamp_start: Timestamp indicating when request transmission started + + timestamp_end: Timestamp indicating when request transmission ended + """ + + def __init__( + self, + httpversion, + status_code, + msg, + headers, + body, + timestamp_start=None, + timestamp_end=None, + ): + Response.__init__( + self, + httpversion, + status_code, + msg, + headers, + body, + timestamp_start=timestamp_start, + timestamp_end=timestamp_end, + ) + + # Is this request replayed? + self.is_replay = False + self.stream = False + + _stateobject_attributes = MessageMixin._stateobject_attributes.copy() + _stateobject_attributes.update( + status_code=int, + msg=str + ) + + @classmethod + def from_state(cls, state): + f = cls(None, None, None, None, None) + f.load_state(state) + return f + + @classmethod + def from_protocol( + self, + protocol, + *args, + **kwargs + ): + resp = protocol.read_response(*args, **kwargs) + return self.wrap(resp) + + @classmethod + def wrap(self, response): + resp = HTTPResponse( + httpversion=response.httpversion, + status_code=response.status_code, + msg=response.msg, + headers=response.headers, + body=response.body, + timestamp_start=response.timestamp_start, + timestamp_end=response.timestamp_end, + ) + if hasattr(response, 'stream_id'): + resp.stream_id = response.stream_id + return resp + + def _refresh_cookie(self, c, delta): + """ + Takes a cookie string c and a time delta in seconds, and returns + a refreshed cookie string. + """ + c = Cookie.SimpleCookie(str(c)) + for i in c.values(): + if "expires" in i: + d = parsedate_tz(i["expires"]) + if d: + d = mktime_tz(d) + delta + i["expires"] = formatdate(d) + else: + # This can happen when the expires tag is invalid. + # reddit.com sends a an expires tag like this: "Thu, 31 Dec + # 2037 23:59:59 GMT", which is valid RFC 1123, but not + # strictly correct according to the cookie spec. Browsers + # appear to parse this tolerantly - maybe we should too. + # For now, we just ignore this. + del i["expires"] + return c.output(header="").strip() + + def refresh(self, now=None): + """ + This fairly complex and heuristic function refreshes a server + response for replay. + + - It adjusts date, expires and last-modified headers. + - It adjusts cookie expiration. + """ + if not now: + now = time.time() + delta = now - self.timestamp_start + refresh_headers = [ + "date", + "expires", + "last-modified", + ] + for i in refresh_headers: + if i in self.headers: + d = parsedate_tz(self.headers[i][0]) + if d: + new = mktime_tz(d) + delta + self.headers[i] = [formatdate(new)] + c = [] + for i in self.headers["set-cookie"]: + c.append(self._refresh_cookie(i, delta)) + if c: + self.headers["set-cookie"] = c + + +class HTTPFlow(Flow): + """ + A HTTPFlow is a collection of objects representing a single HTTP + transaction. The main attributes are: + + request: HTTPRequest object + response: HTTPResponse object + error: Error object + server_conn: ServerConnection object + client_conn: ClientConnection object + + Note that it's possible for a Flow to have both a response and an error + object. This might happen, for instance, when a response was received + from the server, but there was an error sending it back to the client. + + The following additional attributes are exposed: + + intercepted: Is this flow currently being intercepted? + live: Does this flow have a live client connection? + """ + + def __init__(self, client_conn, server_conn, live=None): + super(HTTPFlow, self).__init__("http", client_conn, server_conn, live) + self.request = None + """@type: HTTPRequest""" + self.response = None + """@type: HTTPResponse""" + + _stateobject_attributes = Flow._stateobject_attributes.copy() + _stateobject_attributes.update( + request=HTTPRequest, + response=HTTPResponse + ) + + @classmethod + def from_state(cls, state): + f = cls(None, None) + f.load_state(state) + return f + + def __repr__(self): + s = " + + %d %s + + %s + + """.strip() % (status_code, response, message) + + if not headers: + headers = odict.ODictCaseless() + headers["Server"] = [version.NAMEVERSION] + headers["Connection"] = ["close"] + headers["Content-Length"] = [len(body)] + headers["Content-Type"] = ["text/html"] + + return HTTPResponse( + (1, 1), # FIXME: Should be a string. + status_code, + response, + headers, + body, + ) + + +def make_connect_request(address): + address = Address.wrap(address) + return HTTPRequest( + "authority", "CONNECT", None, address.host, address.port, None, (1, 1), + odict.ODictCaseless(), "" + ) + + +def make_connect_response(httpversion): + headers = odict.ODictCaseless([ + ["Content-Length", "0"], + ["Proxy-Agent", version.NAMEVERSION] + ]) + return HTTPResponse( + httpversion, + 200, + "Connection established", + headers, + "", + ) diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index bbc20dba..c582592b 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -1 +1,12 @@ -from .primitives import * +from __future__ import (absolute_import, print_function, division) +from .base import Layer, ServerConnectionMixin, Log, Kill +from .http import Http1Layer, Http2Layer +from .tls import TlsLayer, is_tls_record_magic +from .rawtcp import RawTCPLayer + +__all__ = [ + "Layer", "ServerConnectionMixin", "Log", "Kill", + "Http1Layer", "Http2Layer", + "TlsLayer", "is_tls_record_magic", + "RawTCPLayer" +] diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py new file mode 100644 index 00000000..d22a71c6 --- /dev/null +++ b/libmproxy/protocol/base.py @@ -0,0 +1,152 @@ +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible settings (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentProxy + Http1Layer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + HttpProxy + Http1Layer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentProxy + TLSLayer + Http2Layer + HttpLayer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can call functions provided by lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" +from __future__ import (absolute_import, print_function, division) +from netlib import tcp +from ..models import ServerConnection +from ..exceptions import ProtocolException + + +class _LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + + def __init__(self, *args, **kwargs): + super(_LayerCodeCompletion, self).__init__(*args, **kwargs) + if True: + return + self.config = None + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(_LayerCodeCompletion): + def __init__(self, ctx, *args, **kwargs): + """ + Args: + ctx: The (read-only) higher layer. + """ + super(Layer, self).__init__(*args, **kwargs) + self.ctx = ctx + + def __call__(self): + """ + Logic of the layer. + Raises: + ProtocolException in case of protocol exceptions. + """ + raise NotImplementedError + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "{}: {}".format(repr(self.client_conn.address), msg) + ] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + @property + def layers(self): + return [self] + self.ctx.layers + + def __repr__(self): + return type(self).__name__ + + +class ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + + def __init__(self, server_address=None): + super(ServerConnectionMixin, self).__init__() + self.server_conn = ServerConnection(server_address) + + def reconnect(self): + address = self.server_conn.address + self._disconnect() + self.server_conn.address = address + self.connect() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + if depth == 1: + if self.server_conn: + self._disconnect() + self.log("Set new server address: " + repr(address), "debug") + self.server_conn.address = address + else: + self.ctx.set_server(address, server_tls, sni, depth - 1) + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + self.server_conn.finish() + self.server_conn.close() + # self.channel.tell("serverdisconnect", self) + self.server_conn = ServerConnection(None) + + def connect(self): + if not self.server_conn.address: + raise ProtocolException("Cannot connect to server, no server address given.") + self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProtocolException( + "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) + + +class Log(object): + def __init__(self, msg, level="info"): + self.msg = msg + self.level = level + + +class Kill(Exception): + """ + Kill a connection. + """ diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index bde7b088..fc57f6df 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,92 +1,538 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) -from .primitives import Flow +from netlib import tcp +from netlib.http import http1, HttpErrorConnClosed, HttpError +from netlib.http.semantics import CONTENT_MISSING +from netlib import odict +from netlib.tcp import NetLibError, Address +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol -from .http_wrappers import decoded, HTTPRequest, HTTPResponse +from .. import utils +from ..exceptions import InvalidCredentials, HttpException, ProtocolException +from ..models import ( + HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error +) +from .base import Layer, Kill -class HTTPFlow(Flow): - """ - A HTTPFlow is a collection of objects representing a single HTTP - transaction. The main attributes are: - request: HTTPRequest object - response: HTTPResponse object - error: Error object - server_conn: ServerConnection object - client_conn: ClientConnection object +class _HttpLayer(Layer): + supports_streaming = False + + def read_request(self): + raise NotImplementedError() + + def send_request(self, request): + raise NotImplementedError() + + def read_response(self, request_method): + raise NotImplementedError() + + def send_response(self, response): + raise NotImplementedError() + + +class _StreamingHttpLayer(_HttpLayer): + supports_streaming = True + + def read_response_headers(self): + raise NotImplementedError + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + raise NotImplementedError() + yield "this is a generator" + + def send_response_headers(self, response): + raise NotImplementedError + + def send_response_body(self, response, chunks): + raise NotImplementedError() + + +class Http1Layer(_StreamingHttpLayer): + def __init__(self, ctx, mode): + super(Http1Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP1Protocol(self.client_conn) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def read_request(self): + return HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + + def send_request(self, request): + self.server_conn.send(self.server_protocol.assemble(request)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True + ) + + def send_response(self, response): + self.client_conn.send(self.client_protocol.assemble(response)) + + def read_response_headers(self): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=None, # does not matter if we don't read the body. + body_size_limit=self.config.body_size_limit, + include_body=False + ) + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + return self.server_protocol.read_http_body_chunked( + headers, + self.config.body_size_limit, + request_method, + response_code, + False, + max_chunk_size + ) + + def send_response_headers(self, response): + h = self.client_protocol._assemble_response_first_line(response) + self.client_conn.wfile.write(h + "\r\n") + h = self.client_protocol._assemble_response_headers( + response, + preserve_transfer_encoding=True + ) + self.client_conn.send(h + "\r\n") + + def send_response_body(self, response, chunks): + if self.client_protocol.has_chunked_encoding(response.headers): + chunks = ( + "%d\r\n%s\r\n" % (len(chunk), chunk) + for chunk in chunks + ) + for chunk in chunks: + self.client_conn.send(chunk) - Note that it's possible for a Flow to have both a response and an error - object. This might happen, for instance, when a response was received - from the server, but there was an error sending it back to the client. + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP1Protocol(self.server_conn) - The following additional attributes are exposed: + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP1Protocol(self.server_conn) - intercepted: Is this flow currently being intercepted? - live: Does this flow have a live client connection? + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def __call__(self): + layer = HttpLayer(self, self.mode) + layer() + + +# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. +class Http2Layer(_HttpLayer): + def __init__(self, ctx, mode): + super(Http2Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + + def read_request(self): + request = HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + self._stream_id = request.stream_id + return request + + def send_request(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.server_conn.send(self.server_protocol.assemble(message)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True, + stream_id=self._stream_id + ) + + def send_response(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.client_conn.send(self.client_protocol.assemble(message)) + + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def __call__(self): + self.server_protocol.perform_connection_preface() + layer = HttpLayer(self, self.mode) + layer() + + def handle_unexpected_frame(self, frm): + self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") + + +class ConnectServerConnection(object): + """ + "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. """ - def __init__(self, client_conn, server_conn, live=None): - super(HTTPFlow, self).__init__("http", client_conn, server_conn, live) - self.request = None - """@type: HTTPRequest""" - self.response = None - """@type: HTTPResponse""" - - _stateobject_attributes = Flow._stateobject_attributes.copy() - _stateobject_attributes.update( - request=HTTPRequest, - response=HTTPResponse - ) - - @classmethod - def from_state(cls, state): - f = cls(None, None) - f.load_state(state) - return f - - def __repr__(self): - s = " clientconnect (transparent mode destination known) + # > serverconnect (required for client tls handshake) + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + self.reconnect() + get_response() + + # call the appropriate script hook - this is an opportunity for an + # inline script to set flow.stream = True + flow = self.channel.ask("responseheaders", flow) + if flow is None or flow == Kill: + raise Kill() + + if self.supports_streaming: + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = "".join(self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code + )) + flow.response.timestamp_end = utils.timestamp() + + # no further manipulation of self.server_conn beyond this point + # we can safely set it as the final attribute value here. + flow.server_conn = self.server_conn + + self.log( + "response", + "debug", + [repr(flow.response)] + ) + response_reply = self.channel.ask("response", flow) + if response_reply is None or response_reply == Kill: + raise Kill() + + def process_request_hook(self, flow): + # Determine .scheme, .host and .port attributes for inline scripts. + # For absolute-form requests, they are directly given in the request. + # For authority-form requests, we only need to determine the request scheme. + # For relative-form requests, we need to determine host and port as + # well. + if self.mode == "regular": + pass # only absolute-form at this point, nothing to do here. + elif self.mode == "upstream": + if flow.request.form_in == "authority": + flow.request.scheme = "http" # pseudo value + else: + flow.request.host = self.__original_server_conn.address.host + flow.request.port = self.__original_server_conn.address.port + flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" + + request_reply = self.channel.ask("request", flow) + if request_reply is None or request_reply == Kill: + raise Kill() + if isinstance(request_reply, HTTPResponse): + flow.response = request_reply + return + + def establish_server_connection(self, flow): + address = tcp.Address((flow.request.host, flow.request.port)) + tls = (flow.request.scheme == "https") + + if self.mode == "regular" or self.mode == "transparent": + # If there's an existing connection that doesn't match our expectations, kill it. + if address != self.server_conn.address or tls != self.server_conn.ssl_established: + self.set_server(address, tls, address.host) + # Establish connection is neccessary. + if not self.server_conn: + self.connect() + + # SetServer is not guaranteed to work with TLS: + # If there's not TlsLayer below which could catch the exception, + # TLS will not be established. + if tls and not self.server_conn.tls_established: + raise ProtocolException( + "Cannot upgrade to SSL, no TLS layer on the protocol stack.") + else: + if not self.server_conn: + self.connect() + if tls: + raise HttpException("Cannot change scheme in upstream proxy mode.") + """ + # This is a very ugly (untested) workaround to solve a very ugly problem. + if self.server_conn and self.server_conn.tls_established and not ssl: + self.reconnect() + elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: + if self.server_conn.tls_established: + self.reconnect() + + self.send_request(make_connect_request(address)) + tls_layer = TlsLayer(self, False, True) + tls_layer._establish_tls_with_server() + """ + + def validate_request(self, request): + if request.form_in == "absolute" and request.scheme != "http": + self.send_response( + make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + raise HttpException("Invalid request scheme: %s" % request.scheme) + + expected_request_forms = { + "regular": ("absolute",), # an authority request would already be handled. + "upstream": ("authority", "absolute"), + "transparent": ("relative",) + } + + allowed_request_forms = expected_request_forms[self.mode] + if request.form_in not in allowed_request_forms: + err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( + " or ".join(allowed_request_forms), request.form_in + ) + self.send_response(make_error_response(400, err_message)) + raise HttpException(err_message) + + if self.mode == "regular": + request.form_out = "relative" + + def authenticate(self, request): + if self.config.authenticator: + if self.config.authenticator.authenticate(request.headers): + self.config.authenticator.clean(request.headers) + else: + self.send_response(make_error_response( + 407, + "Proxy Authentication Required", + odict.ODictCaseless( + [ + [k, v] for k, v in + self.config.authenticator.auth_challenge_headers().items() + ]) + )) + raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py new file mode 100644 index 00000000..e0144c93 --- /dev/null +++ b/libmproxy/protocol/http_replay.py @@ -0,0 +1,95 @@ +import threading + +from netlib.http import HttpError +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError +from ..controller import Channel +from ..models import Error, HTTPResponse, ServerConnection, make_connect_request +from .base import Log, Kill + + +# TODO: Doesn't really belong into libmproxy.protocol... + + +class RequestReplayThread(threading.Thread): + name = "RequestReplayThread" + + def __init__(self, config, flow, masterq, should_exit): + """ + masterqueue can be a queue or None, if no scripthooks should be + processed. + """ + self.config, self.flow = config, flow + if masterq: + self.channel = Channel(masterq, should_exit) + else: + self.channel = None + super(RequestReplayThread, self).__init__() + + def run(self): + r = self.flow.request + form_out_backup = r.form_out + try: + self.flow.response = None + + # If we have a channel, run script hooks. + if self.channel: + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == Kill: + raise Kill() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply + + if not self.flow.response: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + server_address = self.config.upstream_server.address + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + connect_request = make_connect_request((r.host, r.port)) + server.send(protocol.assemble(connect_request)) + resp = protocol.read_response("CONNECT") + if resp.code != 200: + raise HttpError(502, "Upstream server refuses CONNECT request") + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + + server.send(protocol.assemble(r)) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_protocol( + protocol, + r.method, + body_size_limit=self.config.body_size_limit, + ) + if self.channel: + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == Kill: + raise Kill() + except (HttpError, NetLibError) as v: + self.flow.error = Error(repr(v)) + if self.channel: + self.channel.ask("error", self.flow) + except Kill: + # KillSignal should only be raised if there's a channel in the + # first place. + self.channel.tell("log", Log("Connection killed", "info")) + finally: + r.form_out = form_out_backup diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py deleted file mode 100644 index a26ddbb4..00000000 --- a/libmproxy/protocol/http_wrappers.py +++ /dev/null @@ -1,413 +0,0 @@ -from __future__ import absolute_import -import Cookie -import copy -import time -from email.utils import parsedate_tz, formatdate, mktime_tz - -from netlib import odict, encoding -from netlib.http import semantics, CONTENT_MISSING -from .. import utils, stateobject - - -class decoded(object): - """ - A context manager that decodes a request or response, and then - re-encodes it with the same encoding after execution of the block. - - Example: - with decoded(request): - request.content = request.content.replace("foo", "bar") - """ - - def __init__(self, o): - self.o = o - ce = o.headers.get_first("content-encoding") - if ce in encoding.ENCODINGS: - self.ce = ce - else: - self.ce = None - - def __enter__(self): - if self.ce: - self.o.decode() - - def __exit__(self, type, value, tb): - if self.ce: - self.o.encode(self.ce) - - -class MessageMixin(stateobject.StateObject): - _stateobject_attributes = dict( - httpversion=tuple, - headers=odict.ODictCaseless, - body=str, - timestamp_start=float, - timestamp_end=float - ) - _stateobject_long_attributes = {"body"} - - def get_state(self, short=False): - ret = super(MessageMixin, self).get_state(short) - if short: - if self.body: - ret["contentLength"] = len(self.body) - elif self.body == CONTENT_MISSING: - ret["contentLength"] = None - else: - ret["contentLength"] = 0 - return ret - - def get_decoded_content(self): - """ - Returns the decoded content based on the current Content-Encoding - header. - Doesn't change the message iteself or its headers. - """ - ce = self.headers.get_first("content-encoding") - if not self.body or ce not in encoding.ENCODINGS: - return self.body - return encoding.decode(ce, self.body) - - def decode(self): - """ - Decodes body based on the current Content-Encoding header, then - removes the header. If there is no Content-Encoding header, no - action is taken. - - Returns True if decoding succeeded, False otherwise. - """ - ce = self.headers.get_first("content-encoding") - if not self.body or ce not in encoding.ENCODINGS: - return False - data = encoding.decode(ce, self.body) - if data is None: - return False - self.body = data - del self.headers["content-encoding"] - return True - - def encode(self, e): - """ - Encodes body with the encoding e, where e is "gzip", "deflate" - or "identity". - """ - # FIXME: Error if there's an existing encoding header? - self.body = encoding.encode(e, self.body) - self.headers["content-encoding"] = [e] - - def copy(self): - c = copy.copy(self) - c.headers = self.headers.copy() - return c - - def replace(self, pattern, repl, *args, **kwargs): - """ - Replaces a regular expression pattern with repl in both the headers - and the body of the message. Encoded body will be decoded - before replacement, and re-encoded afterwards. - - Returns the number of replacements made. - """ - with decoded(self): - self.body, c = utils.safe_subn( - pattern, repl, self.body, *args, **kwargs - ) - c += self.headers.replace(pattern, repl, *args, **kwargs) - return c - - -class HTTPRequest(MessageMixin, semantics.Request): - """ - An HTTP request. - - Exposes the following attributes: - - method: HTTP method - - scheme: URL scheme (http/https) - - host: Target hostname of the request. This is not neccessarily the - directy upstream server (which could be another proxy), but it's always - the target server we want to reach at the end. This attribute is either - inferred from the request itself (absolute-form, authority-form) or from - the connection metadata (e.g. the host in reverse proxy mode). - - port: Destination port - - path: Path portion of the URL (not present in authority-form) - - httpversion: HTTP version tuple, e.g. (1,1) - - headers: odict.ODictCaseless object - - content: Content of the request, None, or CONTENT_MISSING if there - is content associated, but not present. CONTENT_MISSING evaluates - to False to make checking for the presence of content natural. - - form_in: The request form which mitmproxy has received. The following - values are possible: - - - relative (GET /index.html, OPTIONS *) (covers origin form and - asterisk form) - - absolute (GET http://example.com:80/index.html) - - authority-form (CONNECT example.com:443) - Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 - - form_out: The request form which mitmproxy will send out to the - destination - - timestamp_start: Timestamp indicating when request transmission started - - timestamp_end: Timestamp indicating when request transmission ended - """ - - def __init__( - self, - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - body, - timestamp_start=None, - timestamp_end=None, - form_out=None, - ): - semantics.Request.__init__( - self, - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - body, - timestamp_start, - timestamp_end, - ) - self.form_out = form_out or form_in - - # Have this request's cookies been modified by sticky cookies or auth? - self.stickycookie = False - self.stickyauth = False - - # Is this request replayed? - self.is_replay = False - - _stateobject_attributes = MessageMixin._stateobject_attributes.copy() - _stateobject_attributes.update( - form_in=str, - method=str, - scheme=str, - host=str, - port=int, - path=str, - form_out=str, - is_replay=bool - ) - - @classmethod - def from_state(cls, state): - f = cls( - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None) - f.load_state(state) - return f - - @classmethod - def from_protocol( - self, - protocol, - *args, - **kwargs - ): - req = protocol.read_request(*args, **kwargs) - return self.wrap(req) - - @classmethod - def wrap(self, request): - req = HTTPRequest( - form_in=request.form_in, - method=request.method, - scheme=request.scheme, - host=request.host, - port=request.port, - path=request.path, - httpversion=request.httpversion, - headers=request.headers, - body=request.body, - timestamp_start=request.timestamp_start, - timestamp_end=request.timestamp_end, - form_out=(request.form_out if hasattr(request, 'form_out') else None), - ) - if hasattr(request, 'stream_id'): - req.stream_id = request.stream_id - return req - - def __hash__(self): - return id(self) - - def replace(self, pattern, repl, *args, **kwargs): - """ - Replaces a regular expression pattern with repl in the headers, the - request path and the body of the request. Encoded content will be - decoded before replacement, and re-encoded afterwards. - - Returns the number of replacements made. - """ - c = MessageMixin.replace(self, pattern, repl, *args, **kwargs) - self.path, pc = utils.safe_subn( - pattern, repl, self.path, *args, **kwargs - ) - c += pc - return c - - -class HTTPResponse(MessageMixin, semantics.Response): - """ - An HTTP response. - - Exposes the following attributes: - - httpversion: HTTP version tuple, e.g. (1, 0), (1, 1), or (2, 0) - - status_code: HTTP response status code - - msg: HTTP response message - - headers: ODict Caseless object - - content: Content of the request, None, or CONTENT_MISSING if there - is content associated, but not present. CONTENT_MISSING evaluates - to False to make checking for the presence of content natural. - - timestamp_start: Timestamp indicating when request transmission started - - timestamp_end: Timestamp indicating when request transmission ended - """ - - def __init__( - self, - httpversion, - status_code, - msg, - headers, - body, - timestamp_start=None, - timestamp_end=None, - ): - semantics.Response.__init__( - self, - httpversion, - status_code, - msg, - headers, - body, - timestamp_start=timestamp_start, - timestamp_end=timestamp_end, - ) - - # Is this request replayed? - self.is_replay = False - self.stream = False - - _stateobject_attributes = MessageMixin._stateobject_attributes.copy() - _stateobject_attributes.update( - status_code=int, - msg=str - ) - - @classmethod - def from_state(cls, state): - f = cls(None, None, None, None, None) - f.load_state(state) - return f - - @classmethod - def from_protocol( - self, - protocol, - *args, - **kwargs - ): - resp = protocol.read_response(*args, **kwargs) - return self.wrap(resp) - - @classmethod - def wrap(self, response): - resp = HTTPResponse( - httpversion=response.httpversion, - status_code=response.status_code, - msg=response.msg, - headers=response.headers, - body=response.body, - timestamp_start=response.timestamp_start, - timestamp_end=response.timestamp_end, - ) - if hasattr(response, 'stream_id'): - resp.stream_id = response.stream_id - return resp - - def _refresh_cookie(self, c, delta): - """ - Takes a cookie string c and a time delta in seconds, and returns - a refreshed cookie string. - """ - c = Cookie.SimpleCookie(str(c)) - for i in c.values(): - if "expires" in i: - d = parsedate_tz(i["expires"]) - if d: - d = mktime_tz(d) + delta - i["expires"] = formatdate(d) - else: - # This can happen when the expires tag is invalid. - # reddit.com sends a an expires tag like this: "Thu, 31 Dec - # 2037 23:59:59 GMT", which is valid RFC 1123, but not - # strictly correct according to the cookie spec. Browsers - # appear to parse this tolerantly - maybe we should too. - # For now, we just ignore this. - del i["expires"] - return c.output(header="").strip() - - def refresh(self, now=None): - """ - This fairly complex and heuristic function refreshes a server - response for replay. - - - It adjusts date, expires and last-modified headers. - - It adjusts cookie expiration. - """ - if not now: - now = time.time() - delta = now - self.timestamp_start - refresh_headers = [ - "date", - "expires", - "last-modified", - ] - for i in refresh_headers: - if i in self.headers: - d = parsedate_tz(self.headers[i][0]) - if d: - new = mktime_tz(d) + delta - self.headers[i] = [formatdate(new)] - c = [] - for i in self.headers["set-cookie"]: - c.append(self._refresh_cookie(i, delta)) - if c: - self.headers["set-cookie"] = c diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py deleted file mode 100644 index c663f0c5..00000000 --- a/libmproxy/protocol/primitives.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import absolute_import -import copy -import uuid - -from .. import stateobject, utils, version -from ..proxy.connection import ClientConnection, ServerConnection - -KILL = 0 # const for killed requests - - -class Error(stateobject.StateObject): - """ - An Error. - - This is distinct from an protocol error response (say, a HTTP code 500), - which is represented by a normal HTTPResponse object. This class is - responsible for indicating errors that fall outside of normal protocol - communications, like interrupted connections, timeouts, protocol errors. - - Exposes the following attributes: - - flow: Flow object - msg: Message describing the error - timestamp: Seconds since the epoch - """ - - def __init__(self, msg, timestamp=None): - """ - @type msg: str - @type timestamp: float - """ - self.flow = None # will usually be set by the flow backref mixin - self.msg = msg - self.timestamp = timestamp or utils.timestamp() - - _stateobject_attributes = dict( - msg=str, - timestamp=float - ) - - def __str__(self): - return self.msg - - @classmethod - def from_state(cls, state): - # the default implementation assumes an empty constructor. Override - # accordingly. - f = cls(None) - f.load_state(state) - return f - - def copy(self): - c = copy.copy(self) - return c - - -class Flow(stateobject.StateObject): - """ - A Flow is a collection of objects representing a single transaction. - This class is usually subclassed for each protocol, e.g. HTTPFlow. - """ - - def __init__(self, type, client_conn, server_conn, live=None): - self.type = type - self.id = str(uuid.uuid4()) - self.client_conn = client_conn - """@type: ClientConnection""" - self.server_conn = server_conn - """@type: ServerConnection""" - self.live = live - """@type: LiveConnection""" - - self.error = None - """@type: Error""" - self.intercepted = False - """@type: bool""" - self._backup = None - self.reply = None - - _stateobject_attributes = dict( - id=str, - error=Error, - client_conn=ClientConnection, - server_conn=ServerConnection, - type=str, - intercepted=bool - ) - - def get_state(self, short=False): - d = super(Flow, self).get_state(short) - d.update(version=version.IVERSION) - if self._backup and self._backup != d: - if short: - d.update(modified=True) - else: - d.update(backup=self._backup) - return d - - def __eq__(self, other): - return self is other - - def copy(self): - f = copy.copy(self) - - f.id = str(uuid.uuid4()) - f.live = False - f.client_conn = self.client_conn.copy() - f.server_conn = self.server_conn.copy() - - if self.error: - f.error = self.error.copy() - return f - - def modified(self): - """ - Has this Flow been modified? - """ - if self._backup: - return self._backup != self.get_state() - else: - return False - - def backup(self, force=False): - """ - Save a backup of this Flow, which can be reverted to using a - call to .revert(). - """ - if not self._backup: - self._backup = self.get_state() - - def revert(self): - """ - Revert to the last backed up state. - """ - if self._backup: - self.load_state(self._backup) - self._backup = None - - def kill(self, master): - """ - Kill this request. - """ - self.error = Error("Connection killed") - self.intercepted = False - self.reply(KILL) - master.handle_error(self) - - def intercept(self, master): - """ - Intercept this Flow. Processing will stop until accept_intercept is - called. - """ - if self.intercepted: - return - self.intercepted = True - master.handle_intercept(self) - - def accept_intercept(self, master): - """ - Continue with the flow - called after an intercept(). - """ - if not self.intercepted: - return - self.intercepted = False - self.reply() - master.handle_accept_intercept(self) diff --git a/libmproxy/protocol/rawtcp.py b/libmproxy/protocol/rawtcp.py new file mode 100644 index 00000000..86468773 --- /dev/null +++ b/libmproxy/protocol/rawtcp.py @@ -0,0 +1,66 @@ +from __future__ import (absolute_import, print_function, division) +import socket +import select + +from OpenSSL import SSL + +from netlib.tcp import NetLibError +from netlib.utils import cleanBin +from ..exceptions import ProtocolException +from .base import Layer + + +class RawTCPLayer(Layer): + chunk_size = 4096 + + def __init__(self, ctx, logging=True): + self.logging = logging + super(RawTCPLayer, self).__init__(ctx) + + def __call__(self): + self.connect() + + buf = memoryview(bytearray(self.chunk_size)) + + client = self.client_conn.connection + server = self.server_conn.connection + conns = [client, server] + + try: + while True: + r, _, _ = select.select(conns, [], [], 10) + for conn in r: + dst = server if conn == client else client + + size = conn.recv_into(buf, self.chunk_size) + if not size: + conns.remove(conn) + # Shutdown connection to the other peer + if isinstance(conn, SSL.Connection): + # We can't half-close a connection, so we just close everything here. + # Sockets will be cleaned up on a higher level. + return + else: + dst.shutdown(socket.SHUT_WR) + + if len(conns) == 0: + return + continue + + dst.sendall(buf[:size]) + + if self.logging: + # log messages are prepended with the client address, + # hence the "weird" direction string. + if dst == server: + direction = "-> tcp -> {}".format(repr(self.server_conn.address)) + else: + direction = "<- tcp <- {}".format(repr(self.server_conn.address)) + data = cleanBin(buf[:size].tobytes()) + self.log( + "{}\r\n{}".format(direction, data), + "info" + ) + + except (socket.error, NetLibError, SSL.Error) as e: + raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py new file mode 100644 index 00000000..b85a6595 --- /dev/null +++ b/libmproxy/protocol/tls.py @@ -0,0 +1,288 @@ +from __future__ import (absolute_import, print_function, division) + +import struct + +from construct import ConstructError + +from netlib.tcp import NetLibError, NetLibInvalidCertificateError +from netlib.http.http1 import HTTP1Protocol +from ..contrib.tls._constructs import ClientHello +from ..exceptions import ProtocolException +from .base import Layer + + +def is_tls_record_magic(d): + """ + Returns: + True, if the passed bytes start with the TLS record magic bytes. + False, otherwise. + """ + d = d[:3] + + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + return ( + len(d) == 3 and + d[0] == '\x16' and + d[1] == '\x03' and + d[2] in ('\x00', '\x01', '\x02', '\x03') + ) + + +class TlsLayer(Layer): + def __init__(self, ctx, client_tls, server_tls): + self.client_sni = None + self.client_alpn_protocols = None + + super(TlsLayer, self).__init__(ctx) + self._client_tls = client_tls + self._server_tls = server_tls + + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values. + + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + + client_tls_requires_server_cert = ( + self._client_tls and self._server_tls and not self.config.no_upstream_cert + ) + + if self._client_tls: + self._parse_client_hello() + + if client_tls_requires_server_cert: + self._establish_tls_with_client_and_server() + elif self._client_tls: + self._establish_tls_with_client() + + layer = self.ctx.next_layer(self) + layer() + + def __repr__(self): + if self._client_tls and self._server_tls: + return "TlsLayer(client and server)" + elif self._client_tls: + return "TlsLayer(client)" + elif self._server_tls: + return "TlsLayer(server)" + else: + return "TlsLayer(inactive)" + + def _get_client_hello(self): + """ + Peek into the socket and read all records that contain the initial client hello message. + + Returns: + The raw handshake packet bytes, without TLS record header(s). + """ + client_hello = "" + client_hello_size = 1 + offset = 0 + while len(client_hello) < client_hello_size: + record_header = self.client_conn.rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) != 5: + raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) + record_size = struct.unpack("!H", record_header[3:])[0] + 5 + record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) + client_hello += record_body + offset += record_size + client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 + return client_hello + + def _parse_client_hello(self): + """ + Peek into the connection, read the initial client hello and parse it to obtain ALPN values. + """ + try: + raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + except ProtocolException as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + return + + try: + client_hello = ClientHello.parse(raw_client_hello) + except ConstructError as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") + return + + for extension in client_hello.extensions: + if extension.type == 0x00: + if len(extension.server_names) != 1 or extension.server_names[0].type != 0: + self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") + self.client_sni = extension.server_names[0].name + elif extension.type == 0x10: + self.client_alpn_protocols = list(extension.alpn_protocols) + + self.log( + "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), + "debug" + ) + + def connect(self): + if not self.server_conn: + self.ctx.connect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def reconnect(self): + self.ctx.reconnect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + self.ctx.set_server(address, server_tls, sni, depth) + if depth == 1 and server_tls is not None: + self._sni_from_server_change = sni + self._server_tls = server_tls + + @property + def sni_for_server_connection(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self.client_sni + + @property + def alpn_for_client_connection(self): + return self.server_conn.get_alpn_proto_negotiated() + + def __alpn_select_callback(self, conn_, options): + """ + Once the client signals the alternate protocols it supports, + we reconnect upstream with the same list and pass the server's choice down to the client. + """ + + # This gets triggered if we haven't established an upstream connection yet. + default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 + # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 + + if self.alpn_for_client_connection in options: + choice = bytes(self.alpn_for_client_connection) + elif default_alpn in options: + choice = bytes(default_alpn) + else: + choice = options[0] + self.log("ALPN for client: %s" % choice, "debug") + return choice + + def _establish_tls_with_client_and_server(self): + self.ctx.connect() + + # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless + # to send an error message over TLS. + try: + self._establish_tls_with_server() + except Exception as e: + try: + self._establish_tls_with_client() + except: + pass + raise e + + self._establish_tls_with_client() + + def _establish_tls_with_client(self): + self.log("Establish TLS with client", "debug") + cert, key, chain_file = self._find_cert() + + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file, + alpn_select_callback=self.__alpn_select_callback, + ) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) + + def _establish_tls_with_server(self): + self.log("Establish TLS with server", "debug") + try: + # We only support http/1.1 and h2. + # If the server only supports spdy (next to http/1.1), it may select that + # and mitmproxy would enter TCP passthrough mode, which we want to avoid. + deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") + if self.client_alpn_protocols: + alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + else: + alpn = None + + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni_for_server_connection, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + alpn_protos=alpn, + ) + tls_cert_err = self.server_conn.ssl_verification_error + if tls_cert_err is not None: + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except NetLibInvalidCertificateError as e: + tls_cert_err = self.server_conn.ssl_verification_error + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + + self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") + + def _find_cert(self): + host = self.server_conn.address.host + sans = set() + # Incorporate upstream certificate + use_upstream_cert = ( + self.server_conn and + self.server_conn.tls_established and + (not self.config.no_upstream_cert) + ) + if use_upstream_cert: + upstream_cert = self.server_conn.cert + sans.update(upstream_cert.altnames) + if upstream_cert.cn: + sans.add(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self.client_sni: + sans.add(self.client_sni) + if self._sni_from_server_change: + sans.add(self._sni_from_server_change) + + sans.discard(host) + return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py deleted file mode 100644 index 61b9a77e..00000000 --- a/libmproxy/protocol2/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import (absolute_import, print_function, division) -from .root_context import RootContext -from .socks_proxy import Socks5Proxy -from .reverse_proxy import ReverseProxy -from .http_proxy import HttpProxy, HttpUpstreamProxy -from .transparent_proxy import TransparentProxy -from .http import make_error_response - -__all__ = [ - "RootContext", - "Socks5Proxy", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "TransparentProxy", - "make_error_response" -] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py deleted file mode 100644 index a508ae8b..00000000 --- a/libmproxy/protocol2/http.py +++ /dev/null @@ -1,588 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from netlib import tcp -from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError -from netlib.http.semantics import CONTENT_MISSING -from netlib import odict -from netlib.tcp import NetLibError, Address -from netlib.http.http1 import HTTP1Protocol -from netlib.http.http2 import HTTP2Protocol - -from .. import version, utils -from ..exceptions import InvalidCredentials, HttpException, ProtocolException -from .layer import Layer -from ..proxy import Kill -from libmproxy.protocol import KILL, Error -from libmproxy.protocol.http import HTTPFlow -from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest - - -class _HttpLayer(Layer): - supports_streaming = False - - def read_request(self): - raise NotImplementedError() - - def send_request(self, request): - raise NotImplementedError() - - def read_response(self, request_method): - raise NotImplementedError() - - def send_response(self, response): - raise NotImplementedError() - - -class _StreamingHttpLayer(_HttpLayer): - supports_streaming = True - - def read_response_headers(self): - raise NotImplementedError - - def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): - raise NotImplementedError() - yield "this is a generator" - - def send_response_headers(self, response): - raise NotImplementedError - - def send_response_body(self, response, chunks): - raise NotImplementedError() - - -class Http1Layer(_StreamingHttpLayer): - def __init__(self, ctx, mode): - super(Http1Layer, self).__init__(ctx) - self.mode = mode - self.client_protocol = HTTP1Protocol(self.client_conn) - self.server_protocol = HTTP1Protocol(self.server_conn) - - def read_request(self): - return HTTPRequest.from_protocol( - self.client_protocol, - body_size_limit=self.config.body_size_limit - ) - - def send_request(self, request): - self.server_conn.send(self.server_protocol.assemble(request)) - - def read_response(self, request_method): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=request_method, - body_size_limit=self.config.body_size_limit, - include_body=True - ) - - def send_response(self, response): - self.client_conn.send(self.client_protocol.assemble(response)) - - def read_response_headers(self): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=None, # does not matter if we don't read the body. - body_size_limit=self.config.body_size_limit, - include_body=False - ) - - def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): - return self.server_protocol.read_http_body_chunked( - headers, - self.config.body_size_limit, - request_method, - response_code, - False, - max_chunk_size - ) - - def send_response_headers(self, response): - h = self.client_protocol._assemble_response_first_line(response) - self.client_conn.wfile.write(h + "\r\n") - h = self.client_protocol._assemble_response_headers( - response, - preserve_transfer_encoding=True - ) - self.client_conn.send(h + "\r\n") - - def send_response_body(self, response, chunks): - if self.client_protocol.has_chunked_encoding(response.headers): - chunks = ( - "%d\r\n%s\r\n" % (len(chunk), chunk) - for chunk in chunks - ) - for chunk in chunks: - self.client_conn.send(chunk) - - def connect(self): - self.ctx.connect() - self.server_protocol = HTTP1Protocol(self.server_conn) - - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP1Protocol(self.server_conn) - - def set_server(self, *args, **kwargs): - self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP1Protocol(self.server_conn) - - def __call__(self): - layer = HttpLayer(self, self.mode) - layer() - - -# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. -class Http2Layer(_HttpLayer): - def __init__(self, ctx, mode): - super(Http2Layer, self).__init__(ctx) - self.mode = mode - self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - - def read_request(self): - request = HTTPRequest.from_protocol( - self.client_protocol, - body_size_limit=self.config.body_size_limit - ) - self._stream_id = request.stream_id - return request - - def send_request(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames - self.server_conn.send(self.server_protocol.assemble(message)) - - def read_response(self, request_method): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=request_method, - body_size_limit=self.config.body_size_limit, - include_body=True, - stream_id=self._stream_id - ) - - def send_response(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames - self.client_conn.send(self.client_protocol.assemble(message)) - - def connect(self): - self.ctx.connect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def set_server(self, *args, **kwargs): - self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def __call__(self): - self.server_protocol.perform_connection_preface() - layer = HttpLayer(self, self.mode) - layer() - - def handle_unexpected_frame(self, frm): - self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") - - -def make_error_response(status_code, message, headers=None): - response = status_codes.RESPONSES.get(status_code, "Unknown") - body = """ - - - %d %s - - %s - - """.strip() % (status_code, response, message) - - if not headers: - headers = odict.ODictCaseless() - headers["Server"] = [version.NAMEVERSION] - headers["Connection"] = ["close"] - headers["Content-Length"] = [len(body)] - headers["Content-Type"] = ["text/html"] - - return HTTPResponse( - (1, 1), # FIXME: Should be a string. - status_code, - response, - headers, - body, - ) - - -def make_connect_request(address): - address = Address.wrap(address) - return HTTPRequest( - "authority", "CONNECT", None, address.host, address.port, None, (1, 1), - odict.ODictCaseless(), "" - ) - - -def make_connect_response(httpversion): - headers = odict.ODictCaseless([ - ["Content-Length", "0"], - ["Proxy-Agent", version.NAMEVERSION] - ]) - return HTTPResponse( - httpversion, - 200, - "Connection established", - headers, - "", - ) - - -class ConnectServerConnection(object): - """ - "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. - """ - - def __init__(self, address, ctx): - self.address = tcp.Address.wrap(address) - self._ctx = ctx - - @property - def via(self): - return self._ctx.server_conn - - def __getattr__(self, item): - return getattr(self.via, item) - - -class UpstreamConnectLayer(Layer): - def __init__(self, ctx, connect_request): - super(UpstreamConnectLayer, self).__init__(ctx) - self.connect_request = connect_request - self.server_conn = ConnectServerConnection( - (connect_request.host, connect_request.port), - self.ctx - ) - - def __call__(self): - layer = self.ctx.next_layer(self) - layer() - - def connect(self): - if not self.server_conn: - self.ctx.connect() - self.send_request(self.connect_request) - else: - pass # swallow the message - - def reconnect(self): - self.ctx.reconnect() - self.send_request(self.connect_request) - resp = self.read_response("CONNECT") - if resp.code != 200: - raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") - - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1: - if self.ctx.server_conn: - self.ctx.reconnect() - address = Address.wrap(address) - self.connect_request.host = address.host - self.connect_request.port = address.port - self.server_conn.address = address - else: - self.ctx.set_server(address, server_tls, sni, depth - 1) - - -class HttpLayer(Layer): - def __init__(self, ctx, mode): - super(HttpLayer, self).__init__(ctx) - self.mode = mode - self.__original_server_conn = None - "Contains the original destination in transparent mode, which needs to be restored" - "if an inline script modified the target server for a single http request" - - def __call__(self): - if self.mode == "transparent": - self.__original_server_conn = self.server_conn - while True: - try: - flow = HTTPFlow(self.client_conn, self.server_conn, live=self) - - try: - request = self.read_request() - except tcp.NetLibError: - # don't throw an error for disconnects that happen - # before/between requests. - return - - self.log("request", "debug", [repr(request)]) - - # Handle Proxy Authentication - self.authenticate(request) - - # Regular Proxy Mode: Handle CONNECT - if self.mode == "regular" and request.form_in == "authority": - self.handle_regular_mode_connect(request) - return - - # Make sure that the incoming request matches our expectations - self.validate_request(request) - - flow.request = request - self.process_request_hook(flow) - - if not flow.response: - self.establish_server_connection(flow) - self.get_response_from_server(flow) - - self.send_response_to_client(flow) - - if self.check_close_connection(flow): - return - - # TODO: Implement HTTP Upgrade - - # Upstream Proxy Mode: Handle CONNECT - if flow.request.form_in == "authority" and flow.response.code == 200: - self.handle_upstream_mode_connect(flow.request.copy()) - return - - except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: - if flow.request and not flow.response: - flow.error = Error(repr(e)) - self.channel.ask("error", flow) - try: - self.send_response(make_error_response( - getattr(e, "code", 502), - repr(e) - )) - except NetLibError: - pass - if isinstance(e, ProtocolException): - raise e - else: - raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) - finally: - flow.live = False - - def handle_regular_mode_connect(self, request): - self.set_server((request.host, request.port)) - self.send_response(make_connect_response(request.httpversion)) - layer = self.ctx.next_layer(self) - layer() - - def handle_upstream_mode_connect(self, connect_request): - layer = UpstreamConnectLayer(self, connect_request) - layer() - - def check_close_connection(self, flow): - """ - Checks if the connection should be closed depending on the HTTP - semantics. Returns True, if so. - """ - - # TODO: add logic for HTTP/2 - - close_connection = ( - http1.HTTP1Protocol.connection_close( - flow.request.httpversion, - flow.request.headers - ) or http1.HTTP1Protocol.connection_close( - flow.response.httpversion, - flow.response.headers - ) or http1.HTTP1Protocol.expected_http_body_size( - flow.response.headers, - False, - flow.request.method, - flow.response.code) == -1 - ) - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 - # and no Content-Length header - - return False - return close_connection - - def send_response_to_client(self, flow): - if not (self.supports_streaming and flow.response.stream): - # no streaming: - # we already received the full response from the server and can - # send it to the client straight away. - self.send_response(flow.response) - else: - # streaming: - # First send the headers and then transfer the response incrementally - self.send_response_headers(flow.response) - chunks = self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code, - max_chunk_size=4096 - ) - if callable(flow.response.stream): - chunks = flow.response.stream(chunks) - self.send_response_body(flow.response, chunks) - flow.response.timestamp_end = utils.timestamp() - - def get_response_from_server(self, flow): - def get_response(): - self.send_request(flow.request) - if self.supports_streaming: - flow.response = self.read_response_headers() - else: - flow.response = self.read_response() - - try: - get_response() - except (tcp.NetLibError, HttpErrorConnClosed) as v: - self.log( - "server communication error: %s" % repr(v), - level="debug" - ) - # In any case, we try to reconnect at least once. This is - # necessary because it might be possible that we already - # initiated an upstream connection after clientconnect that - # has already been expired, e.g consider the following event - # log: - # > clientconnect (transparent mode destination known) - # > serverconnect (required for client tls handshake) - # > read n% of large request - # > server detects timeout, disconnects - # > read (100-n)% of large request - # > send large request upstream - self.reconnect() - get_response() - - # call the appropriate script hook - this is an opportunity for an - # inline script to set flow.stream = True - flow = self.channel.ask("responseheaders", flow) - if flow is None or flow == KILL: - raise Kill() - - if self.supports_streaming: - if flow.response.stream: - flow.response.content = CONTENT_MISSING - else: - flow.response.content = "".join(self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code - )) - flow.response.timestamp_end = utils.timestamp() - - # no further manipulation of self.server_conn beyond this point - # we can safely set it as the final attribute value here. - flow.server_conn = self.server_conn - - self.log( - "response", - "debug", - [repr(flow.response)] - ) - response_reply = self.channel.ask("response", flow) - if response_reply is None or response_reply == KILL: - raise Kill() - - def process_request_hook(self, flow): - # Determine .scheme, .host and .port attributes for inline scripts. - # For absolute-form requests, they are directly given in the request. - # For authority-form requests, we only need to determine the request scheme. - # For relative-form requests, we need to determine host and port as - # well. - if self.mode == "regular": - pass # only absolute-form at this point, nothing to do here. - elif self.mode == "upstream": - if flow.request.form_in == "authority": - flow.request.scheme = "http" # pseudo value - else: - flow.request.host = self.__original_server_conn.address.host - flow.request.port = self.__original_server_conn.address.port - flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" - - request_reply = self.channel.ask("request", flow) - if request_reply is None or request_reply == KILL: - raise Kill() - if isinstance(request_reply, HTTPResponse): - flow.response = request_reply - return - - def establish_server_connection(self, flow): - address = tcp.Address((flow.request.host, flow.request.port)) - tls = (flow.request.scheme == "https") - - if self.mode == "regular" or self.mode == "transparent": - # If there's an existing connection that doesn't match our expectations, kill it. - if address != self.server_conn.address or tls != self.server_conn.ssl_established: - self.set_server(address, tls, address.host) - # Establish connection is neccessary. - if not self.server_conn: - self.connect() - - # SetServer is not guaranteed to work with TLS: - # If there's not TlsLayer below which could catch the exception, - # TLS will not be established. - if tls and not self.server_conn.tls_established: - raise ProtocolException( - "Cannot upgrade to SSL, no TLS layer on the protocol stack.") - else: - if not self.server_conn: - self.connect() - if tls: - raise HttpException("Cannot change scheme in upstream proxy mode.") - """ - # This is a very ugly (untested) workaround to solve a very ugly problem. - if self.server_conn and self.server_conn.tls_established and not ssl: - self.reconnect() - elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: - if self.server_conn.tls_established: - self.reconnect() - - self.send_request(make_connect_request(address)) - tls_layer = TlsLayer(self, False, True) - tls_layer._establish_tls_with_server() - """ - - def validate_request(self, request): - if request.form_in == "absolute" and request.scheme != "http": - self.send_response( - make_error_response(400, "Invalid request scheme: %s" % request.scheme)) - raise HttpException("Invalid request scheme: %s" % request.scheme) - - expected_request_forms = { - "regular": ("absolute",), # an authority request would already be handled. - "upstream": ("authority", "absolute"), - "transparent": ("relative",) - } - - allowed_request_forms = expected_request_forms[self.mode] - if request.form_in not in allowed_request_forms: - err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( - " or ".join(allowed_request_forms), request.form_in - ) - self.send_response(make_error_response(400, err_message)) - raise HttpException(err_message) - - if self.mode == "regular": - request.form_out = "relative" - - def authenticate(self, request): - if self.config.authenticator: - if self.config.authenticator.authenticate(request.headers): - self.config.authenticator.clean(request.headers) - else: - self.send_response(make_error_response( - 407, - "Proxy Authentication Required", - odict.ODictCaseless( - [ - [k, v] for k, v in - self.config.authenticator.auth_challenge_headers().items() - ]) - )) - raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py deleted file mode 100644 index 2876c022..00000000 --- a/libmproxy/protocol2/http_proxy.py +++ /dev/null @@ -1,26 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from .layer import Layer, ServerConnectionMixin - - -class HttpProxy(Layer, ServerConnectionMixin): - def __call__(self): - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() - - -class HttpUpstreamProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx, server_address): - super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) - - def __call__(self): - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() diff --git a/libmproxy/protocol2/http_replay.py b/libmproxy/protocol2/http_replay.py deleted file mode 100644 index 872ef9cd..00000000 --- a/libmproxy/protocol2/http_replay.py +++ /dev/null @@ -1,95 +0,0 @@ -import threading -from netlib.http import HttpError -from netlib.http.http1 import HTTP1Protocol -from netlib.tcp import NetLibError - -from ..controller import Channel -from ..protocol import KILL, Error -from ..protocol.http_wrappers import HTTPResponse -from ..proxy import Log, Kill -from ..proxy.connection import ServerConnection -from .http import make_connect_request - - -class RequestReplayThread(threading.Thread): - name = "RequestReplayThread" - - def __init__(self, config, flow, masterq, should_exit): - """ - masterqueue can be a queue or None, if no scripthooks should be - processed. - """ - self.config, self.flow = config, flow - if masterq: - self.channel = Channel(masterq, should_exit) - else: - self.channel = None - super(RequestReplayThread, self).__init__() - - def run(self): - r = self.flow.request - form_out_backup = r.form_out - try: - self.flow.response = None - - # If we have a channel, run script hooks. - if self.channel: - request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == KILL: - raise Kill() - elif isinstance(request_reply, HTTPResponse): - self.flow.response = request_reply - - if not self.flow.response: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - server_address = self.config.upstream_server.address - server = ServerConnection(server_address) - server.connect() - protocol = HTTP1Protocol(server) - if r.scheme == "https": - connect_request = make_connect_request((r.host, r.port)) - server.send(protocol.assemble(connect_request)) - resp = protocol.read_response("CONNECT") - if resp.code != 200: - raise HttpError(502, "Upstream server refuses CONNECT request") - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - else: - r.form_out = "absolute" - else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - protocol = HTTP1Protocol(server) - if r.scheme == "https": - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - - server.send(protocol.assemble(r)) - self.flow.server_conn = server - self.flow.response = HTTPResponse.from_protocol( - protocol, - r.method, - body_size_limit=self.config.body_size_limit, - ) - if self.channel: - response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == KILL: - raise Kill() - except (HttpError, NetLibError) as v: - self.flow.error = Error(repr(v)) - if self.channel: - self.channel.ask("error", self.flow) - except Kill: - # KillSignal should only be raised if there's a channel in the - # first place. - self.channel.tell("log", Log("Connection killed", "info")) - finally: - r.form_out = form_out_backup diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py deleted file mode 100644 index 2b47cc26..00000000 --- a/libmproxy/protocol2/layer.py +++ /dev/null @@ -1,138 +0,0 @@ -""" -mitmproxy protocol architecture - -In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. -For example, the following scenarios depict possible scenarios (lowest layer first): - -Transparent HTTP proxy, no SSL: - TransparentModeLayer - HttpLayer - -Regular proxy, CONNECT request with WebSockets over SSL: - RegularModeLayer - HttpLayer - SslLayer - WebsocketLayer (or TcpLayer) - -Automated protocol detection by peeking into the buffer: - TransparentModeLayer - SslLayer - Http2Layer - -Communication between layers is done as follows: - - lower layers provide context information to higher layers - - higher layers can call functions provided by lower layers, - which are propagated until they reach a suitable layer. - -Further goals: - - Connections should always be peekable to make automatic protocol detection work. - - Upstream connections should be established as late as possible; - inline scripts shall have a chance to handle everything locally. -""" -from __future__ import (absolute_import, print_function, division) -from netlib import tcp -from ..proxy import Log -from ..proxy.connection import ServerConnection -from ..exceptions import ProtocolException - - -class _LayerCodeCompletion(object): - """ - Dummy class that provides type hinting in PyCharm, which simplifies development a lot. - """ - - def __init__(self, *args, **kwargs): - super(_LayerCodeCompletion, self).__init__(*args, **kwargs) - if True: - return - self.config = None - """@type: libmproxy.proxy.config.ProxyConfig""" - self.client_conn = None - """@type: libmproxy.proxy.connection.ClientConnection""" - self.channel = None - """@type: libmproxy.controller.Channel""" - - -class Layer(_LayerCodeCompletion): - def __init__(self, ctx, *args, **kwargs): - """ - Args: - ctx: The (read-only) higher layer. - """ - super(Layer, self).__init__(*args, **kwargs) - self.ctx = ctx - - def __call__(self): - """ - Logic of the layer. - Raises: - ProtocolException in case of protocol exceptions. - """ - raise NotImplementedError - - def __getattr__(self, name): - """ - Attributes not present on the current layer may exist on a higher layer. - """ - return getattr(self.ctx, name) - - def log(self, msg, level, subs=()): - full_msg = [ - "{}: {}".format(repr(self.client_conn.address), msg) - ] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - @property - def layers(self): - return [self] + self.ctx.layers - - def __repr__(self): - return type(self).__name__ - - -class ServerConnectionMixin(object): - """ - Mixin that provides a layer with the capabilities to manage a server connection. - """ - - def __init__(self, server_address=None): - super(ServerConnectionMixin, self).__init__() - self.server_conn = ServerConnection(server_address) - - def reconnect(self): - address = self.server_conn.address - self._disconnect() - self.server_conn.address = address - self.connect() - - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1: - if self.server_conn: - self._disconnect() - self.log("Set new server address: " + repr(address), "debug") - self.server_conn.address = address - else: - self.ctx.set_server(address, server_tls, sni, depth - 1) - - def _disconnect(self): - """ - Deletes (and closes) an existing server connection. - """ - self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) - self.server_conn.finish() - self.server_conn.close() - # self.channel.tell("serverdisconnect", self) - self.server_conn = ServerConnection(None) - - def connect(self): - if not self.server_conn.address: - raise ProtocolException("Cannot connect to server, no server address given.") - self.log("serverconnect", "debug", [repr(self.server_conn.address)]) - try: - self.server_conn.connect() - except tcp.NetLibError as e: - raise ProtocolException( - "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py deleted file mode 100644 index b10217f1..00000000 --- a/libmproxy/protocol2/rawtcp.py +++ /dev/null @@ -1,66 +0,0 @@ -from __future__ import (absolute_import, print_function, division) -import socket -import select - -from OpenSSL import SSL - -from netlib.tcp import NetLibError -from netlib.utils import cleanBin -from ..exceptions import ProtocolException -from .layer import Layer - - -class RawTcpLayer(Layer): - chunk_size = 4096 - - def __init__(self, ctx, logging=True): - self.logging = logging - super(RawTcpLayer, self).__init__(ctx) - - def __call__(self): - self.connect() - - buf = memoryview(bytearray(self.chunk_size)) - - client = self.client_conn.connection - server = self.server_conn.connection - conns = [client, server] - - try: - while True: - r, _, _ = select.select(conns, [], [], 10) - for conn in r: - dst = server if conn == client else client - - size = conn.recv_into(buf, self.chunk_size) - if not size: - conns.remove(conn) - # Shutdown connection to the other peer - if isinstance(conn, SSL.Connection): - # We can't half-close a connection, so we just close everything here. - # Sockets will be cleaned up on a higher level. - return - else: - dst.shutdown(socket.SHUT_WR) - - if len(conns) == 0: - return - continue - - dst.sendall(buf[:size]) - - if self.logging: - # log messages are prepended with the client address, - # hence the "weird" direction string. - if dst == server: - direction = "-> tcp -> {}".format(repr(self.server_conn.address)) - else: - direction = "<- tcp <- {}".format(repr(self.server_conn.address)) - data = cleanBin(buf[:size].tobytes()) - self.log( - "{}\r\n{}".format(direction, data), - "info" - ) - - except (socket.error, NetLibError, SSL.Error) as e: - raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py deleted file mode 100644 index 3ca998d5..00000000 --- a/libmproxy/protocol2/reverse_proxy.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from .layer import Layer, ServerConnectionMixin - - -class ReverseProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx, server_address, server_tls): - super(ReverseProxy, self).__init__(ctx, server_address=server_address) - self.server_tls = server_tls - - def __call__(self): - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py deleted file mode 100644 index daea54bd..00000000 --- a/libmproxy/protocol2/root_context.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from netlib.http.http1 import HTTP1Protocol -from netlib.http.http2 import HTTP2Protocol - -from .rawtcp import RawTcpLayer -from .tls import TlsLayer, is_tls_record_magic -from .http import Http1Layer, Http2Layer -from .layer import ServerConnectionMixin -from .http_proxy import HttpProxy, HttpUpstreamProxy -from .reverse_proxy import ReverseProxy - - -class RootContext(object): - """ - The outmost context provided to the root layer. - As a consequence, every layer has .client_conn, .channel, .next_layer() and .config. - """ - - def __init__(self, client_conn, config, channel): - self.client_conn = client_conn # Client Connection - self.channel = channel # provides .ask() method to communicate with FlowMaster - self.config = config # Proxy Configuration - - def next_layer(self, top_layer): - """ - This function determines the next layer in the protocol stack. - - Arguments: - top_layer: the current top layer. - - Returns: - The next layer - """ - - # 1. Check for --ignore. - if self.config.check_ignore(top_layer.server_conn.address): - return RawTcpLayer(top_layer, logging=False) - - d = top_layer.client_conn.rfile.peek(3) - client_tls = is_tls_record_magic(d) - - # 2. Always insert a TLS layer, even if there's neither client nor server tls. - # An inline script may upgrade from http to https, - # in which case we need some form of TLS layer. - if isinstance(top_layer, ReverseProxy): - return TlsLayer(top_layer, client_tls, top_layer.server_tls) - if isinstance(top_layer, ServerConnectionMixin): - return TlsLayer(top_layer, client_tls, client_tls) - - # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. - if isinstance(top_layer, TlsLayer): - if isinstance(top_layer.ctx, HttpProxy): - return Http1Layer(top_layer, "regular") - if isinstance(top_layer.ctx, HttpUpstreamProxy): - return Http1Layer(top_layer, "upstream") - - # 4. Check for other TLS cases (e.g. after CONNECT). - if client_tls: - return TlsLayer(top_layer, True, True) - - # 4. Check for --tcp - if self.config.check_tcp(top_layer.server_conn.address): - return RawTcpLayer(top_layer) - - # 5. Check for TLS ALPN (HTTP1/HTTP2) - if isinstance(top_layer, TlsLayer): - alpn = top_layer.client_conn.get_alpn_proto_negotiated() - if alpn == HTTP2Protocol.ALPN_PROTO_H2: - return Http2Layer(top_layer, 'transparent') - if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: - return Http1Layer(top_layer, 'transparent') - - # 6. Assume HTTP1 by default - return Http1Layer(top_layer, 'transparent') - - # In a future version, we want to implement TCP passthrough as the last fallback, - # but we don't have the UI part ready for that. - # - # d = top_layer.client_conn.rfile.peek(3) - # is_ascii = ( - # len(d) == 3 and - # # better be safe here and don't expect uppercase... - # all(x in string.ascii_letters for x in d) - # ) - # # TODO: This could block if there are not enough bytes available? - # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) - # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) - - @property - def layers(self): - return [] - - def __repr__(self): - return "RootContext" diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py deleted file mode 100644 index 525520e8..00000000 --- a/libmproxy/protocol2/socks_proxy.py +++ /dev/null @@ -1,59 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from netlib import socks -from netlib.tcp import NetLibError -from ..exceptions import Socks5Exception -from .layer import Layer, ServerConnectionMixin - - -class Socks5Proxy(Layer, ServerConnectionMixin): - def __call__(self): - try: - # Parse Client Greeting - client_greet = socks.ClientGreeting.from_file(self.client_conn.rfile, fail_early=True) - client_greet.assert_socks5() - if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: - raise socks.SocksError( - socks.METHOD.NO_ACCEPTABLE_METHODS, - "mitmproxy only supports SOCKS without authentication" - ) - - # Send Server Greeting - server_greet = socks.ServerGreeting( - socks.VERSION.SOCKS5, - socks.METHOD.NO_AUTHENTICATION_REQUIRED - ) - server_greet.to_file(self.client_conn.wfile) - self.client_conn.wfile.flush() - - # Parse Connect Request - connect_request = socks.Message.from_file(self.client_conn.rfile) - connect_request.assert_socks5() - if connect_request.msg != socks.CMD.CONNECT: - raise socks.SocksError( - socks.REP.COMMAND_NOT_SUPPORTED, - "mitmproxy only supports SOCKS5 CONNECT." - ) - - # We always connect lazily, but we need to pretend to the client that we connected. - connect_reply = socks.Message( - socks.VERSION.SOCKS5, - socks.REP.SUCCEEDED, - connect_request.atyp, - # dummy value, we don't have an upstream connection yet. - connect_request.addr - ) - connect_reply.to_file(self.client_conn.wfile) - self.client_conn.wfile.flush() - - except (socks.SocksError, NetLibError) as e: - raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e) - - self.server_conn.address = connect_request.addr - - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py deleted file mode 100644 index 73bb12f3..00000000 --- a/libmproxy/protocol2/tls.py +++ /dev/null @@ -1,288 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -import struct - -from construct import ConstructError - -from netlib.tcp import NetLibError, NetLibInvalidCertificateError -from netlib.http.http1 import HTTP1Protocol -from ..contrib.tls._constructs import ClientHello -from ..exceptions import ProtocolException -from .layer import Layer - - -def is_tls_record_magic(d): - """ - Returns: - True, if the passed bytes start with the TLS record magic bytes. - False, otherwise. - """ - d = d[:3] - - # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 - # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello - return ( - len(d) == 3 and - d[0] == '\x16' and - d[1] == '\x03' and - d[2] in ('\x00', '\x01', '\x02', '\x03') - ) - - -class TlsLayer(Layer): - def __init__(self, ctx, client_tls, server_tls): - self.client_sni = None - self.client_alpn_protocols = None - - super(TlsLayer, self).__init__(ctx) - self._client_tls = client_tls - self._server_tls = server_tls - - self._sni_from_server_change = None - - def __call__(self): - """ - The strategy for establishing SSL is as follows: - First, we determine whether we need the server cert to establish ssl with the client. - If so, we first connect to the server and then to the client. - If not, we only connect to the client and do the server_ssl lazily on a Connect message. - - An additional complexity is that establish ssl with the server may require a SNI value from the client. - In an ideal world, we'd do the following: - 1. Start the SSL handshake with the client - 2. Check if the client sends a SNI. - 3. Pause the client handshake, establish SSL with the server. - 4. Finish the client handshake with the certificate from the server. - There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( - Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values. - - Further notes: - - OpenSSL 1.0.2 introduces a callback that would help here: - https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html - - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 - """ - - client_tls_requires_server_cert = ( - self._client_tls and self._server_tls and not self.config.no_upstream_cert - ) - - if self._client_tls: - self._parse_client_hello() - - if client_tls_requires_server_cert: - self._establish_tls_with_client_and_server() - elif self._client_tls: - self._establish_tls_with_client() - - layer = self.ctx.next_layer(self) - layer() - - def __repr__(self): - if self._client_tls and self._server_tls: - return "TlsLayer(client and server)" - elif self._client_tls: - return "TlsLayer(client)" - elif self._server_tls: - return "TlsLayer(server)" - else: - return "TlsLayer(inactive)" - - def _get_client_hello(self): - """ - Peek into the socket and read all records that contain the initial client hello message. - - Returns: - The raw handshake packet bytes, without TLS record header(s). - """ - client_hello = "" - client_hello_size = 1 - offset = 0 - while len(client_hello) < client_hello_size: - record_header = self.client_conn.rfile.peek(offset + 5)[offset:] - if not is_tls_record_magic(record_header) or len(record_header) != 5: - raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) - record_size = struct.unpack("!H", record_header[3:])[0] + 5 - record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] - if len(record_body) != record_size - 5: - raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) - client_hello += record_body - offset += record_size - client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 - return client_hello - - def _parse_client_hello(self): - """ - Peek into the connection, read the initial client hello and parse it to obtain ALPN values. - """ - try: - raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. - except ProtocolException as e: - self.log("Cannot parse Client Hello: %s" % repr(e), "error") - return - - try: - client_hello = ClientHello.parse(raw_client_hello) - except ConstructError as e: - self.log("Cannot parse Client Hello: %s" % repr(e), "error") - self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") - return - - for extension in client_hello.extensions: - if extension.type == 0x00: - if len(extension.server_names) != 1 or extension.server_names[0].type != 0: - self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") - self.client_sni = extension.server_names[0].name - elif extension.type == 0x10: - self.client_alpn_protocols = list(extension.alpn_protocols) - - self.log( - "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), - "debug" - ) - - def connect(self): - if not self.server_conn: - self.ctx.connect() - if self._server_tls and not self.server_conn.tls_established: - self._establish_tls_with_server() - - def reconnect(self): - self.ctx.reconnect() - if self._server_tls and not self.server_conn.tls_established: - self._establish_tls_with_server() - - def set_server(self, address, server_tls=None, sni=None, depth=1): - self.ctx.set_server(address, server_tls, sni, depth) - if depth == 1 and server_tls is not None: - self._sni_from_server_change = sni - self._server_tls = server_tls - - @property - def sni_for_server_connection(self): - if self._sni_from_server_change is False: - return None - else: - return self._sni_from_server_change or self.client_sni - - @property - def alpn_for_client_connection(self): - return self.server_conn.get_alpn_proto_negotiated() - - def __alpn_select_callback(self, conn_, options): - """ - Once the client signals the alternate protocols it supports, - we reconnect upstream with the same list and pass the server's choice down to the client. - """ - - # This gets triggered if we haven't established an upstream connection yet. - default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 - # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 - - if self.alpn_for_client_connection in options: - choice = bytes(self.alpn_for_client_connection) - elif default_alpn in options: - choice = bytes(default_alpn) - else: - choice = options[0] - self.log("ALPN for client: %s" % choice, "debug") - return choice - - def _establish_tls_with_client_and_server(self): - self.ctx.connect() - - # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless - # to send an error message over TLS. - try: - self._establish_tls_with_server() - except Exception as e: - try: - self._establish_tls_with_client() - except: - pass - raise e - - self._establish_tls_with_client() - - def _establish_tls_with_client(self): - self.log("Establish TLS with client", "debug") - cert, key, chain_file = self._find_cert() - - try: - self.client_conn.convert_to_ssl( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file, - alpn_select_callback=self.__alpn_select_callback, - ) - except NetLibError as e: - raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) - - def _establish_tls_with_server(self): - self.log("Establish TLS with server", "debug") - try: - # We only support http/1.1 and h2. - # If the server only supports spdy (next to http/1.1), it may select that - # and mitmproxy would enter TCP passthrough mode, which we want to avoid. - deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") - if self.client_alpn_protocols: - alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) - else: - alpn = None - - self.server_conn.establish_ssl( - self.config.clientcerts, - self.sni_for_server_connection, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify_options=self.config.openssl_verification_mode_server, - ca_path=self.config.openssl_trusted_cadir_server, - ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, - alpn_protos=alpn, - ) - tls_cert_err = self.server_conn.ssl_verification_error - if tls_cert_err is not None: - self.log( - "TLS verification failed for upstream server at depth %s with error: %s" % - (tls_cert_err['depth'], tls_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") - except NetLibInvalidCertificateError as e: - tls_cert_err = self.server_conn.ssl_verification_error - self.log( - "TLS verification failed for upstream server at depth %s with error: %s" % - (tls_cert_err['depth'], tls_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") - raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) - except NetLibError as e: - raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) - - self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") - - def _find_cert(self): - host = self.server_conn.address.host - sans = set() - # Incorporate upstream certificate - use_upstream_cert = ( - self.server_conn and - self.server_conn.tls_established and - (not self.config.no_upstream_cert) - ) - if use_upstream_cert: - upstream_cert = self.server_conn.cert - sans.update(upstream_cert.altnames) - if upstream_cert.cn: - sans.add(host) - host = upstream_cert.cn.decode("utf8").encode("idna") - # Also add SNI values. - if self.client_sni: - sans.add(self.client_sni) - if self._sni_from_server_change: - sans.add(self._sni_from_server_change) - - sans.discard(host) - return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py deleted file mode 100644 index e6ebf115..00000000 --- a/libmproxy/protocol2/transparent_proxy.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from ..exceptions import ProtocolException -from .. import platform -from .layer import Layer, ServerConnectionMixin - - -class TransparentProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx): - super(TransparentProxy, self).__init__(ctx) - self.resolver = platform.resolver() - - def __call__(self): - try: - self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) - except Exception as e: - raise ProtocolException("Transparent mode failure: %s" % repr(e), e) - - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index 709654cb..d5297cb1 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1,11 +1,9 @@ from __future__ import (absolute_import, print_function, division) -from .primitives import Log, Kill +from .server import ProxyServer, DummyServer from .config import ProxyConfig -from .connection import ClientConnection, ServerConnection __all__ = [ - "Log", "Kill", + "ProxyServer", "DummyServer", "ProxyConfig", - "ClientConnection", "ServerConnection" -] \ No newline at end of file +] diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index b360abbd..65029087 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -6,9 +6,9 @@ from OpenSSL import SSL from netlib import certutils, tcp from netlib.http import authentication +from netlib.tcp import Address, sslversion_choices from .. import utils, platform -from netlib.tcp import Address, sslversion_choices CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py deleted file mode 100644 index 94f318f6..00000000 --- a/libmproxy/proxy/connection.py +++ /dev/null @@ -1,193 +0,0 @@ -from __future__ import absolute_import - -import copy -import os - -from netlib import tcp, certutils -from .. import stateobject, utils - - -class ClientConnection(tcp.BaseHandler, stateobject.StateObject): - def __init__(self, client_connection, address, server): - # Eventually, this object is restored from state. We don't have a - # connection then. - if client_connection: - super(ClientConnection, self).__init__(client_connection, address, server) - else: - self.connection = None - self.server = None - self.wfile = None - self.rfile = None - self.address = None - self.clientcert = None - self.ssl_established = None - - self.timestamp_start = utils.timestamp() - self.timestamp_end = None - self.timestamp_ssl_setup = None - self.protocol = None - - def __nonzero__(self): - return bool(self.connection) and not self.finished - - def __repr__(self): - return "".format( - ssl="[ssl] " if self.ssl_established else "", - host=self.address.host, - port=self.address.port - ) - - @property - def tls_established(self): - return self.ssl_established - - _stateobject_attributes = dict( - ssl_established=bool, - timestamp_start=float, - timestamp_end=float, - timestamp_ssl_setup=float - ) - - def get_state(self, short=False): - d = super(ClientConnection, self).get_state(short) - d.update( - address={ - "address": self.address(), - "use_ipv6": self.address.use_ipv6}, - clientcert=self.cert.to_pem() if self.clientcert else None) - return d - - def load_state(self, state): - super(ClientConnection, self).load_state(state) - self.address = tcp.Address( - **state["address"]) if state["address"] else None - self.clientcert = certutils.SSLCert.from_pem( - state["clientcert"]) if state["clientcert"] else None - - def copy(self): - return copy.copy(self) - - def send(self, message): - if isinstance(message, list): - message = b''.join(message) - self.wfile.write(message) - self.wfile.flush() - - @classmethod - def from_state(cls, state): - f = cls(None, tuple(), None) - f.load_state(state) - return f - - def convert_to_ssl(self, *args, **kwargs): - super(ClientConnection, self).convert_to_ssl(*args, **kwargs) - self.timestamp_ssl_setup = utils.timestamp() - - def finish(self): - super(ClientConnection, self).finish() - self.timestamp_end = utils.timestamp() - - -class ServerConnection(tcp.TCPClient, stateobject.StateObject): - def __init__(self, address): - tcp.TCPClient.__init__(self, address) - - self.via = None - self.timestamp_start = None - self.timestamp_end = None - self.timestamp_tcp_setup = None - self.timestamp_ssl_setup = None - self.protocol = None - - def __nonzero__(self): - return bool(self.connection) and not self.finished - - def __repr__(self): - if self.ssl_established and self.sni: - ssl = "[ssl: {0}] ".format(self.sni) - elif self.ssl_established: - ssl = "[ssl] " - else: - ssl = "" - return "".format( - ssl=ssl, - host=self.address.host, - port=self.address.port - ) - - @property - def tls_established(self): - return self.ssl_established - - _stateobject_attributes = dict( - timestamp_start=float, - timestamp_end=float, - timestamp_tcp_setup=float, - timestamp_ssl_setup=float, - address=tcp.Address, - source_address=tcp.Address, - cert=certutils.SSLCert, - ssl_established=bool, - sni=str - ) - _stateobject_long_attributes = {"cert"} - - def get_state(self, short=False): - d = super(ServerConnection, self).get_state(short) - d.update( - address={"address": self.address(), - "use_ipv6": self.address.use_ipv6}, - source_address=({"address": self.source_address(), - "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), - cert=self.cert.to_pem() if self.cert else None - ) - return d - - def load_state(self, state): - super(ServerConnection, self).load_state(state) - - self.address = tcp.Address( - **state["address"]) if state["address"] else None - self.source_address = tcp.Address( - **state["source_address"]) if state["source_address"] else None - self.cert = certutils.SSLCert.from_pem( - state["cert"]) if state["cert"] else None - - @classmethod - def from_state(cls, state): - f = cls(tuple()) - f.load_state(state) - return f - - def copy(self): - return copy.copy(self) - - def connect(self): - self.timestamp_start = utils.timestamp() - tcp.TCPClient.connect(self) - self.timestamp_tcp_setup = utils.timestamp() - - def send(self, message): - if isinstance(message, list): - message = b''.join(message) - self.wfile.write(message) - self.wfile.flush() - - def establish_ssl(self, clientcerts, sni, **kwargs): - clientcert = None - if clientcerts: - path = os.path.join( - clientcerts, - self.address.host.encode("idna")) + ".pem" - if os.path.exists(path): - clientcert = path - - self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) - self.sni = sni - self.timestamp_ssl_setup = utils.timestamp() - - def finish(self): - tcp.TCPClient.finish(self) - self.timestamp_end = utils.timestamp() - -ServerConnection._stateobject_attributes["via"] = ServerConnection diff --git a/libmproxy/proxy/modes/__init__.py b/libmproxy/proxy/modes/__init__.py new file mode 100644 index 00000000..f014ed98 --- /dev/null +++ b/libmproxy/proxy/modes/__init__.py @@ -0,0 +1,12 @@ +from __future__ import (absolute_import, print_function, division) +from .http_proxy import HttpProxy, HttpUpstreamProxy +from .reverse_proxy import ReverseProxy +from .socks_proxy import Socks5Proxy +from .transparent_proxy import TransparentProxy + +__all__ = [ + "HttpProxy", "HttpUpstreamProxy", + "ReverseProxy", + "Socks5Proxy", + "TransparentProxy" +] diff --git a/libmproxy/proxy/modes/http_proxy.py b/libmproxy/proxy/modes/http_proxy.py new file mode 100644 index 00000000..90c54cc6 --- /dev/null +++ b/libmproxy/proxy/modes/http_proxy.py @@ -0,0 +1,26 @@ +from __future__ import (absolute_import, print_function, division) + +from ...protocol import Layer, ServerConnectionMixin + + +class HttpProxy(Layer, ServerConnectionMixin): + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() + + +class HttpUpstreamProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address): + super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) + + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/modes/reverse_proxy.py b/libmproxy/proxy/modes/reverse_proxy.py new file mode 100644 index 00000000..b57ac5eb --- /dev/null +++ b/libmproxy/proxy/modes/reverse_proxy.py @@ -0,0 +1,17 @@ +from __future__ import (absolute_import, print_function, division) + +from ...protocol import Layer, ServerConnectionMixin + + +class ReverseProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address, server_tls): + super(ReverseProxy, self).__init__(ctx, server_address=server_address) + self.server_tls = server_tls + + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/modes/socks_proxy.py b/libmproxy/proxy/modes/socks_proxy.py new file mode 100644 index 00000000..ebaf939e --- /dev/null +++ b/libmproxy/proxy/modes/socks_proxy.py @@ -0,0 +1,60 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib import socks +from netlib.tcp import NetLibError + +from ...exceptions import Socks5Exception +from ...protocol import Layer, ServerConnectionMixin + + +class Socks5Proxy(Layer, ServerConnectionMixin): + def __call__(self): + try: + # Parse Client Greeting + client_greet = socks.ClientGreeting.from_file(self.client_conn.rfile, fail_early=True) + client_greet.assert_socks5() + if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: + raise socks.SocksError( + socks.METHOD.NO_ACCEPTABLE_METHODS, + "mitmproxy only supports SOCKS without authentication" + ) + + # Send Server Greeting + server_greet = socks.ServerGreeting( + socks.VERSION.SOCKS5, + socks.METHOD.NO_AUTHENTICATION_REQUIRED + ) + server_greet.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + # Parse Connect Request + connect_request = socks.Message.from_file(self.client_conn.rfile) + connect_request.assert_socks5() + if connect_request.msg != socks.CMD.CONNECT: + raise socks.SocksError( + socks.REP.COMMAND_NOT_SUPPORTED, + "mitmproxy only supports SOCKS5 CONNECT." + ) + + # We always connect lazily, but we need to pretend to the client that we connected. + connect_reply = socks.Message( + socks.VERSION.SOCKS5, + socks.REP.SUCCEEDED, + connect_request.atyp, + # dummy value, we don't have an upstream connection yet. + connect_request.addr + ) + connect_reply.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + except (socks.SocksError, NetLibError) as e: + raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e) + + self.server_conn.address = connect_request.addr + + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/modes/transparent_proxy.py b/libmproxy/proxy/modes/transparent_proxy.py new file mode 100644 index 00000000..96ad86c4 --- /dev/null +++ b/libmproxy/proxy/modes/transparent_proxy.py @@ -0,0 +1,24 @@ +from __future__ import (absolute_import, print_function, division) + +from ... import platform +from ...exceptions import ProtocolException +from ...protocol import Layer, ServerConnectionMixin + + +class TransparentProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx): + super(TransparentProxy, self).__init__(ctx) + self.resolver = platform.resolver() + + def __call__(self): + try: + self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) + except Exception as e: + raise ProtocolException("Transparent mode failure: %s" % repr(e), e) + + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py deleted file mode 100644 index 2e440fe8..00000000 --- a/libmproxy/proxy/primitives.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import absolute_import -import collections -from netlib import socks, tcp - - -class Log(object): - def __init__(self, msg, level="info"): - self.msg = msg - self.level = level - - -class Kill(Exception): - """ - Kill a connection. - """ \ No newline at end of file diff --git a/libmproxy/proxy/root_context.py b/libmproxy/proxy/root_context.py new file mode 100644 index 00000000..35909612 --- /dev/null +++ b/libmproxy/proxy/root_context.py @@ -0,0 +1,93 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol + +from ..protocol import ( + RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin +) +from .modes import HttpProxy, HttpUpstreamProxy, ReverseProxy + + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel, .next_layer() and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def next_layer(self, top_layer): + """ + This function determines the next layer in the protocol stack. + + Arguments: + top_layer: the current top layer. + + Returns: + The next layer + """ + + # 1. Check for --ignore. + if self.config.check_ignore(top_layer.server_conn.address): + return RawTCPLayer(top_layer, logging=False) + + d = top_layer.client_conn.rfile.peek(3) + client_tls = is_tls_record_magic(d) + + # 2. Always insert a TLS layer, even if there's neither client nor server tls. + # An inline script may upgrade from http to https, + # in which case we need some form of TLS layer. + if isinstance(top_layer, ReverseProxy): + return TlsLayer(top_layer, client_tls, top_layer.server_tls) + if isinstance(top_layer, ServerConnectionMixin): + return TlsLayer(top_layer, client_tls, client_tls) + + # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. + if isinstance(top_layer, TlsLayer): + if isinstance(top_layer.ctx, HttpProxy): + return Http1Layer(top_layer, "regular") + if isinstance(top_layer.ctx, HttpUpstreamProxy): + return Http1Layer(top_layer, "upstream") + + # 4. Check for other TLS cases (e.g. after CONNECT). + if client_tls: + return TlsLayer(top_layer, True, True) + + # 4. Check for --tcp + if self.config.check_tcp(top_layer.server_conn.address): + return RawTCPLayer(top_layer) + + # 5. Check for TLS ALPN (HTTP1/HTTP2) + if isinstance(top_layer, TlsLayer): + alpn = top_layer.client_conn.get_alpn_proto_negotiated() + if alpn == HTTP2Protocol.ALPN_PROTO_H2: + return Http2Layer(top_layer, 'transparent') + if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: + return Http1Layer(top_layer, 'transparent') + + # 6. Assume HTTP1 by default + return Http1Layer(top_layer, 'transparent') + + # In a future version, we want to implement TCP passthrough as the last fallback, + # but we don't have the UI part ready for that. + # + # d = top_layer.client_conn.rfile.peek(3) + # is_ascii = ( + # len(d) == 3 and + # # better be safe here and don't expect uppercase... + # all(x in string.ascii_letters for x in d) + # ) + # # TODO: This could block if there are not enough bytes available? + # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) + # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) + + @property + def layers(self): + return [] + + def __repr__(self): + return "RootContext" diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 5abd0877..2a451ba1 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -3,14 +3,15 @@ from __future__ import absolute_import, print_function import traceback import sys import socket + from netlib import tcp from netlib.http.http1 import HTTP1Protocol from netlib.tcp import NetLibError - -from .. import protocol2 from ..exceptions import ProtocolException, ServerException -from .primitives import Log, Kill -from .connection import ClientConnection +from ..protocol import Log, Kill +from ..models import ClientConnection, make_error_response +from .modes import HttpUpstreamProxy, HttpProxy, ReverseProxy, TransparentProxy, Socks5Proxy +from .root_context import RootContext class DummyServer: @@ -71,7 +72,7 @@ class ConnectionHandler(object): """@type: libmproxy.controller.Channel""" def _create_root_layer(self): - root_context = protocol2.RootContext( + root_context = RootContext( self.client_conn, self.config, self.channel @@ -79,23 +80,23 @@ class ConnectionHandler(object): mode = self.config.mode if mode == "upstream": - return protocol2.HttpUpstreamProxy( + return HttpUpstreamProxy( root_context, self.config.upstream_server.address ) elif mode == "transparent": - return protocol2.TransparentProxy(root_context) + return TransparentProxy(root_context) elif mode == "reverse": server_tls = self.config.upstream_server.scheme == "https" - return protocol2.ReverseProxy( + return ReverseProxy( root_context, self.config.upstream_server.address, server_tls ) elif mode == "socks5": - return protocol2.Socks5Proxy(root_context) + return Socks5Proxy(root_context) elif mode == "regular": - return protocol2.HttpProxy(root_context) + return HttpProxy(root_context) elif callable(mode): # pragma: nocover return mode(root_context) else: # pragma: nocover @@ -116,7 +117,7 @@ class ConnectionHandler(object): # we send an HTTP error response, which is both # understandable by HTTP clients and humans. try: - error_response = protocol2.make_error_response(502, repr(e)) + error_response = make_error_response(502, repr(e)) self.client_conn.send(HTTP1Protocol().assemble(error_response)) except NetLibError: pass -- cgit v1.2.3 From 63ad4a4f5117d34ba6e9692eef1fc88f68b19c3d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 15:59:50 +0200 Subject: coverage++ --- libmproxy/models/__init__.py | 4 ++-- libmproxy/protocol/base.py | 4 ++-- libmproxy/protocol/http.py | 2 +- 3 files changed, 5 insertions(+), 5 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/models/__init__.py b/libmproxy/models/__init__.py index 3947847c..a54f305f 100644 --- a/libmproxy/models/__init__.py +++ b/libmproxy/models/__init__.py @@ -8,8 +8,8 @@ from .connections import ClientConnection, ServerConnection from .flow import Flow, Error __all__ = [ - "HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded" - "make_error_response", "make_connect_request", + "HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded", + "make_error_response", "make_connect_request", "make_connect_response", "ClientConnection", "ServerConnection", "Flow", "Error", diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index d22a71c6..1c9b356c 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -43,7 +43,7 @@ class _LayerCodeCompletion(object): Dummy class that provides type hinting in PyCharm, which simplifies development a lot. """ - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): # pragma: nocover super(_LayerCodeCompletion, self).__init__(*args, **kwargs) if True: return @@ -70,7 +70,7 @@ class Layer(_LayerCodeCompletion): Raises: ProtocolException in case of protocol exceptions. """ - raise NotImplementedError + raise NotImplementedError() def __getattr__(self, name): """ diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index fc57f6df..345b3aa8 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -40,7 +40,7 @@ class _StreamingHttpLayer(_HttpLayer): def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): raise NotImplementedError() - yield "this is a generator" + yield "this is a generator" # pragma: no cover def send_response_headers(self, response): raise NotImplementedError -- cgit v1.2.3 From 1e9aef5b1e3e1d60a2bb94d47be03b780c10a497 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 00:14:42 +0200 Subject: fix upstream proxy server change, update example --- libmproxy/protocol/base.py | 4 ++++ libmproxy/protocol/http.py | 25 ++++++++++++------------- libmproxy/protocol/tls.py | 4 +++- 3 files changed, 19 insertions(+), 14 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 1c9b356c..3440cb01 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -116,6 +116,10 @@ class ServerConnectionMixin(object): self._disconnect() self.log("Set new server address: " + repr(address), "debug") self.server_conn.address = address + if server_tls: + raise ProtocolException( + "Cannot upgrade to TLS, no TLS layer on the protocol stack." + ) else: self.ctx.set_server(address, server_tls, sni, depth - 1) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 345b3aa8..3b62c389 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -207,6 +207,9 @@ class ConnectServerConnection(object): def __getattr__(self, item): return getattr(self.via, item) + def __nonzero__(self): + return bool(self.via) + class UpstreamConnectLayer(Layer): def __init__(self, ctx, connect_request): @@ -221,19 +224,22 @@ class UpstreamConnectLayer(Layer): layer = self.ctx.next_layer(self) layer() + def _send_connect_request(self): + self.send_request(self.connect_request) + resp = self.read_response("CONNECT") + if resp.code != 200: + raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") + def connect(self): if not self.server_conn: self.ctx.connect() - self.send_request(self.connect_request) + self._send_connect_request() else: pass # swallow the message def reconnect(self): self.ctx.reconnect() - self.send_request(self.connect_request) - resp = self.read_response("CONNECT") - if resp.code != 200: - raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") + self._send_connect_request() def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: @@ -386,7 +392,7 @@ class HttpLayer(Layer): if self.supports_streaming: flow.response = self.read_response_headers() else: - flow.response = self.read_response() + flow.response = self.read_response(flow.request.method) try: get_response() @@ -473,13 +479,6 @@ class HttpLayer(Layer): # Establish connection is neccessary. if not self.server_conn: self.connect() - - # SetServer is not guaranteed to work with TLS: - # If there's not TlsLayer below which could catch the exception, - # TLS will not be established. - if tls and not self.server_conn.tls_established: - raise ProtocolException( - "Cannot upgrade to SSL, no TLS layer on the protocol stack.") else: if not self.server_conn: self.connect() diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index b85a6595..2646ec4f 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -152,10 +152,12 @@ class TlsLayer(Layer): self._establish_tls_with_server() def set_server(self, address, server_tls=None, sni=None, depth=1): - self.ctx.set_server(address, server_tls, sni, depth) if depth == 1 and server_tls is not None: + self.ctx.set_server(address, None, None, 1) self._sni_from_server_change = sni self._server_tls = server_tls + else: + self.ctx.set_server(address, server_tls, sni, depth) @property def sni_for_server_connection(self): -- cgit v1.2.3 From 7450bef615436d39bcd2a0d2a8892b8f42beea6f Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 13:43:30 +0200 Subject: fix dns_spoofing example, avoid connecting to itself --- libmproxy/models/connections.py | 2 +- libmproxy/models/flow.py | 2 +- libmproxy/protocol/base.py | 21 ++++++++++++++++++++- libmproxy/protocol/http_replay.py | 1 + libmproxy/proxy/config.py | 2 +- libmproxy/proxy/server.py | 2 +- 6 files changed, 25 insertions(+), 5 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/models/connections.py b/libmproxy/models/connections.py index 98bae3cc..f1e10de9 100644 --- a/libmproxy/models/connections.py +++ b/libmproxy/models/connections.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) import copy import os diff --git a/libmproxy/models/flow.py b/libmproxy/models/flow.py index 58287e5b..8eff18f4 100644 --- a/libmproxy/models/flow.py +++ b/libmproxy/models/flow.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) import copy import uuid diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 3440cb01..d1af547f 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -103,6 +103,7 @@ class ServerConnectionMixin(object): def __init__(self, server_address=None): super(ServerConnectionMixin, self).__init__() self.server_conn = ServerConnection(server_address) + self._check_self_connect() def reconnect(self): address = self.server_conn.address @@ -110,12 +111,30 @@ class ServerConnectionMixin(object): self.server_conn.address = address self.connect() + def _check_self_connect(self): + """ + We try to protect the proxy from _accidentally_ connecting to itself, + e.g. because of a failed transparent lookup or an invalid configuration. + """ + address = self.server_conn.address + if address: + self_connect = ( + address.port == self.config.port and + address.host in ("localhost", "127.0.0.1", "::1") + ) + if self_connect: + raise ProtocolException( + "Invalid server address: {}\r\n" + "The proxy shall not connect to itself.".format(repr(address)) + ) + def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: if self.server_conn: self._disconnect() self.log("Set new server address: " + repr(address), "debug") self.server_conn.address = address + self._check_self_connect() if server_tls: raise ProtocolException( "Cannot upgrade to TLS, no TLS layer on the protocol stack." @@ -141,7 +160,7 @@ class ServerConnectionMixin(object): self.server_conn.connect() except tcp.NetLibError as e: raise ProtocolException( - "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) + "Server connection to %s failed: %s" % (repr(self.server_conn.address), e), e) class Log(object): diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py index e0144c93..c37fd131 100644 --- a/libmproxy/protocol/http_replay.py +++ b/libmproxy/protocol/http_replay.py @@ -1,3 +1,4 @@ +from __future__ import (absolute_import, print_function, division) import threading from netlib.http import HttpError diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 65029087..8d2a286d 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) import collections import os import re diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 2a451ba1..b565ef86 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, print_function +from __future__ import (absolute_import, print_function, division) import traceback import sys -- cgit v1.2.3 From 41e6e538dfa758b7d9f867f85f62e881ae408684 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 13:49:47 +0200 Subject: fix layer initialization --- libmproxy/protocol/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index d1af547f..4eb843e4 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -61,8 +61,8 @@ class Layer(_LayerCodeCompletion): Args: ctx: The (read-only) higher layer. """ - super(Layer, self).__init__(*args, **kwargs) self.ctx = ctx + super(Layer, self).__init__(*args, **kwargs) def __call__(self): """ -- cgit v1.2.3 From b04e6e56ab1e69853abebfb950539e3a3aefbdf2 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 17:05:52 +0200 Subject: update inline script hooks --- libmproxy/flow.py | 4 ++++ libmproxy/protocol/base.py | 10 +++++++--- libmproxy/protocol/http.py | 6 +++--- libmproxy/protocol/http_replay.py | 4 ++-- libmproxy/proxy/server.py | 5 +++++ 5 files changed, 21 insertions(+), 8 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 00ec83d2..5eac8da9 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -961,6 +961,10 @@ class FlowMaster(controller.Master): self.run_script_hook("serverconnect", sc) sc.reply() + def handle_serverdisconnect(self, sc): + self.run_script_hook("serverdisconnect", sc) + sc.reply() + def handle_error(self, f): self.state.update_flow(f) self.run_script_hook("error", f) diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 4eb843e4..40ec0536 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -48,9 +48,11 @@ class _LayerCodeCompletion(object): if True: return self.config = None - """@type: libmproxy.proxy.config.ProxyConfig""" + """@type: libmproxy.proxy.ProxyConfig""" self.client_conn = None - """@type: libmproxy.proxy.connection.ClientConnection""" + """@type: libmproxy.models.ClientConnection""" + self.server_conn = None + """@type: libmproxy.models.ServerConnection""" self.channel = None """@type: libmproxy.controller.Channel""" @@ -62,6 +64,7 @@ class Layer(_LayerCodeCompletion): ctx: The (read-only) higher layer. """ self.ctx = ctx + """@type: libmproxy.protocol.Layer""" super(Layer, self).__init__(*args, **kwargs) def __call__(self): @@ -149,13 +152,14 @@ class ServerConnectionMixin(object): self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) self.server_conn.finish() self.server_conn.close() - # self.channel.tell("serverdisconnect", self) + self.channel.tell("serverdisconnect", self.server_conn) self.server_conn = ServerConnection(None) def connect(self): if not self.server_conn.address: raise ProtocolException("Cannot connect to server, no server address given.") self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + self.channel.ask("serverconnect", self.server_conn) try: self.server_conn.connect() except tcp.NetLibError as e: diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 3b62c389..f0f4ac24 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -418,7 +418,7 @@ class HttpLayer(Layer): # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True flow = self.channel.ask("responseheaders", flow) - if flow is None or flow == Kill: + if flow == Kill: raise Kill() if self.supports_streaming: @@ -442,7 +442,7 @@ class HttpLayer(Layer): [repr(flow.response)] ) response_reply = self.channel.ask("response", flow) - if response_reply is None or response_reply == Kill: + if response_reply == Kill: raise Kill() def process_request_hook(self, flow): @@ -462,7 +462,7 @@ class HttpLayer(Layer): flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" request_reply = self.channel.ask("request", flow) - if request_reply is None or request_reply == Kill: + if request_reply == Kill: raise Kill() if isinstance(request_reply, HTTPResponse): flow.response = request_reply diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py index c37fd131..2759a019 100644 --- a/libmproxy/protocol/http_replay.py +++ b/libmproxy/protocol/http_replay.py @@ -36,7 +36,7 @@ class RequestReplayThread(threading.Thread): # If we have a channel, run script hooks. if self.channel: request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == Kill: + if request_reply == Kill: raise Kill() elif isinstance(request_reply, HTTPResponse): self.flow.response = request_reply @@ -82,7 +82,7 @@ class RequestReplayThread(threading.Thread): ) if self.channel: response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == Kill: + if response_reply == Kill: raise Kill() except (HttpError, NetLibError) as v: self.flow.error = Error(repr(v)) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index b565ef86..e9e8df09 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -106,6 +106,10 @@ class ConnectionHandler(object): self.log("clientconnect", "info") root_layer = self._create_root_layer() + root_layer = self.channel.ask("clientconnect", root_layer) + if root_layer == Kill: + def root_layer(): + raise Kill() try: root_layer() @@ -128,6 +132,7 @@ class ConnectionHandler(object): print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) self.log("clientdisconnect", "info") + self.channel.tell("clientdisconnect", root_layer) self.client_conn.finish() def log(self, msg, level): -- cgit v1.2.3 From 481cc6ea842dc3c531c45a4bd228bdd6ebcc4229 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 17:29:14 +0200 Subject: we don't support socks auth, refs #738 --- libmproxy/proxy/config.py | 7 +++++++ 1 file changed, 7 insertions(+) (limited to 'libmproxy') diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 8d2a286d..2a1b84cb 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -136,6 +136,13 @@ def process_proxy_options(parser, options): ) if options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd: + + if options.socks_proxy: + return parser.error( + "Proxy Authentication not supported in SOCKS mode. " + "https://github.com/mitmproxy/mitmproxy/issues/738" + ) + if options.auth_singleuser: if len(options.auth_singleuser.split(':')) != 2: return parser.error( -- cgit v1.2.3 From c4d6b357262a5964a8d10ea20b92d22efc9c68a4 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 31 Aug 2015 22:20:12 +0200 Subject: do not log WindowUpdateFrame frames --- libmproxy/protocol/http.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index f0f4ac24..7f57d17c 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -7,6 +7,7 @@ from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol +from netlib.http.http2.frame import WindowUpdateFrame from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException @@ -187,8 +188,15 @@ class Http2Layer(_HttpLayer): layer = HttpLayer(self, self.mode) layer() - def handle_unexpected_frame(self, frm): - self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") + def handle_unexpected_frame(self, frame): + if isinstance(frame, WindowUpdateFrame): + # Clients are sending WindowUpdate frames depending on their flow control algorithm. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should keep track of our own flow control window and + # stall transmission if the outgoing flow control buffer is full. + return + self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") class ConnectServerConnection(object): -- cgit v1.2.3 From f1c8b47b1eb153d448061c0ddce21030c31af2b7 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 1 Sep 2015 19:24:36 +0200 Subject: better tls error messages, fix #672 --- libmproxy/protocol/tls.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index 2646ec4f..a8dc8bb2 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -259,9 +259,17 @@ class TlsLayer(Layer): (tls_cert_err['depth'], tls_cert_err['errno']), "error") self.log("Aborting connection attempt", "error") - raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + address=repr(self.server_conn.address), + sni=self.sni_for_server_connection, + e=repr(e), + ), e) except NetLibError as e: - raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + address=repr(self.server_conn.address), + sni=self.sni_for_server_connection, + e=repr(e), + ), e) self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") -- cgit v1.2.3