From 863113f989ee2a089c86b06a88a22e92d840348b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 24 Jul 2015 13:31:55 +0200 Subject: first initial proof-of-concept --- libmproxy/proxy/layer.py | 181 +++++++++++++++++++++++++++++++++++++++++++++ libmproxy/proxy/message.py | 40 ++++++++++ libmproxy/proxy/server.py | 69 +++++++++++++++-- 3 files changed, 282 insertions(+), 8 deletions(-) create mode 100644 libmproxy/proxy/layer.py create mode 100644 libmproxy/proxy/message.py diff --git a/libmproxy/proxy/layer.py b/libmproxy/proxy/layer.py new file mode 100644 index 00000000..500fb6ba --- /dev/null +++ b/libmproxy/proxy/layer.py @@ -0,0 +1,181 @@ +from __future__ import (absolute_import, print_function, division, unicode_literals) +from libmproxy.protocol.tcp import TCPHandler +from libmproxy.proxy.connection import ServerConnection +from netlib import tcp +from .primitives import Socks5ProxyMode, ProxyError, Log +from .message import Connect, Reconnect, ChangeServer + + +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible scenarios (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentModeLayer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + RegularModeLayer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentModeLayer + AutoLayer + SslLayer + AutoLayer + Http2Layer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can "yield" commands to lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" + + +class ProxyError2(Exception): + def __init__(self, message, cause=None): + super(ProxyError2, self).__init__(message) + self.cause = cause + + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def __getattr__(self, name): + """ + Accessing a nonexisting attribute does not throw an error but returns None instead. + """ + return None + + +class LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + def __init__(self): + if True: + return + self.config = None + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(LayerCodeCompletion): + def __init__(self, ctx): + """ + Args: + ctx: The (read-only) higher layer. + """ + super(Layer, self).__init__() + self.ctx = ctx + + def __call__(self): + """ + Logic of the layer. + Raises: + ProxyError2 in case of protocol exceptions. + """ + raise NotImplementedError + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "%s:%s: %s" % + (self.client_conn.address.host, + self.client_conn.address.port, + msg)] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + +class _ServerConnectionMixin(object): + def __init__(self): + self._server_address = None + self.server_conn = None + + def _handle_message(self, message): + if message == Reconnect: + self._disconnect() + self._connect() + return True + elif message == Connect: + self._connect() + return True + elif message == ChangeServer: + raise NotImplementedError + return False + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + self.server_conn.finish() + self.server_conn.close() + # self.channel.tell("serverdisconnect", self) + self.server_conn = None + + def _connect(self): + self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProxyError2("Server connection to '%s' failed: %s" % (self._server_address, repr(e)), e) + + def _set_address(self, address): + a = tcp.Address.wrap(address) + self.log("Set new server address: " + repr(a), "debug") + self.server_conn = ServerConnection(a) + + +class Socks5IncomingLayer(Layer, _ServerConnectionMixin): + def __call__(self): + try: + s5mode = Socks5ProxyMode(self.config.ssl_ports) + address = s5mode.get_upstream_server(self.client_conn)[2:] + except ProxyError as e: + raise ProxyError2(str(e), e) + + self._set_address(address) + + layer = TcpLayer(self) + for message in layer(): + if not self._handle_message(message): + yield message + + +class TcpLayer(Layer): + def __call__(self): + yield Connect() + tcp_handler = TCPHandler(self) + tcp_handler.handle_messages() + + def establish_server_connection(self): + print("establish server conn called") \ No newline at end of file diff --git a/libmproxy/proxy/message.py b/libmproxy/proxy/message.py new file mode 100644 index 00000000..a667123c --- /dev/null +++ b/libmproxy/proxy/message.py @@ -0,0 +1,40 @@ +""" +This module contains all valid messages layers can send to the underlying layers. +""" + + +class _Message(object): + def __eq__(self, other): + # Allow message == Connect checks. + # FIXME: make Connect == message work. + if isinstance(self, other): + return True + return self is other + + +class Connect(_Message): + """ + Connect to the server + """ + + +class Reconnect(_Message): + """ + Re-establish the server connection + """ + + +class ChangeServer(_Message): + """ + Change the upstream server. + """ + + def __init__(self, address, server_ssl, sni, depth=1): + self.address = address + self.server_ssl = server_ssl + self.sni = sni + + # upstream proxy scenario: you may want to change either the final target or the upstream proxy. + # We can express this neatly as the "nth-server-providing-layer" + # ServerConnection could get a `via` attribute. + self.depth = depth diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 2f6ee061..a45276d4 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -1,13 +1,14 @@ -from __future__ import absolute_import +from __future__ import absolute_import, print_function +import traceback +import sys import socket -from OpenSSL import SSL from netlib import tcp +from . import layer from .primitives import ProxyServerError, Log, ProxyError from .connection import ClientConnection, ServerConnection from ..protocol.handle import protocol_handler -from .. import version class DummyServer: @@ -46,7 +47,7 @@ class ProxyServer(tcp.TCPServer): self.channel = channel def handle_client_connection(self, conn, client_address): - h = ConnectionHandler( + h = ConnectionHandler2( self.config, conn, client_address, @@ -56,6 +57,57 @@ class ProxyServer(tcp.TCPServer): h.finish() +class ConnectionHandler2: + # FIXME: parameter ordering + # FIXME: remove server attribute + def __init__(self, config, client_conn, client_address, server, channel): + self.config = config + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = ClientConnection( + client_conn, + client_address, + server) + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = channel + """@type: libmproxy.controller.Channel""" + + def handle(self): + self.log("clientconnect", "info") + + root_context = layer.RootContext( + self.client_conn, + self.config, + self.channel + ) + root_layer = layer.Socks5IncomingLayer(root_context) + + try: + for message in root_layer(): + print("Root layer receveived: %s" % message) + except layer.ProxyError2 as e: + self.log(e, "info") + except Exception: + self.log(traceback.format_exc(), "error") + print(traceback.format_exc(), file=sys.stderr) + print("mitmproxy has crashed!", file=sys.stderr) + print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) + + def finish(self): + self.client_conn.finish() + + def log(self, msg, level, subs=()): + # FIXME: Duplicate code + full_msg = [ + "%s:%s: %s" % + (self.client_conn.address.host, + self.client_conn.address.port, + msg)] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + class ConnectionHandler: def __init__( self, @@ -74,6 +126,7 @@ class ConnectionHandler: self.server_conn = None """@type: libmproxy.proxy.connection.ServerConnection""" self.channel = channel + """@type: libmproxy.controller.Channel""" self.conntype = "http" @@ -144,9 +197,9 @@ class ConnectionHandler: import sys self.log(traceback.format_exc(), "error") - print >> sys.stderr, traceback.format_exc() - print >> sys.stderr, "mitmproxy has crashed!" - print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy" + print(traceback.format_exc(), file=sys.stderr) + print("mitmproxy has crashed!", file=sys.stderr) + print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) finally: # Make sure that we close the server connection in any case. # The client connection is closed by the ProxyServer and does not @@ -201,7 +254,7 @@ class ConnectionHandler: "serverconnect", "debug", [ "%s:%s" % self.server_conn.address()[ - :2]]) + :2]]) if ask: self.channel.ask("serverconnect", self) try: -- cgit v1.2.3 From be995ddbd62579621ed06ed7197c8f22939c16d1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 24 Jul 2015 17:48:27 +0200 Subject: add ssl layer --- libmproxy/proxy/layer.py | 255 ++++++++++++++++++++++++++++++++++++++++-- libmproxy/proxy/message.py | 4 +- libmproxy/proxy/primitives.py | 2 +- 3 files changed, 247 insertions(+), 14 deletions(-) diff --git a/libmproxy/proxy/layer.py b/libmproxy/proxy/layer.py index 500fb6ba..e4878bdf 100644 --- a/libmproxy/proxy/layer.py +++ b/libmproxy/proxy/layer.py @@ -1,11 +1,13 @@ from __future__ import (absolute_import, print_function, division, unicode_literals) +import Queue +import threading +import traceback from libmproxy.protocol.tcp import TCPHandler from libmproxy.proxy.connection import ServerConnection from netlib import tcp from .primitives import Socks5ProxyMode, ProxyError, Log from .message import Connect, Reconnect, ChangeServer - """ mitmproxy protocol architecture @@ -65,10 +67,11 @@ class RootContext(object): return None -class LayerCodeCompletion(object): +class _LayerCodeCompletion(object): """ Dummy class that provides type hinting in PyCharm, which simplifies development a lot. """ + def __init__(self): if True: return @@ -80,7 +83,7 @@ class LayerCodeCompletion(object): """@type: libmproxy.controller.Channel""" -class Layer(LayerCodeCompletion): +class Layer(_LayerCodeCompletion): def __init__(self, ctx): """ Args: @@ -116,11 +119,15 @@ class Layer(LayerCodeCompletion): class _ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + def __init__(self): - self._server_address = None + self.server_address = None self.server_conn = None - def _handle_message(self, message): + def _handle_server_message(self, message): if message == Reconnect: self._disconnect() self._connect() @@ -136,23 +143,24 @@ class _ServerConnectionMixin(object): """ Deletes (and closes) an existing server connection. """ - self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + self.log("serverdisconnect", "debug", [repr(self.server_address)]) self.server_conn.finish() self.server_conn.close() # self.channel.tell("serverdisconnect", self) self.server_conn = None def _connect(self): - self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + self.log("serverconnect", "debug", [repr(self.server_address)]) + self.server_conn = ServerConnection(self.server_address) try: self.server_conn.connect() except tcp.NetLibError as e: - raise ProxyError2("Server connection to '%s' failed: %s" % (self._server_address, repr(e)), e) + raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) def _set_address(self, address): a = tcp.Address.wrap(address) self.log("Set new server address: " + repr(a), "debug") - self.server_conn = ServerConnection(a) + self.server_address = address class Socks5IncomingLayer(Layer, _ServerConnectionMixin): @@ -161,13 +169,17 @@ class Socks5IncomingLayer(Layer, _ServerConnectionMixin): s5mode = Socks5ProxyMode(self.config.ssl_ports) address = s5mode.get_upstream_server(self.client_conn)[2:] except ProxyError as e: + # TODO: Unmonkeypatch raise ProxyError2(str(e), e) self._set_address(address) - layer = TcpLayer(self) + if address[1] == 443: + layer = SslLayer(self, True, True) + else: + layer = TcpLayer(self) for message in layer(): - if not self._handle_message(message): + if not self._handle_server_message(message): yield message @@ -178,4 +190,223 @@ class TcpLayer(Layer): tcp_handler.handle_messages() def establish_server_connection(self): - print("establish server conn called") \ No newline at end of file + pass + # FIXME: Remove method, currently just here to mock TCPHandler's call to it. + + +class ReconnectRequest(object): + def __init__(self): + self.done = threading.Event() + + +class SslLayer(Layer): + def __init__(self, ctx, client_ssl, server_ssl): + super(SslLayer, self).__init__(ctx) + self._client_ssl = client_ssl + self._server_ssl = server_ssl + self._connected = False + self._sni_from_handshake = None + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we resort to the following workaround when establishing SSL with the server: + 1. Try to establish SSL with the server without SNI. If this fails, we ignore it. + 2. Establish SSL with client. + - If there's a SNI callback, reconnect to the server with SNI. + - If not and the server connect failed, raise the original exception. + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + client_ssl_requires_server_cert = ( + self._client_ssl and self._server_ssl and not self.config.no_upstream_cert + ) + lazy_server_ssl = ( + self._server_ssl and not client_ssl_requires_server_cert + ) + + if client_ssl_requires_server_cert: + for m in self._establish_ssl_with_client_and_server(): + yield m + elif self.client_ssl: + self._establish_ssl_with_client() + + layer = TcpLayer(self) + for message in layer(): + if message != Connect or not self._connected: + yield message + if message == Connect: + if lazy_server_ssl: + self._establish_ssl_with_server() + if message == ChangeServer and message.depth == 1: + self.server_ssl = message.server_ssl + self._sni_from_server_change = message.sni + if message == Reconnect or message == ChangeServer: + if self.server_ssl: + self._establish_ssl_with_server() + + @property + def sni(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self._sni_from_handshake + + def _establish_ssl_with_client_and_server(self): + """ + This function deals with the problem that the server may require a SNI value from the client. + """ + + # First, try to connect to the server. + yield Connect() + self._connected = True + server_err = None + try: + self._establish_ssl_with_server() + except ProxyError2 as e: + server_err = e + + # The part below is a bit ugly as we cannot yield from the handle_sni callback. + # The workaround is to do that in a separate thread and yield from the main thread. + + # client_ssl_queue may contain the following elements + # - True, if ssl was successfully established + # - An Exception thrown by self._establish_ssl_with_client() + # - A threading.Event, which singnifies a request for a reconnect from the sni callback + self.__client_ssl_queue = Queue.Queue() + + def establish_client_ssl(): + try: + self._establish_ssl_with_client() + self.__client_ssl_queue.put(True) + except Exception as client_err: + self.__client_ssl_queue.put(client_err) + + threading.Thread(target=establish_client_ssl, name="ClientSSLThread").start() + e = self.__client_ssl_queue.get() + if isinstance(e, ReconnectRequest): + yield Reconnect() + self._establish_ssl_with_server() + e.done.set() + e = self.__client_ssl_queue.get() + if e is not True: + raise ProxyError2("Error when establish client SSL: " + repr(e), e) + self.__client_ssl_queue = None + + if server_err and not self._sni_from_handshake: + raise server_err + + def handle_sni(self, connection): + """ + This callback gets called during the SSL handshake with the client. + The client has just sent the Sever Name Indication (SNI). + """ + try: + sn = connection.get_servername() + if not sn: + return + sni = sn.decode("utf8").encode("idna") + + if sni != self.sni: + self._sni_from_handshake = sni + + # Perform reconnect + if self.server_ssl: + reconnect = ReconnectRequest() + self.__client_ssl_queue.put() + reconnect.done.wait() + + # Now, change client context to reflect changed certificate: + cert, key, chain_file = self.find_cert() + new_context = self.client_conn.create_ssl_context( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file + ) + connection.set_context(new_context) + # An unhandled exception in this method will core dump PyOpenSSL, so + # make dang sure it doesn't happen. + except: # pragma: no cover + self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") + + def _establish_ssl_with_client(self): + self.log("Establish SSL with client", "debug") + cert, key, chain_file = self.find_cert() + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + handle_sni=self.handle_sni, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file + ) + except tcp.NetLibError as e: + raise ProxyError2(repr(e), e) + + def _establish_ssl_with_server(self): + self.log("Establish SSL with server", "debug") + try: + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + ) + ssl_cert_err = self.server_conn.ssl_verification_error + if ssl_cert_err is not None: + self.log( + "SSL verification failed for upstream server at depth %s with error: %s" % + (ssl_cert_err['depth'], ssl_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except tcp.NetLibInvalidCertificateError as e: + ssl_cert_err = self.server_conn.ssl_verification_error + self.log( + "SSL verification failed for upstream server at depth %s with error: %s" % + (ssl_cert_err['depth'], ssl_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProxyError2(repr(e), e) + except Exception as e: + raise ProxyError2(repr(e), e) + + def find_cert(self): + host = self.server_conn.address.host + sans = [] + # Incorporate upstream certificate + if self.server_conn.ssl_established and (not self.config.no_upstream_cert): + upstream_cert = self.server_conn.cert + sans.extend(upstream_cert.altnames) + if upstream_cert.cn: + sans.append(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self._sni_from_handshake: + sans.append(self._sni_from_handshake) + if self._sni_from_server_change: + sans.append(self._sni_from_server_change) + + return self.config.certstore.get_cert(host, sans) diff --git a/libmproxy/proxy/message.py b/libmproxy/proxy/message.py index a667123c..7eb59344 100644 --- a/libmproxy/proxy/message.py +++ b/libmproxy/proxy/message.py @@ -6,11 +6,13 @@ This module contains all valid messages layers can send to the underlying layers class _Message(object): def __eq__(self, other): # Allow message == Connect checks. - # FIXME: make Connect == message work. if isinstance(self, other): return True return self is other + def __ne__(self, other): + return not self.__eq__(other) + class Connect(_Message): """ diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index 923f84ca..a9f31181 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -151,7 +151,7 @@ class Socks5ProxyMode(ProxyMode): return ssl, ssl, connect_request.addr.host, connect_request.addr.port except (socks.SocksError, tcp.NetLibError) as e: - raise ProxyError(502, "SOCKS5 mode failure: %s" % str(e)) + raise ProxyError(502, "SOCKS5 mode failure: %s" % repr(e)) class _ConstDestinationProxyMode(ProxyMode): -- cgit v1.2.3 From c1d016823c67fc834a2fdb6c77181d14b5fd8008 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 24 Jul 2015 18:29:13 +0200 Subject: move files around --- libmproxy/protocol2/__init__.py | 157 +++++++++++++++ libmproxy/protocol2/messages.py | 43 +++++ libmproxy/protocol2/rawtcp.py | 13 ++ libmproxy/protocol2/socks.py | 26 +++ libmproxy/protocol2/ssl.py | 228 ++++++++++++++++++++++ libmproxy/proxy/__init__.py | 2 +- libmproxy/proxy/layer.py | 412 ---------------------------------------- libmproxy/proxy/message.py | 42 ---- libmproxy/proxy/primitives.py | 6 + libmproxy/proxy/server.py | 14 +- 10 files changed, 481 insertions(+), 462 deletions(-) create mode 100644 libmproxy/protocol2/__init__.py create mode 100644 libmproxy/protocol2/messages.py create mode 100644 libmproxy/protocol2/rawtcp.py create mode 100644 libmproxy/protocol2/socks.py create mode 100644 libmproxy/protocol2/ssl.py delete mode 100644 libmproxy/proxy/layer.py delete mode 100644 libmproxy/proxy/message.py diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py new file mode 100644 index 00000000..9374a5bf --- /dev/null +++ b/libmproxy/protocol2/__init__.py @@ -0,0 +1,157 @@ +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible scenarios (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentModeLayer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + RegularModeLayer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentModeLayer + AutoLayer + SslLayer + AutoLayer + Http2Layer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can "yield" commands to lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" + +from __future__ import (absolute_import, print_function, division, unicode_literals) +from netlib import tcp +from ..proxy import ProxyError2, Log +from ..proxy.connection import ServerConnection +from .messages import * + + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def __getattr__(self, name): + """ + Accessing a nonexisting attribute does not throw an error but returns None instead. + """ + return None + + +class _LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + + def __init__(self): + if True: + return + self.config = None + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(_LayerCodeCompletion): + def __init__(self, ctx): + """ + Args: + ctx: The (read-only) higher layer. + """ + super(Layer, self).__init__() + self.ctx = ctx + + def __call__(self): + """ + Logic of the layer. + Raises: + ProxyError2 in case of protocol exceptions. + """ + raise NotImplementedError + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "%s:%s: %s" % + (self.client_conn.address.host, + self.client_conn.address.port, + msg)] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + +class ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + + def __init__(self): + self.server_address = None + self.server_conn = None + + def _handle_server_message(self, message): + if message == Reconnect: + self._disconnect() + self._connect() + return True + elif message == Connect: + self._connect() + return True + elif message == ChangeServer: + raise NotImplementedError + return False + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_address)]) + self.server_conn.finish() + self.server_conn.close() + # self.channel.tell("serverdisconnect", self) + self.server_conn = None + + def _connect(self): + self.log("serverconnect", "debug", [repr(self.server_address)]) + self.server_conn = ServerConnection(self.server_address) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) + + def _set_address(self, address): + a = tcp.Address.wrap(address) + self.log("Set new server address: " + repr(a), "debug") + self.server_address = address + + +from .socks import Socks5IncomingLayer +from .rawtcp import TcpLayer \ No newline at end of file diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py new file mode 100644 index 00000000..52bb5a44 --- /dev/null +++ b/libmproxy/protocol2/messages.py @@ -0,0 +1,43 @@ +""" +This module contains all valid messages layers can send to the underlying layers. +""" +from __future__ import (absolute_import, print_function, division, unicode_literals) + + +class _Message(object): + def __eq__(self, other): + # Allow message == Connect checks. + if isinstance(self, other): + return True + return self is other + + def __ne__(self, other): + return not self.__eq__(other) + + +class Connect(_Message): + """ + Connect to the server + """ + + +class Reconnect(_Message): + """ + Re-establish the server connection + """ + + +class ChangeServer(_Message): + """ + Change the upstream server. + """ + + def __init__(self, address, server_ssl, sni, depth=1): + self.address = address + self.server_ssl = server_ssl + self.sni = sni + + # upstream proxy scenario: you may want to change either the final target or the upstream proxy. + # We can express this neatly as the "nth-server-providing-layer" + # ServerConnection could get a `via` attribute. + self.depth = depth diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py new file mode 100644 index 00000000..db9a48fa --- /dev/null +++ b/libmproxy/protocol2/rawtcp.py @@ -0,0 +1,13 @@ +from . import Layer, Connect +from ..protocol.tcp import TCPHandler + + +class TcpLayer(Layer): + def __call__(self): + yield Connect() + tcp_handler = TCPHandler(self) + tcp_handler.handle_messages() + + def establish_server_connection(self): + pass + # FIXME: Remove method, currently just here to mock TCPHandler's call to it. diff --git a/libmproxy/protocol2/socks.py b/libmproxy/protocol2/socks.py new file mode 100644 index 00000000..90771015 --- /dev/null +++ b/libmproxy/protocol2/socks.py @@ -0,0 +1,26 @@ +from __future__ import (absolute_import, print_function, division, unicode_literals) + +from ..proxy import ProxyError, Socks5ProxyMode, ProxyError2 +from . import Layer, ServerConnectionMixin +from .rawtcp import TcpLayer +from .ssl import SslLayer + + +class Socks5IncomingLayer(Layer, ServerConnectionMixin): + def __call__(self): + try: + s5mode = Socks5ProxyMode(self.config.ssl_ports) + address = s5mode.get_upstream_server(self.client_conn)[2:] + except ProxyError as e: + # TODO: Unmonkeypatch + raise ProxyError2(str(e), e) + + self._set_address(address) + + if address[1] == 443: + layer = SslLayer(self, True, True) + else: + layer = TcpLayer(self) + for message in layer(): + if not self._handle_server_message(message): + yield message diff --git a/libmproxy/protocol2/ssl.py b/libmproxy/protocol2/ssl.py new file mode 100644 index 00000000..6b44bf42 --- /dev/null +++ b/libmproxy/protocol2/ssl.py @@ -0,0 +1,228 @@ +from __future__ import (absolute_import, print_function, division, unicode_literals) +import Queue +import threading +import traceback +from netlib import tcp + +from ..proxy import ProxyError2 +from . import Layer +from .messages import Connect, Reconnect, ChangeServer +from .rawtcp import TcpLayer + + +class ReconnectRequest(object): + def __init__(self): + self.done = threading.Event() + + +class SslLayer(Layer): + def __init__(self, ctx, client_ssl, server_ssl): + super(SslLayer, self).__init__(ctx) + self._client_ssl = client_ssl + self._server_ssl = server_ssl + self._connected = False + self._sni_from_handshake = None + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we resort to the following workaround when establishing SSL with the server: + 1. Try to establish SSL with the server without SNI. If this fails, we ignore it. + 2. Establish SSL with client. + - If there's a SNI callback, reconnect to the server with SNI. + - If not and the server connect failed, raise the original exception. + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + client_ssl_requires_server_cert = ( + self._client_ssl and self._server_ssl and not self.config.no_upstream_cert + ) + lazy_server_ssl = ( + self._server_ssl and not client_ssl_requires_server_cert + ) + + if client_ssl_requires_server_cert: + for m in self._establish_ssl_with_client_and_server(): + yield m + elif self.client_ssl: + self._establish_ssl_with_client() + + layer = TcpLayer(self) + for message in layer(): + if message != Connect or not self._connected: + yield message + if message == Connect: + if lazy_server_ssl: + self._establish_ssl_with_server() + if message == ChangeServer and message.depth == 1: + self.server_ssl = message.server_ssl + self._sni_from_server_change = message.sni + if message == Reconnect or message == ChangeServer: + if self.server_ssl: + self._establish_ssl_with_server() + + @property + def sni(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self._sni_from_handshake + + def _establish_ssl_with_client_and_server(self): + """ + This function deals with the problem that the server may require a SNI value from the client. + """ + + # First, try to connect to the server. + yield Connect() + self._connected = True + server_err = None + try: + self._establish_ssl_with_server() + except ProxyError2 as e: + server_err = e + + # The part below is a bit ugly as we cannot yield from the handle_sni callback. + # The workaround is to do that in a separate thread and yield from the main thread. + + # client_ssl_queue may contain the following elements + # - True, if ssl was successfully established + # - An Exception thrown by self._establish_ssl_with_client() + # - A threading.Event, which singnifies a request for a reconnect from the sni callback + self.__client_ssl_queue = Queue.Queue() + + def establish_client_ssl(): + try: + self._establish_ssl_with_client() + self.__client_ssl_queue.put(True) + except Exception as client_err: + self.__client_ssl_queue.put(client_err) + + threading.Thread(target=establish_client_ssl, name="ClientSSLThread").start() + e = self.__client_ssl_queue.get() + if isinstance(e, ReconnectRequest): + yield Reconnect() + self._establish_ssl_with_server() + e.done.set() + e = self.__client_ssl_queue.get() + if e is not True: + raise ProxyError2("Error when establish client SSL: " + repr(e), e) + self.__client_ssl_queue = None + + if server_err and not self._sni_from_handshake: + raise server_err + + def handle_sni(self, connection): + """ + This callback gets called during the SSL handshake with the client. + The client has just sent the Sever Name Indication (SNI). + """ + try: + sn = connection.get_servername() + if not sn: + return + sni = sn.decode("utf8").encode("idna") + + if sni != self.sni: + self._sni_from_handshake = sni + + # Perform reconnect + if self.server_ssl: + reconnect = ReconnectRequest() + self.__client_ssl_queue.put() + reconnect.done.wait() + + # Now, change client context to reflect changed certificate: + cert, key, chain_file = self.find_cert() + new_context = self.client_conn.create_ssl_context( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file + ) + connection.set_context(new_context) + # An unhandled exception in this method will core dump PyOpenSSL, so + # make dang sure it doesn't happen. + except: # pragma: no cover + self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") + + def _establish_ssl_with_client(self): + self.log("Establish SSL with client", "debug") + cert, key, chain_file = self.find_cert() + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + handle_sni=self.handle_sni, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file + ) + except tcp.NetLibError as e: + raise ProxyError2(repr(e), e) + + def _establish_ssl_with_server(self): + self.log("Establish SSL with server", "debug") + try: + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + ) + ssl_cert_err = self.server_conn.ssl_verification_error + if ssl_cert_err is not None: + self.log( + "SSL verification failed for upstream server at depth %s with error: %s" % + (ssl_cert_err['depth'], ssl_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except tcp.NetLibInvalidCertificateError as e: + ssl_cert_err = self.server_conn.ssl_verification_error + self.log( + "SSL verification failed for upstream server at depth %s with error: %s" % + (ssl_cert_err['depth'], ssl_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProxyError2(repr(e), e) + except Exception as e: + raise ProxyError2(repr(e), e) + + def find_cert(self): + host = self.server_conn.address.host + sans = [] + # Incorporate upstream certificate + if self.server_conn.ssl_established and (not self.config.no_upstream_cert): + upstream_cert = self.server_conn.cert + sans.extend(upstream_cert.altnames) + if upstream_cert.cn: + sans.append(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self._sni_from_handshake: + sans.append(self._sni_from_handshake) + if self._sni_from_server_change: + sans.append(self._sni_from_server_change) + + return self.config.certstore.get_cert(host, sans) diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index f33d323b..7d664707 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1,2 +1,2 @@ from .primitives import * -from .config import ProxyConfig, process_proxy_options +from .config import ProxyConfig, process_proxy_options \ No newline at end of file diff --git a/libmproxy/proxy/layer.py b/libmproxy/proxy/layer.py deleted file mode 100644 index e4878bdf..00000000 --- a/libmproxy/proxy/layer.py +++ /dev/null @@ -1,412 +0,0 @@ -from __future__ import (absolute_import, print_function, division, unicode_literals) -import Queue -import threading -import traceback -from libmproxy.protocol.tcp import TCPHandler -from libmproxy.proxy.connection import ServerConnection -from netlib import tcp -from .primitives import Socks5ProxyMode, ProxyError, Log -from .message import Connect, Reconnect, ChangeServer - -""" -mitmproxy protocol architecture - -In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. -For example, the following scenarios depict possible scenarios (lowest layer first): - -Transparent HTTP proxy, no SSL: - TransparentModeLayer - HttpLayer - -Regular proxy, CONNECT request with WebSockets over SSL: - RegularModeLayer - HttpLayer - SslLayer - WebsocketLayer (or TcpLayer) - -Automated protocol detection by peeking into the buffer: - TransparentModeLayer - AutoLayer - SslLayer - AutoLayer - Http2Layer - -Communication between layers is done as follows: - - lower layers provide context information to higher layers - - higher layers can "yield" commands to lower layers, - which are propagated until they reach a suitable layer. - -Further goals: - - Connections should always be peekable to make automatic protocol detection work. - - Upstream connections should be established as late as possible; - inline scripts shall have a chance to handle everything locally. -""" - - -class ProxyError2(Exception): - def __init__(self, message, cause=None): - super(ProxyError2, self).__init__(message) - self.cause = cause - - -class RootContext(object): - """ - The outmost context provided to the root layer. - As a consequence, every layer has .client_conn, .channel and .config. - """ - - def __init__(self, client_conn, config, channel): - self.client_conn = client_conn # Client Connection - self.channel = channel # provides .ask() method to communicate with FlowMaster - self.config = config # Proxy Configuration - - def __getattr__(self, name): - """ - Accessing a nonexisting attribute does not throw an error but returns None instead. - """ - return None - - -class _LayerCodeCompletion(object): - """ - Dummy class that provides type hinting in PyCharm, which simplifies development a lot. - """ - - def __init__(self): - if True: - return - self.config = None - """@type: libmproxy.proxy.config.ProxyConfig""" - self.client_conn = None - """@type: libmproxy.proxy.connection.ClientConnection""" - self.channel = None - """@type: libmproxy.controller.Channel""" - - -class Layer(_LayerCodeCompletion): - def __init__(self, ctx): - """ - Args: - ctx: The (read-only) higher layer. - """ - super(Layer, self).__init__() - self.ctx = ctx - - def __call__(self): - """ - Logic of the layer. - Raises: - ProxyError2 in case of protocol exceptions. - """ - raise NotImplementedError - - def __getattr__(self, name): - """ - Attributes not present on the current layer may exist on a higher layer. - """ - return getattr(self.ctx, name) - - def log(self, msg, level, subs=()): - full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - -class _ServerConnectionMixin(object): - """ - Mixin that provides a layer with the capabilities to manage a server connection. - """ - - def __init__(self): - self.server_address = None - self.server_conn = None - - def _handle_server_message(self, message): - if message == Reconnect: - self._disconnect() - self._connect() - return True - elif message == Connect: - self._connect() - return True - elif message == ChangeServer: - raise NotImplementedError - return False - - def _disconnect(self): - """ - Deletes (and closes) an existing server connection. - """ - self.log("serverdisconnect", "debug", [repr(self.server_address)]) - self.server_conn.finish() - self.server_conn.close() - # self.channel.tell("serverdisconnect", self) - self.server_conn = None - - def _connect(self): - self.log("serverconnect", "debug", [repr(self.server_address)]) - self.server_conn = ServerConnection(self.server_address) - try: - self.server_conn.connect() - except tcp.NetLibError as e: - raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) - - def _set_address(self, address): - a = tcp.Address.wrap(address) - self.log("Set new server address: " + repr(a), "debug") - self.server_address = address - - -class Socks5IncomingLayer(Layer, _ServerConnectionMixin): - def __call__(self): - try: - s5mode = Socks5ProxyMode(self.config.ssl_ports) - address = s5mode.get_upstream_server(self.client_conn)[2:] - except ProxyError as e: - # TODO: Unmonkeypatch - raise ProxyError2(str(e), e) - - self._set_address(address) - - if address[1] == 443: - layer = SslLayer(self, True, True) - else: - layer = TcpLayer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message - - -class TcpLayer(Layer): - def __call__(self): - yield Connect() - tcp_handler = TCPHandler(self) - tcp_handler.handle_messages() - - def establish_server_connection(self): - pass - # FIXME: Remove method, currently just here to mock TCPHandler's call to it. - - -class ReconnectRequest(object): - def __init__(self): - self.done = threading.Event() - - -class SslLayer(Layer): - def __init__(self, ctx, client_ssl, server_ssl): - super(SslLayer, self).__init__(ctx) - self._client_ssl = client_ssl - self._server_ssl = server_ssl - self._connected = False - self._sni_from_handshake = None - self._sni_from_server_change = None - - def __call__(self): - """ - The strategy for establishing SSL is as follows: - First, we determine whether we need the server cert to establish ssl with the client. - If so, we first connect to the server and then to the client. - If not, we only connect to the client and do the server_ssl lazily on a Connect message. - - An additional complexity is that establish ssl with the server may require a SNI value from the client. - In an ideal world, we'd do the following: - 1. Start the SSL handshake with the client - 2. Check if the client sends a SNI. - 3. Pause the client handshake, establish SSL with the server. - 4. Finish the client handshake with the certificate from the server. - There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( - Thus, we resort to the following workaround when establishing SSL with the server: - 1. Try to establish SSL with the server without SNI. If this fails, we ignore it. - 2. Establish SSL with client. - - If there's a SNI callback, reconnect to the server with SNI. - - If not and the server connect failed, raise the original exception. - Further notes: - - OpenSSL 1.0.2 introduces a callback that would help here: - https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html - - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 - """ - client_ssl_requires_server_cert = ( - self._client_ssl and self._server_ssl and not self.config.no_upstream_cert - ) - lazy_server_ssl = ( - self._server_ssl and not client_ssl_requires_server_cert - ) - - if client_ssl_requires_server_cert: - for m in self._establish_ssl_with_client_and_server(): - yield m - elif self.client_ssl: - self._establish_ssl_with_client() - - layer = TcpLayer(self) - for message in layer(): - if message != Connect or not self._connected: - yield message - if message == Connect: - if lazy_server_ssl: - self._establish_ssl_with_server() - if message == ChangeServer and message.depth == 1: - self.server_ssl = message.server_ssl - self._sni_from_server_change = message.sni - if message == Reconnect or message == ChangeServer: - if self.server_ssl: - self._establish_ssl_with_server() - - @property - def sni(self): - if self._sni_from_server_change is False: - return None - else: - return self._sni_from_server_change or self._sni_from_handshake - - def _establish_ssl_with_client_and_server(self): - """ - This function deals with the problem that the server may require a SNI value from the client. - """ - - # First, try to connect to the server. - yield Connect() - self._connected = True - server_err = None - try: - self._establish_ssl_with_server() - except ProxyError2 as e: - server_err = e - - # The part below is a bit ugly as we cannot yield from the handle_sni callback. - # The workaround is to do that in a separate thread and yield from the main thread. - - # client_ssl_queue may contain the following elements - # - True, if ssl was successfully established - # - An Exception thrown by self._establish_ssl_with_client() - # - A threading.Event, which singnifies a request for a reconnect from the sni callback - self.__client_ssl_queue = Queue.Queue() - - def establish_client_ssl(): - try: - self._establish_ssl_with_client() - self.__client_ssl_queue.put(True) - except Exception as client_err: - self.__client_ssl_queue.put(client_err) - - threading.Thread(target=establish_client_ssl, name="ClientSSLThread").start() - e = self.__client_ssl_queue.get() - if isinstance(e, ReconnectRequest): - yield Reconnect() - self._establish_ssl_with_server() - e.done.set() - e = self.__client_ssl_queue.get() - if e is not True: - raise ProxyError2("Error when establish client SSL: " + repr(e), e) - self.__client_ssl_queue = None - - if server_err and not self._sni_from_handshake: - raise server_err - - def handle_sni(self, connection): - """ - This callback gets called during the SSL handshake with the client. - The client has just sent the Sever Name Indication (SNI). - """ - try: - sn = connection.get_servername() - if not sn: - return - sni = sn.decode("utf8").encode("idna") - - if sni != self.sni: - self._sni_from_handshake = sni - - # Perform reconnect - if self.server_ssl: - reconnect = ReconnectRequest() - self.__client_ssl_queue.put() - reconnect.done.wait() - - # Now, change client context to reflect changed certificate: - cert, key, chain_file = self.find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - connection.set_context(new_context) - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except: # pragma: no cover - self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") - - def _establish_ssl_with_client(self): - self.log("Establish SSL with client", "debug") - cert, key, chain_file = self.find_cert() - try: - self.client_conn.convert_to_ssl( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - handle_sni=self.handle_sni, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - except tcp.NetLibError as e: - raise ProxyError2(repr(e), e) - - def _establish_ssl_with_server(self): - self.log("Establish SSL with server", "debug") - try: - self.server_conn.establish_ssl( - self.config.clientcerts, - self.sni, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify_options=self.config.openssl_verification_mode_server, - ca_path=self.config.openssl_trusted_cadir_server, - ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, - ) - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") - except tcp.NetLibInvalidCertificateError as e: - ssl_cert_err = self.server_conn.ssl_verification_error - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") - raise ProxyError2(repr(e), e) - except Exception as e: - raise ProxyError2(repr(e), e) - - def find_cert(self): - host = self.server_conn.address.host - sans = [] - # Incorporate upstream certificate - if self.server_conn.ssl_established and (not self.config.no_upstream_cert): - upstream_cert = self.server_conn.cert - sans.extend(upstream_cert.altnames) - if upstream_cert.cn: - sans.append(host) - host = upstream_cert.cn.decode("utf8").encode("idna") - # Also add SNI values. - if self._sni_from_handshake: - sans.append(self._sni_from_handshake) - if self._sni_from_server_change: - sans.append(self._sni_from_server_change) - - return self.config.certstore.get_cert(host, sans) diff --git a/libmproxy/proxy/message.py b/libmproxy/proxy/message.py deleted file mode 100644 index 7eb59344..00000000 --- a/libmproxy/proxy/message.py +++ /dev/null @@ -1,42 +0,0 @@ -""" -This module contains all valid messages layers can send to the underlying layers. -""" - - -class _Message(object): - def __eq__(self, other): - # Allow message == Connect checks. - if isinstance(self, other): - return True - return self is other - - def __ne__(self, other): - return not self.__eq__(other) - - -class Connect(_Message): - """ - Connect to the server - """ - - -class Reconnect(_Message): - """ - Re-establish the server connection - """ - - -class ChangeServer(_Message): - """ - Change the upstream server. - """ - - def __init__(self, address, server_ssl, sni, depth=1): - self.address = address - self.server_ssl = server_ssl - self.sni = sni - - # upstream proxy scenario: you may want to change either the final target or the upstream proxy. - # We can express this neatly as the "nth-server-providing-layer" - # ServerConnection could get a `via` attribute. - self.depth = depth diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index a9f31181..fd4eb882 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -2,6 +2,12 @@ from __future__ import absolute_import from netlib import socks, tcp +class ProxyError2(Exception): + def __init__(self, message, cause=None): + super(ProxyError2, self).__init__(message) + self.cause = cause + + class ProxyError(Exception): def __init__(self, code, message, headers=None): super(ProxyError, self).__init__(message) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index a45276d4..c8990a9a 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -3,12 +3,12 @@ from __future__ import absolute_import, print_function import traceback import sys import socket - from netlib import tcp -from . import layer -from .primitives import ProxyServerError, Log, ProxyError -from .connection import ClientConnection, ServerConnection + from ..protocol.handle import protocol_handler +from .. import protocol2 +from .primitives import ProxyServerError, Log, ProxyError, ProxyError2 +from .connection import ClientConnection, ServerConnection class DummyServer: @@ -74,17 +74,17 @@ class ConnectionHandler2: def handle(self): self.log("clientconnect", "info") - root_context = layer.RootContext( + root_context = protocol2.RootContext( self.client_conn, self.config, self.channel ) - root_layer = layer.Socks5IncomingLayer(root_context) + root_layer = protocol2.Socks5IncomingLayer(root_context) try: for message in root_layer(): print("Root layer receveived: %s" % message) - except layer.ProxyError2 as e: + except ProxyError2 as e: self.log(e, "info") except Exception: self.log(traceback.format_exc(), "error") -- cgit v1.2.3 From e815915b22ef266ac4122027a10c59d9e036d0b4 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 25 Jul 2015 13:31:55 +0200 Subject: add auto layer, multiple other fixes --- libmproxy/protocol2/__init__.py | 159 +--------------------------------------- libmproxy/protocol2/auto.py | 16 ++++ libmproxy/protocol2/layer.py | 151 ++++++++++++++++++++++++++++++++++++++ libmproxy/protocol2/rawtcp.py | 4 +- libmproxy/protocol2/socks.py | 10 +-- libmproxy/protocol2/ssl.py | 8 +- 6 files changed, 183 insertions(+), 165 deletions(-) create mode 100644 libmproxy/protocol2/auto.py create mode 100644 libmproxy/protocol2/layer.py diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index 9374a5bf..6f4bfe44 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -1,157 +1,6 @@ -""" -mitmproxy protocol architecture - -In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. -For example, the following scenarios depict possible scenarios (lowest layer first): - -Transparent HTTP proxy, no SSL: - TransparentModeLayer - HttpLayer - -Regular proxy, CONNECT request with WebSockets over SSL: - RegularModeLayer - HttpLayer - SslLayer - WebsocketLayer (or TcpLayer) - -Automated protocol detection by peeking into the buffer: - TransparentModeLayer - AutoLayer - SslLayer - AutoLayer - Http2Layer - -Communication between layers is done as follows: - - lower layers provide context information to higher layers - - higher layers can "yield" commands to lower layers, - which are propagated until they reach a suitable layer. - -Further goals: - - Connections should always be peekable to make automatic protocol detection work. - - Upstream connections should be established as late as possible; - inline scripts shall have a chance to handle everything locally. -""" - from __future__ import (absolute_import, print_function, division, unicode_literals) -from netlib import tcp -from ..proxy import ProxyError2, Log -from ..proxy.connection import ServerConnection -from .messages import * - - -class RootContext(object): - """ - The outmost context provided to the root layer. - As a consequence, every layer has .client_conn, .channel and .config. - """ - - def __init__(self, client_conn, config, channel): - self.client_conn = client_conn # Client Connection - self.channel = channel # provides .ask() method to communicate with FlowMaster - self.config = config # Proxy Configuration - - def __getattr__(self, name): - """ - Accessing a nonexisting attribute does not throw an error but returns None instead. - """ - return None - - -class _LayerCodeCompletion(object): - """ - Dummy class that provides type hinting in PyCharm, which simplifies development a lot. - """ - - def __init__(self): - if True: - return - self.config = None - """@type: libmproxy.proxy.config.ProxyConfig""" - self.client_conn = None - """@type: libmproxy.proxy.connection.ClientConnection""" - self.channel = None - """@type: libmproxy.controller.Channel""" - - -class Layer(_LayerCodeCompletion): - def __init__(self, ctx): - """ - Args: - ctx: The (read-only) higher layer. - """ - super(Layer, self).__init__() - self.ctx = ctx - - def __call__(self): - """ - Logic of the layer. - Raises: - ProxyError2 in case of protocol exceptions. - """ - raise NotImplementedError - - def __getattr__(self, name): - """ - Attributes not present on the current layer may exist on a higher layer. - """ - return getattr(self.ctx, name) - - def log(self, msg, level, subs=()): - full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - -class ServerConnectionMixin(object): - """ - Mixin that provides a layer with the capabilities to manage a server connection. - """ - - def __init__(self): - self.server_address = None - self.server_conn = None - - def _handle_server_message(self, message): - if message == Reconnect: - self._disconnect() - self._connect() - return True - elif message == Connect: - self._connect() - return True - elif message == ChangeServer: - raise NotImplementedError - return False - - def _disconnect(self): - """ - Deletes (and closes) an existing server connection. - """ - self.log("serverdisconnect", "debug", [repr(self.server_address)]) - self.server_conn.finish() - self.server_conn.close() - # self.channel.tell("serverdisconnect", self) - self.server_conn = None - - def _connect(self): - self.log("serverconnect", "debug", [repr(self.server_address)]) - self.server_conn = ServerConnection(self.server_address) - try: - self.server_conn.connect() - except tcp.NetLibError as e: - raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) - - def _set_address(self, address): - a = tcp.Address.wrap(address) - self.log("Set new server address: " + repr(a), "debug") - self.server_address = address - - +from .layer import RootContext from .socks import Socks5IncomingLayer -from .rawtcp import TcpLayer \ No newline at end of file +from .rawtcp import TcpLayer +from .auto import AutoLayer +__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext"] \ No newline at end of file diff --git a/libmproxy/protocol2/auto.py b/libmproxy/protocol2/auto.py new file mode 100644 index 00000000..1c4293ac --- /dev/null +++ b/libmproxy/protocol2/auto.py @@ -0,0 +1,16 @@ +from __future__ import (absolute_import, print_function, division, unicode_literals) +from .layer import Layer + + +class AutoLayer(Layer): + def __call__(self): + d = self.client_conn.rfile.peek(1) + if d[0] == "\x16": + layer = SslLayer(self, True, True) + else: + layer = TcpLayer(self) + for m in layer(): + yield m + +from .rawtcp import TcpLayer +from .ssl import SslLayer diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py new file mode 100644 index 00000000..14263f64 --- /dev/null +++ b/libmproxy/protocol2/layer.py @@ -0,0 +1,151 @@ +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible scenarios (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentModeLayer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + RegularModeLayer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentModeLayer + AutoLayer + SslLayer + AutoLayer + Http2Layer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can "yield" commands to lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" +from __future__ import (absolute_import, print_function, division, unicode_literals) +from netlib import tcp +from ..proxy import ProxyError2, Log +from ..proxy.connection import ServerConnection +from .messages import Connect, Reconnect, ChangeServer + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def __getattr__(self, name): + """ + Accessing a nonexisting attribute does not throw an error but returns None instead. + """ + return None + + +class _LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + + def __init__(self): + if True: + return + self.config = None + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(_LayerCodeCompletion): + def __init__(self, ctx): + """ + Args: + ctx: The (read-only) higher layer. + """ + super(Layer, self).__init__() + self.ctx = ctx + + def __call__(self): + """ + Logic of the layer. + Raises: + ProxyError2 in case of protocol exceptions. + """ + raise NotImplementedError + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "%s:%s: %s" % + (self.client_conn.address.host, + self.client_conn.address.port, + msg)] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + +class ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + + def __init__(self): + self.server_address = None + self.server_conn = None + + def _handle_server_message(self, message): + if message == Reconnect: + self._disconnect() + self._connect() + return True + elif message == Connect: + self._connect() + return True + elif message == ChangeServer: + raise NotImplementedError + return False + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_address)]) + self.server_conn.finish() + self.server_conn.close() + # self.channel.tell("serverdisconnect", self) + self.server_conn = None + + def _connect(self): + self.log("serverconnect", "debug", [repr(self.server_address)]) + self.server_conn = ServerConnection(self.server_address) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) + + def _set_address(self, address): + a = tcp.Address.wrap(address) + self.log("Set new server address: " + repr(a), "debug") + self.server_address = address \ No newline at end of file diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index db9a48fa..b40c569f 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -1,5 +1,7 @@ -from . import Layer, Connect +from __future__ import (absolute_import, print_function, division, unicode_literals) from ..protocol.tcp import TCPHandler +from .layer import Layer +from .messages import Connect class TcpLayer(Layer): diff --git a/libmproxy/protocol2/socks.py b/libmproxy/protocol2/socks.py index 90771015..9ca30bb4 100644 --- a/libmproxy/protocol2/socks.py +++ b/libmproxy/protocol2/socks.py @@ -1,9 +1,7 @@ from __future__ import (absolute_import, print_function, division, unicode_literals) from ..proxy import ProxyError, Socks5ProxyMode, ProxyError2 -from . import Layer, ServerConnectionMixin -from .rawtcp import TcpLayer -from .ssl import SslLayer +from .layer import Layer, ServerConnectionMixin class Socks5IncomingLayer(Layer, ServerConnectionMixin): @@ -18,9 +16,11 @@ class Socks5IncomingLayer(Layer, ServerConnectionMixin): self._set_address(address) if address[1] == 443: - layer = SslLayer(self, True, True) + layer = AutoLayer(self) else: - layer = TcpLayer(self) + layer = AutoLayer(self) for message in layer(): if not self._handle_server_message(message): yield message + +from .auto import AutoLayer diff --git a/libmproxy/protocol2/ssl.py b/libmproxy/protocol2/ssl.py index 6b44bf42..e8ff16cf 100644 --- a/libmproxy/protocol2/ssl.py +++ b/libmproxy/protocol2/ssl.py @@ -5,9 +5,9 @@ import traceback from netlib import tcp from ..proxy import ProxyError2 -from . import Layer +from .layer import Layer from .messages import Connect, Reconnect, ChangeServer -from .rawtcp import TcpLayer +from .auto import AutoLayer class ReconnectRequest(object): @@ -61,7 +61,7 @@ class SslLayer(Layer): elif self.client_ssl: self._establish_ssl_with_client() - layer = TcpLayer(self) + layer = AutoLayer(self) for message in layer(): if message != Connect or not self._connected: yield message @@ -225,4 +225,4 @@ class SslLayer(Layer): if self._sni_from_server_change: sans.append(self._sni_from_server_change) - return self.config.certstore.get_cert(host, sans) + return self.config.certstore.get_cert(host, sans) \ No newline at end of file -- cgit v1.2.3 From 531ca4a35684a83e57d4655922e9817814de41f6 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 25 Jul 2015 14:48:50 +0200 Subject: minor fixes --- libmproxy/protocol2/__init__.py | 2 +- libmproxy/protocol2/auto.py | 2 ++ libmproxy/protocol2/layer.py | 3 ++- libmproxy/protocol2/socks.py | 8 ++------ 4 files changed, 7 insertions(+), 8 deletions(-) diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index 6f4bfe44..20e5a888 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -3,4 +3,4 @@ from .layer import RootContext from .socks import Socks5IncomingLayer from .rawtcp import TcpLayer from .auto import AutoLayer -__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext"] \ No newline at end of file +__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext"] diff --git a/libmproxy/protocol2/auto.py b/libmproxy/protocol2/auto.py index 1c4293ac..a00f1f52 100644 --- a/libmproxy/protocol2/auto.py +++ b/libmproxy/protocol2/auto.py @@ -5,6 +5,8 @@ from .layer import Layer class AutoLayer(Layer): def __call__(self): d = self.client_conn.rfile.peek(1) + + # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello if d[0] == "\x16": layer = SslLayer(self, True, True) else: diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 14263f64..1cc8df70 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -37,6 +37,7 @@ from ..proxy import ProxyError2, Log from ..proxy.connection import ServerConnection from .messages import Connect, Reconnect, ChangeServer + class RootContext(object): """ The outmost context provided to the root layer. @@ -148,4 +149,4 @@ class ServerConnectionMixin(object): def _set_address(self, address): a = tcp.Address.wrap(address) self.log("Set new server address: " + repr(a), "debug") - self.server_address = address \ No newline at end of file + self.server_address = address diff --git a/libmproxy/protocol2/socks.py b/libmproxy/protocol2/socks.py index 9ca30bb4..7835b1a4 100644 --- a/libmproxy/protocol2/socks.py +++ b/libmproxy/protocol2/socks.py @@ -2,6 +2,7 @@ from __future__ import (absolute_import, print_function, division, unicode_liter from ..proxy import ProxyError, Socks5ProxyMode, ProxyError2 from .layer import Layer, ServerConnectionMixin +from .auto import AutoLayer class Socks5IncomingLayer(Layer, ServerConnectionMixin): @@ -15,12 +16,7 @@ class Socks5IncomingLayer(Layer, ServerConnectionMixin): self._set_address(address) - if address[1] == 443: - layer = AutoLayer(self) - else: - layer = AutoLayer(self) + layer = AutoLayer(self) for message in layer(): if not self._handle_server_message(message): yield message - -from .auto import AutoLayer -- cgit v1.2.3 From c46e3f90bbc38080a41a278340aaad27d8881fd9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 6 Aug 2015 11:09:01 +0200 Subject: apply fixes from proxy-refactor-cb branch --- libmproxy/protocol2/__init__.py | 2 +- libmproxy/protocol2/auto.py | 4 +++- libmproxy/protocol2/layer.py | 25 ++++++++++++------------- libmproxy/protocol2/messages.py | 2 +- libmproxy/protocol2/rawtcp.py | 2 +- libmproxy/protocol2/socks.py | 4 ++-- libmproxy/protocol2/ssl.py | 34 ++++++++++++++++++---------------- 7 files changed, 38 insertions(+), 35 deletions(-) diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index 20e5a888..95f67c6c 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -1,4 +1,4 @@ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) from .layer import RootContext from .socks import Socks5IncomingLayer from .rawtcp import TcpLayer diff --git a/libmproxy/protocol2/auto.py b/libmproxy/protocol2/auto.py index a00f1f52..fc111758 100644 --- a/libmproxy/protocol2/auto.py +++ b/libmproxy/protocol2/auto.py @@ -1,4 +1,4 @@ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) from .layer import Layer @@ -6,6 +6,8 @@ class AutoLayer(Layer): def __call__(self): d = self.client_conn.rfile.peek(1) + if not d: + return # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello if d[0] == "\x16": layer = SslLayer(self, True, True) diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 1cc8df70..30aed350 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -31,7 +31,7 @@ Further goals: - Upstream connections should be established as late as possible; inline scripts shall have a chance to handle everything locally. """ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) from netlib import tcp from ..proxy import ProxyError2, Log from ..proxy.connection import ServerConnection @@ -49,12 +49,6 @@ class RootContext(object): self.channel = channel # provides .ask() method to communicate with FlowMaster self.config = config # Proxy Configuration - def __getattr__(self, name): - """ - Accessing a nonexisting attribute does not throw an error but returns None instead. - """ - return None - class _LayerCodeCompletion(object): """ @@ -113,7 +107,7 @@ class ServerConnectionMixin(object): """ def __init__(self): - self.server_address = None + self._server_address = None self.server_conn = None def _handle_server_message(self, message): @@ -128,6 +122,16 @@ class ServerConnectionMixin(object): raise NotImplementedError return False + @property + def server_address(self): + return self._server_address + + @server_address.setter + def server_address(self, address): + self._server_address = tcp.Address.wrap(address) + self.log("Set new server address: " + repr(self.server_address), "debug") + + def _disconnect(self): """ Deletes (and closes) an existing server connection. @@ -145,8 +149,3 @@ class ServerConnectionMixin(object): self.server_conn.connect() except tcp.NetLibError as e: raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) - - def _set_address(self, address): - a = tcp.Address.wrap(address) - self.log("Set new server address: " + repr(a), "debug") - self.server_address = address diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index 52bb5a44..baf4312d 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -1,7 +1,7 @@ """ This module contains all valid messages layers can send to the underlying layers. """ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) class _Message(object): diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index b40c569f..39e48e24 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -1,4 +1,4 @@ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) from ..protocol.tcp import TCPHandler from .layer import Layer from .messages import Connect diff --git a/libmproxy/protocol2/socks.py b/libmproxy/protocol2/socks.py index 7835b1a4..14564521 100644 --- a/libmproxy/protocol2/socks.py +++ b/libmproxy/protocol2/socks.py @@ -1,4 +1,4 @@ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) from ..proxy import ProxyError, Socks5ProxyMode, ProxyError2 from .layer import Layer, ServerConnectionMixin @@ -14,7 +14,7 @@ class Socks5IncomingLayer(Layer, ServerConnectionMixin): # TODO: Unmonkeypatch raise ProxyError2(str(e), e) - self._set_address(address) + self.server_address = address layer = AutoLayer(self) for message in layer(): diff --git a/libmproxy/protocol2/ssl.py b/libmproxy/protocol2/ssl.py index e8ff16cf..c21956b7 100644 --- a/libmproxy/protocol2/ssl.py +++ b/libmproxy/protocol2/ssl.py @@ -1,4 +1,4 @@ -from __future__ import (absolute_import, print_function, division, unicode_literals) +from __future__ import (absolute_import, print_function, division) import Queue import threading import traceback @@ -76,7 +76,7 @@ class SslLayer(Layer): self._establish_ssl_with_server() @property - def sni(self): + def sni_for_upstream_connection(self): if self._sni_from_server_change is False: return None else: @@ -132,21 +132,22 @@ class SslLayer(Layer): The client has just sent the Sever Name Indication (SNI). """ try: + old_upstream_sni = self.sni_for_upstream_connection + sn = connection.get_servername() if not sn: return - sni = sn.decode("utf8").encode("idna") - - if sni != self.sni: - self._sni_from_handshake = sni + self._sni_from_handshake = sn.decode("utf8").encode("idna") + if old_upstream_sni != self.sni_for_upstream_connection: # Perform reconnect if self.server_ssl: reconnect = ReconnectRequest() - self.__client_ssl_queue.put() + self.__client_ssl_queue.put(reconnect) reconnect.done.wait() - # Now, change client context to reflect changed certificate: + if self._sni_from_handshake: + # Now, change client context to reflect possibly changed certificate: cert, key, chain_file = self.find_cert() new_context = self.client_conn.create_ssl_context( cert, key, @@ -183,7 +184,7 @@ class SslLayer(Layer): try: self.server_conn.establish_ssl( self.config.clientcerts, - self.sni, + self.sni_for_upstream_connection, method=self.config.openssl_method_server, options=self.config.openssl_options_server, verify_options=self.config.openssl_verification_mode_server, @@ -206,23 +207,24 @@ class SslLayer(Layer): "error") self.log("Aborting connection attempt", "error") raise ProxyError2(repr(e), e) - except Exception as e: + except tcp.NetLibError as e: raise ProxyError2(repr(e), e) def find_cert(self): host = self.server_conn.address.host - sans = [] + # TODO: Better use an OrderedSet here + sans = set() # Incorporate upstream certificate if self.server_conn.ssl_established and (not self.config.no_upstream_cert): upstream_cert = self.server_conn.cert - sans.extend(upstream_cert.altnames) + sans.update(upstream_cert.altnames) if upstream_cert.cn: - sans.append(host) + sans.add(host) host = upstream_cert.cn.decode("utf8").encode("idna") # Also add SNI values. if self._sni_from_handshake: - sans.append(self._sni_from_handshake) + sans.add(self._sni_from_handshake) if self._sni_from_server_change: - sans.append(self._sni_from_server_change) + sans.add(self._sni_from_server_change) - return self.config.certstore.get_cert(host, sans) \ No newline at end of file + return self.config.certstore.get_cert(host, list(sans)) \ No newline at end of file -- cgit v1.2.3 From aac0ab23ebb0e4d88306b12efee1dd31338f7664 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 6 Aug 2015 12:13:23 +0200 Subject: simplify layer code, add yield_from_callback decorator --- libmproxy/protocol2/layer.py | 58 +++++++++++++++++++++++++++++++++++++++++++- libmproxy/protocol2/ssl.py | 51 +++++++++----------------------------- 2 files changed, 69 insertions(+), 40 deletions(-) diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 30aed350..aaa51baf 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -32,6 +32,8 @@ Further goals: inline scripts shall have a chance to handle everything locally. """ from __future__ import (absolute_import, print_function, division) +import Queue +import threading from netlib import tcp from ..proxy import ProxyError2, Log from ..proxy.connection import ServerConnection @@ -131,7 +133,6 @@ class ServerConnectionMixin(object): self._server_address = tcp.Address.wrap(address) self.log("Set new server address: " + repr(self.server_address), "debug") - def _disconnect(self): """ Deletes (and closes) an existing server connection. @@ -149,3 +150,58 @@ class ServerConnectionMixin(object): self.server_conn.connect() except tcp.NetLibError as e: raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) + + +def yield_from_callback(fun): + """ + Decorator which makes it possible to yield from callbacks in the original thread. + As a use case, take the pyOpenSSL handle_sni callback: If we receive a new SNI from the client, + we need to reconnect to the server with the new SNI. Reconnecting would normally be done using "yield Reconnect()", + but we're in a pyOpenSSL callback here, outside of the main program flow. With this decorator, it looks as follows: + + def handle_sni(self): + # ... + self.yield_from_callback(Reconnect()) + + @yield_from_callback + def establish_ssl_with_client(): + self.client_conn.convert_to_ssl(...) + + for message in self.establish_ssl_with_client(): # will yield Reconnect at some point + yield message + + + Limitations: + - You cannot yield True. + """ + yield_queue = Queue.Queue() + + def do_yield(self, msg): + yield_queue.put(msg) + yield_queue.get() + + def wrapper(self, *args, **kwargs): + self.yield_from_callback = do_yield + + def run(): + try: + fun(self, *args, **kwargs) + yield_queue.put(True) + except Exception as e: + yield_queue.put(e) + + threading.Thread(target=run, name="YieldFromCallbackThread").start() + while True: + e = yield_queue.get() + if e is True: + break + elif isinstance(e, Exception): + # TODO: Include func name? + raise ProxyError2("Error from callback: " + repr(e), e) + else: + yield e + yield_queue.put(None) + + self.yield_from_callback = None + + return wrapper diff --git a/libmproxy/protocol2/ssl.py b/libmproxy/protocol2/ssl.py index c21956b7..32798e72 100644 --- a/libmproxy/protocol2/ssl.py +++ b/libmproxy/protocol2/ssl.py @@ -1,20 +1,13 @@ from __future__ import (absolute_import, print_function, division) -import Queue -import threading import traceback from netlib import tcp from ..proxy import ProxyError2 -from .layer import Layer +from .layer import Layer, yield_from_callback from .messages import Connect, Reconnect, ChangeServer from .auto import AutoLayer -class ReconnectRequest(object): - def __init__(self): - self.done = threading.Event() - - class SslLayer(Layer): def __init__(self, ctx, client_ssl, server_ssl): super(SslLayer, self).__init__(ctx) @@ -59,7 +52,8 @@ class SslLayer(Layer): for m in self._establish_ssl_with_client_and_server(): yield m elif self.client_ssl: - self._establish_ssl_with_client() + for m in self._establish_ssl_with_client(): + yield m layer = AutoLayer(self) for message in layer(): @@ -96,32 +90,12 @@ class SslLayer(Layer): except ProxyError2 as e: server_err = e - # The part below is a bit ugly as we cannot yield from the handle_sni callback. - # The workaround is to do that in a separate thread and yield from the main thread. - - # client_ssl_queue may contain the following elements - # - True, if ssl was successfully established - # - An Exception thrown by self._establish_ssl_with_client() - # - A threading.Event, which singnifies a request for a reconnect from the sni callback - self.__client_ssl_queue = Queue.Queue() - - def establish_client_ssl(): - try: - self._establish_ssl_with_client() - self.__client_ssl_queue.put(True) - except Exception as client_err: - self.__client_ssl_queue.put(client_err) - - threading.Thread(target=establish_client_ssl, name="ClientSSLThread").start() - e = self.__client_ssl_queue.get() - if isinstance(e, ReconnectRequest): - yield Reconnect() - self._establish_ssl_with_server() - e.done.set() - e = self.__client_ssl_queue.get() - if e is not True: - raise ProxyError2("Error when establish client SSL: " + repr(e), e) - self.__client_ssl_queue = None + for message in self._establish_ssl_with_client(): + if message == Reconnect: + yield message + self._establish_ssl_with_server() + else: + raise RuntimeError("Unexpected Message: %s" % message) if server_err and not self._sni_from_handshake: raise server_err @@ -142,9 +116,7 @@ class SslLayer(Layer): if old_upstream_sni != self.sni_for_upstream_connection: # Perform reconnect if self.server_ssl: - reconnect = ReconnectRequest() - self.__client_ssl_queue.put(reconnect) - reconnect.done.wait() + self.yield_from_callback(Reconnect()) if self._sni_from_handshake: # Now, change client context to reflect possibly changed certificate: @@ -163,6 +135,7 @@ class SslLayer(Layer): except: # pragma: no cover self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") + @yield_from_callback def _establish_ssl_with_client(self): self.log("Establish SSL with client", "debug") cert, key, chain_file = self.find_cert() @@ -227,4 +200,4 @@ class SslLayer(Layer): if self._sni_from_server_change: sans.add(self._sni_from_server_change) - return self.config.certstore.get_cert(host, list(sans)) \ No newline at end of file + return self.config.certstore.get_cert(host, list(sans)) -- cgit v1.2.3 From 314e0f5839fcd4a1c35323f61938b207232de287 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 6 Aug 2015 12:32:33 +0200 Subject: add reverseproxy mode, fix bugs --- libmproxy/protocol2/__init__.py | 3 ++- libmproxy/protocol2/layer.py | 12 ++++++------ libmproxy/protocol2/reverse_proxy.py | 19 +++++++++++++++++++ libmproxy/protocol2/ssl.py | 14 +++++++------- libmproxy/proxy/server.py | 2 +- 5 files changed, 35 insertions(+), 15 deletions(-) create mode 100644 libmproxy/protocol2/reverse_proxy.py diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index 95f67c6c..3f714f62 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -1,6 +1,7 @@ from __future__ import (absolute_import, print_function, division) from .layer import RootContext from .socks import Socks5IncomingLayer +from .reverse_proxy import ReverseProxy from .rawtcp import TcpLayer from .auto import AutoLayer -__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext"] +__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext", "ReverseProxy"] diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index aaa51baf..c18be83c 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -176,7 +176,7 @@ def yield_from_callback(fun): """ yield_queue = Queue.Queue() - def do_yield(self, msg): + def do_yield(msg): yield_queue.put(msg) yield_queue.get() @@ -192,14 +192,14 @@ def yield_from_callback(fun): threading.Thread(target=run, name="YieldFromCallbackThread").start() while True: - e = yield_queue.get() - if e is True: + msg = yield_queue.get() + if msg is True: break - elif isinstance(e, Exception): + elif isinstance(msg, Exception): # TODO: Include func name? - raise ProxyError2("Error from callback: " + repr(e), e) + raise ProxyError2("Error in %s: %s" % (fun.__name__, repr(msg)), msg) else: - yield e + yield msg yield_queue.put(None) self.yield_from_callback = None diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py new file mode 100644 index 00000000..dfffd2f2 --- /dev/null +++ b/libmproxy/protocol2/reverse_proxy.py @@ -0,0 +1,19 @@ +from __future__ import (absolute_import, print_function, division) + +from .layer import Layer, ServerConnectionMixin +from .ssl import SslLayer + + +class ReverseProxy(Layer, ServerConnectionMixin): + + def __init__(self, ctx, server_address, client_ssl, server_ssl): + super(ReverseProxy, self).__init__(ctx) + self.server_address = server_address + self.client_ssl = client_ssl + self.server_ssl = server_ssl + + def __call__(self): + layer = SslLayer(self, self.client_ssl, self.server_ssl) + for message in layer(): + if not self._handle_server_message(message): + yield message diff --git a/libmproxy/protocol2/ssl.py b/libmproxy/protocol2/ssl.py index 32798e72..a744a979 100644 --- a/libmproxy/protocol2/ssl.py +++ b/libmproxy/protocol2/ssl.py @@ -14,7 +14,7 @@ class SslLayer(Layer): self._client_ssl = client_ssl self._server_ssl = server_ssl self._connected = False - self._sni_from_handshake = None + self.client_sni = None self._sni_from_server_change = None def __call__(self): @@ -74,7 +74,7 @@ class SslLayer(Layer): if self._sni_from_server_change is False: return None else: - return self._sni_from_server_change or self._sni_from_handshake + return self._sni_from_server_change or self.client_sni def _establish_ssl_with_client_and_server(self): """ @@ -97,7 +97,7 @@ class SslLayer(Layer): else: raise RuntimeError("Unexpected Message: %s" % message) - if server_err and not self._sni_from_handshake: + if server_err and not self.client_sni: raise server_err def handle_sni(self, connection): @@ -111,14 +111,14 @@ class SslLayer(Layer): sn = connection.get_servername() if not sn: return - self._sni_from_handshake = sn.decode("utf8").encode("idna") + self.client_sni = sn.decode("utf8").encode("idna") if old_upstream_sni != self.sni_for_upstream_connection: # Perform reconnect if self.server_ssl: self.yield_from_callback(Reconnect()) - if self._sni_from_handshake: + if self.client_sni: # Now, change client context to reflect possibly changed certificate: cert, key, chain_file = self.find_cert() new_context = self.client_conn.create_ssl_context( @@ -195,8 +195,8 @@ class SslLayer(Layer): sans.add(host) host = upstream_cert.cn.decode("utf8").encode("idna") # Also add SNI values. - if self._sni_from_handshake: - sans.add(self._sni_from_handshake) + if self.client_sni: + sans.add(self.client_sni) if self._sni_from_server_change: sans.add(self._sni_from_server_change) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index c8990a9a..32d596ad 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -79,7 +79,7 @@ class ConnectionHandler2: self.config, self.channel ) - root_layer = protocol2.Socks5IncomingLayer(root_context) + root_layer = protocol2.ReverseProxy(root_context, ("localhost", 5000), True, True) try: for message in root_layer(): -- cgit v1.2.3 From 026330a3b014f24f095b839b29186036854de3bc Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 8 Aug 2015 16:08:57 +0200 Subject: cleaner Exceptions, ssl -> tls, upstream proxy mode --- libmproxy/exceptions.py | 22 ++++ libmproxy/protocol2/__init__.py | 3 +- libmproxy/protocol2/auto.py | 4 +- libmproxy/protocol2/layer.py | 10 +- libmproxy/protocol2/messages.py | 4 +- libmproxy/protocol2/rawtcp.py | 8 +- libmproxy/protocol2/reverse_proxy.py | 10 +- libmproxy/protocol2/socks.py | 6 +- libmproxy/protocol2/ssl.py | 203 ------------------------------- libmproxy/protocol2/tls.py | 203 +++++++++++++++++++++++++++++++ libmproxy/protocol2/transparent_proxy.py | 24 ++++ libmproxy/protocol2/upstream_proxy.py | 18 +++ libmproxy/proxy/connection.py | 8 ++ libmproxy/proxy/primitives.py | 6 - libmproxy/proxy/server.py | 6 +- 15 files changed, 306 insertions(+), 229 deletions(-) create mode 100644 libmproxy/exceptions.py delete mode 100644 libmproxy/protocol2/ssl.py create mode 100644 libmproxy/protocol2/tls.py create mode 100644 libmproxy/protocol2/transparent_proxy.py create mode 100644 libmproxy/protocol2/upstream_proxy.py diff --git a/libmproxy/exceptions.py b/libmproxy/exceptions.py new file mode 100644 index 00000000..4d98c024 --- /dev/null +++ b/libmproxy/exceptions.py @@ -0,0 +1,22 @@ +from __future__ import (absolute_import, print_function, division) + + +class ProxyException(Exception): + """ + Base class for all exceptions thrown by libmproxy. + """ + def __init__(self, message, cause=None): + """ + :param message: Error Message + :param cause: Exception object that caused this exception to be thrown. + """ + super(ProxyException, self).__init__(message) + self.cause = cause + + +class ProtocolException(ProxyException): + pass + + +class ServerException(ProxyException): + pass \ No newline at end of file diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index 3f714f62..0d232b13 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -2,6 +2,7 @@ from __future__ import (absolute_import, print_function, division) from .layer import RootContext from .socks import Socks5IncomingLayer from .reverse_proxy import ReverseProxy +from .upstream_proxy import UpstreamProxy from .rawtcp import TcpLayer from .auto import AutoLayer -__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext", "ReverseProxy"] +__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext", "ReverseProxy", "UpstreamProxy"] diff --git a/libmproxy/protocol2/auto.py b/libmproxy/protocol2/auto.py index fc111758..4a930720 100644 --- a/libmproxy/protocol2/auto.py +++ b/libmproxy/protocol2/auto.py @@ -10,11 +10,11 @@ class AutoLayer(Layer): return # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello if d[0] == "\x16": - layer = SslLayer(self, True, True) + layer = TlsLayer(self, True, True) else: layer = TcpLayer(self) for m in layer(): yield m from .rawtcp import TcpLayer -from .ssl import SslLayer +from .tls import TlsLayer diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index c18be83c..8aede22e 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -35,9 +35,10 @@ from __future__ import (absolute_import, print_function, division) import Queue import threading from netlib import tcp -from ..proxy import ProxyError2, Log +from ..proxy import Log from ..proxy.connection import ServerConnection from .messages import Connect, Reconnect, ChangeServer +from ..exceptions import ProtocolException class RootContext(object): @@ -51,6 +52,9 @@ class RootContext(object): self.channel = channel # provides .ask() method to communicate with FlowMaster self.config = config # Proxy Configuration + def next_layer(self): + print(type(self)) + class _LayerCodeCompletion(object): """ @@ -149,7 +153,7 @@ class ServerConnectionMixin(object): try: self.server_conn.connect() except tcp.NetLibError as e: - raise ProxyError2("Server connection to '%s' failed: %s" % (self.server_address, e), e) + raise ProtocolException("Server connection to '%s' failed: %s" % (self.server_address, e), e) def yield_from_callback(fun): @@ -197,7 +201,7 @@ def yield_from_callback(fun): break elif isinstance(msg, Exception): # TODO: Include func name? - raise ProxyError2("Error in %s: %s" % (fun.__name__, repr(msg)), msg) + raise ProtocolException("Error in %s: %s" % (fun.__name__, repr(msg)), msg) else: yield msg yield_queue.put(None) diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index baf4312d..3f53fbd4 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -32,9 +32,9 @@ class ChangeServer(_Message): Change the upstream server. """ - def __init__(self, address, server_ssl, sni, depth=1): + def __init__(self, address, server_tls, sni, depth=1): self.address = address - self.server_ssl = server_ssl + self.server_tls = server_tls self.sni = sni # upstream proxy scenario: you may want to change either the final target or the upstream proxy. diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index 39e48e24..608a53e3 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -1,4 +1,6 @@ from __future__ import (absolute_import, print_function, division) +import OpenSSL +from ..exceptions import ProtocolException from ..protocol.tcp import TCPHandler from .layer import Layer from .messages import Connect @@ -8,7 +10,11 @@ class TcpLayer(Layer): def __call__(self): yield Connect() tcp_handler = TCPHandler(self) - tcp_handler.handle_messages() + try: + tcp_handler.handle_messages() + except OpenSSL.SSL.Error as e: + raise ProtocolException("SSL error: %s" % repr(e), e) + def establish_server_connection(self): pass diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index dfffd2f2..cb6d1d78 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -1,19 +1,19 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin -from .ssl import SslLayer +from .tls import TlsLayer class ReverseProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx, server_address, client_ssl, server_ssl): + def __init__(self, ctx, server_address, client_tls, server_tls): super(ReverseProxy, self).__init__(ctx) self.server_address = server_address - self.client_ssl = client_ssl - self.server_ssl = server_ssl + self._client_tls = client_tls + self._server_tls = server_tls def __call__(self): - layer = SslLayer(self, self.client_ssl, self.server_ssl) + layer = TlsLayer(self, self._client_tls, self._server_tls) for message in layer(): if not self._handle_server_message(message): yield message diff --git a/libmproxy/protocol2/socks.py b/libmproxy/protocol2/socks.py index 14564521..1222ef5c 100644 --- a/libmproxy/protocol2/socks.py +++ b/libmproxy/protocol2/socks.py @@ -1,10 +1,10 @@ from __future__ import (absolute_import, print_function, division) -from ..proxy import ProxyError, Socks5ProxyMode, ProxyError2 +from ..exceptions import ProtocolException +from ..proxy import ProxyError, Socks5ProxyMode from .layer import Layer, ServerConnectionMixin from .auto import AutoLayer - class Socks5IncomingLayer(Layer, ServerConnectionMixin): def __call__(self): try: @@ -12,7 +12,7 @@ class Socks5IncomingLayer(Layer, ServerConnectionMixin): address = s5mode.get_upstream_server(self.client_conn)[2:] except ProxyError as e: # TODO: Unmonkeypatch - raise ProxyError2(str(e), e) + raise ProtocolException(str(e), e) self.server_address = address diff --git a/libmproxy/protocol2/ssl.py b/libmproxy/protocol2/ssl.py deleted file mode 100644 index a744a979..00000000 --- a/libmproxy/protocol2/ssl.py +++ /dev/null @@ -1,203 +0,0 @@ -from __future__ import (absolute_import, print_function, division) -import traceback -from netlib import tcp - -from ..proxy import ProxyError2 -from .layer import Layer, yield_from_callback -from .messages import Connect, Reconnect, ChangeServer -from .auto import AutoLayer - - -class SslLayer(Layer): - def __init__(self, ctx, client_ssl, server_ssl): - super(SslLayer, self).__init__(ctx) - self._client_ssl = client_ssl - self._server_ssl = server_ssl - self._connected = False - self.client_sni = None - self._sni_from_server_change = None - - def __call__(self): - """ - The strategy for establishing SSL is as follows: - First, we determine whether we need the server cert to establish ssl with the client. - If so, we first connect to the server and then to the client. - If not, we only connect to the client and do the server_ssl lazily on a Connect message. - - An additional complexity is that establish ssl with the server may require a SNI value from the client. - In an ideal world, we'd do the following: - 1. Start the SSL handshake with the client - 2. Check if the client sends a SNI. - 3. Pause the client handshake, establish SSL with the server. - 4. Finish the client handshake with the certificate from the server. - There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( - Thus, we resort to the following workaround when establishing SSL with the server: - 1. Try to establish SSL with the server without SNI. If this fails, we ignore it. - 2. Establish SSL with client. - - If there's a SNI callback, reconnect to the server with SNI. - - If not and the server connect failed, raise the original exception. - Further notes: - - OpenSSL 1.0.2 introduces a callback that would help here: - https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html - - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 - """ - client_ssl_requires_server_cert = ( - self._client_ssl and self._server_ssl and not self.config.no_upstream_cert - ) - lazy_server_ssl = ( - self._server_ssl and not client_ssl_requires_server_cert - ) - - if client_ssl_requires_server_cert: - for m in self._establish_ssl_with_client_and_server(): - yield m - elif self.client_ssl: - for m in self._establish_ssl_with_client(): - yield m - - layer = AutoLayer(self) - for message in layer(): - if message != Connect or not self._connected: - yield message - if message == Connect: - if lazy_server_ssl: - self._establish_ssl_with_server() - if message == ChangeServer and message.depth == 1: - self.server_ssl = message.server_ssl - self._sni_from_server_change = message.sni - if message == Reconnect or message == ChangeServer: - if self.server_ssl: - self._establish_ssl_with_server() - - @property - def sni_for_upstream_connection(self): - if self._sni_from_server_change is False: - return None - else: - return self._sni_from_server_change or self.client_sni - - def _establish_ssl_with_client_and_server(self): - """ - This function deals with the problem that the server may require a SNI value from the client. - """ - - # First, try to connect to the server. - yield Connect() - self._connected = True - server_err = None - try: - self._establish_ssl_with_server() - except ProxyError2 as e: - server_err = e - - for message in self._establish_ssl_with_client(): - if message == Reconnect: - yield message - self._establish_ssl_with_server() - else: - raise RuntimeError("Unexpected Message: %s" % message) - - if server_err and not self.client_sni: - raise server_err - - def handle_sni(self, connection): - """ - This callback gets called during the SSL handshake with the client. - The client has just sent the Sever Name Indication (SNI). - """ - try: - old_upstream_sni = self.sni_for_upstream_connection - - sn = connection.get_servername() - if not sn: - return - self.client_sni = sn.decode("utf8").encode("idna") - - if old_upstream_sni != self.sni_for_upstream_connection: - # Perform reconnect - if self.server_ssl: - self.yield_from_callback(Reconnect()) - - if self.client_sni: - # Now, change client context to reflect possibly changed certificate: - cert, key, chain_file = self.find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - connection.set_context(new_context) - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except: # pragma: no cover - self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") - - @yield_from_callback - def _establish_ssl_with_client(self): - self.log("Establish SSL with client", "debug") - cert, key, chain_file = self.find_cert() - try: - self.client_conn.convert_to_ssl( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - handle_sni=self.handle_sni, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - except tcp.NetLibError as e: - raise ProxyError2(repr(e), e) - - def _establish_ssl_with_server(self): - self.log("Establish SSL with server", "debug") - try: - self.server_conn.establish_ssl( - self.config.clientcerts, - self.sni_for_upstream_connection, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify_options=self.config.openssl_verification_mode_server, - ca_path=self.config.openssl_trusted_cadir_server, - ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, - ) - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") - except tcp.NetLibInvalidCertificateError as e: - ssl_cert_err = self.server_conn.ssl_verification_error - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") - raise ProxyError2(repr(e), e) - except tcp.NetLibError as e: - raise ProxyError2(repr(e), e) - - def find_cert(self): - host = self.server_conn.address.host - # TODO: Better use an OrderedSet here - sans = set() - # Incorporate upstream certificate - if self.server_conn.ssl_established and (not self.config.no_upstream_cert): - upstream_cert = self.server_conn.cert - sans.update(upstream_cert.altnames) - if upstream_cert.cn: - sans.add(host) - host = upstream_cert.cn.decode("utf8").encode("idna") - # Also add SNI values. - if self.client_sni: - sans.add(self.client_sni) - if self._sni_from_server_change: - sans.add(self._sni_from_server_change) - - return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py new file mode 100644 index 00000000..2362b2b2 --- /dev/null +++ b/libmproxy/protocol2/tls.py @@ -0,0 +1,203 @@ +from __future__ import (absolute_import, print_function, division) +import traceback +from netlib import tcp + +from ..exceptions import ProtocolException +from .layer import Layer, yield_from_callback +from .messages import Connect, Reconnect, ChangeServer +from .auto import AutoLayer + + +class TlsLayer(Layer): + def __init__(self, ctx, client_tls, server_tls): + super(TlsLayer, self).__init__(ctx) + self._client_tls = client_tls + self._server_tls = server_tls + self._connected = False + self.client_sni = None + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we resort to the following workaround when establishing SSL with the server: + 1. Try to establish SSL with the server without SNI. If this fails, we ignore it. + 2. Establish SSL with client. + - If there's a SNI callback, reconnect to the server with SNI. + - If not and the server connect failed, raise the original exception. + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + client_tls_requires_server_cert = ( + self._client_tls and self._server_tls and not self.config.no_upstream_cert + ) + lazy_server_tls = ( + self._server_tls and not client_tls_requires_server_cert + ) + + if client_tls_requires_server_cert: + for m in self._establish_tls_with_client_and_server(): + yield m + elif self._client_tls: + for m in self._establish_tls_with_client(): + yield m + + self.next_layer() + layer = AutoLayer(self) + for message in layer(): + if message != Connect or not self._connected: + yield message + if message == Connect: + if lazy_server_tls: + self._establish_tls_with_server() + if message == ChangeServer and message.depth == 1: + self._server_tls = message.server_tls + self._sni_from_server_change = message.sni + if message == Reconnect or message == ChangeServer: + if self._server_tls: + self._establish_tls_with_server() + + @property + def sni_for_upstream_connection(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self.client_sni + + def _establish_tls_with_client_and_server(self): + """ + This function deals with the problem that the server may require a SNI value from the client. + """ + + # First, try to connect to the server. + yield Connect() + self._connected = True + server_err = None + try: + self._establish_tls_with_server() + except ProtocolException as e: + server_err = e + + for message in self._establish_tls_with_client(): + if message == Reconnect: + yield message + self._establish_tls_with_server() + else: + raise RuntimeError("Unexpected Message: %s" % message) + + if server_err and not self.client_sni: + raise server_err + + def handle_sni(self, connection): + """ + This callback gets called during the TLS handshake with the client. + The client has just sent the Sever Name Indication (SNI). + """ + try: + old_upstream_sni = self.sni_for_upstream_connection + + sn = connection.get_servername() + if not sn: + return + self.client_sni = sn.decode("utf8").encode("idna") + + if old_upstream_sni != self.sni_for_upstream_connection: + # Perform reconnect + if self._server_tls: + self.yield_from_callback(Reconnect()) + + if self.client_sni: + # Now, change client context to reflect possibly changed certificate: + cert, key, chain_file = self.find_cert() + new_context = self.client_conn.create_ssl_context( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file + ) + connection.set_context(new_context) + # An unhandled exception in this method will core dump PyOpenSSL, so + # make dang sure it doesn't happen. + except: # pragma: no cover + self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") + + @yield_from_callback + def _establish_tls_with_client(self): + self.log("Establish TLS with client", "debug") + cert, key, chain_file = self.find_cert() + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + handle_sni=self.handle_sni, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file + ) + except tcp.NetLibError as e: + raise ProtocolException(repr(e), e) + + def _establish_tls_with_server(self): + self.log("Establish TLS with server", "debug") + try: + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni_for_upstream_connection, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + ) + tls_cert_err = self.server_conn.ssl_verification_error + if tls_cert_err is not None: + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except tcp.NetLibInvalidCertificateError as e: + tls_cert_err = self.server_conn.ssl_verification_error + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProtocolException(repr(e), e) + except tcp.NetLibError as e: + raise ProtocolException(repr(e), e) + + def find_cert(self): + host = self.server_conn.address.host + sans = set() + # Incorporate upstream certificate + if self.server_conn.tls_established and (not self.config.no_upstream_cert): + upstream_cert = self.server_conn.cert + sans.update(upstream_cert.altnames) + if upstream_cert.cn: + sans.add(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self.client_sni: + sans.add(self.client_sni) + if self._sni_from_server_change: + sans.add(self._sni_from_server_change) + + return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py new file mode 100644 index 00000000..078954c2 --- /dev/null +++ b/libmproxy/protocol2/transparent_proxy.py @@ -0,0 +1,24 @@ +from __future__ import (absolute_import, print_function, division) + +from ..exceptions import ProtocolException +from .. import platform +from .layer import Layer, ServerConnectionMixin +from .auto import AutoLayer + + +class TransparentProxy(Layer, ServerConnectionMixin): + + def __init__(self, ctx): + super(TransparentProxy, self).__init__(ctx) + self.resolver = platform.resolver() + + def __call__(self): + try: + self.server_address = self.resolver.original_addr(self.client_conn.connection) + except Exception as e: + raise ProtocolException("Transparent mode failure: %s" % repr(e), e) + + layer = AutoLayer(self) + for message in layer(): + if not self._handle_server_message(message): + yield message diff --git a/libmproxy/protocol2/upstream_proxy.py b/libmproxy/protocol2/upstream_proxy.py new file mode 100644 index 00000000..bd920309 --- /dev/null +++ b/libmproxy/protocol2/upstream_proxy.py @@ -0,0 +1,18 @@ +from __future__ import (absolute_import, print_function, division) + +from .layer import Layer, ServerConnectionMixin +#from .http import HttpLayer + + +class UpstreamProxy(Layer, ServerConnectionMixin): + + def __init__(self, ctx, server_address): + super(UpstreamProxy, self).__init__(ctx) + self.server_address = server_address + + def __call__(self): + #layer = HttpLayer(self) + layer = None + for message in layer(): + if not self._handle_server_message(message): + yield message diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index 9e03157a..49210e47 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -32,6 +32,10 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): port=self.address.port ) + @property + def tls_established(self): + return self.ssl_established + _stateobject_attributes = dict( ssl_established=bool, timestamp_start=float, @@ -112,6 +116,10 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): port=self.address.port ) + @property + def tls_established(self): + return self.ssl_established + _stateobject_attributes = dict( state=list, timestamp_start=float, diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index fd4eb882..a9f31181 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -2,12 +2,6 @@ from __future__ import absolute_import from netlib import socks, tcp -class ProxyError2(Exception): - def __init__(self, message, cause=None): - super(ProxyError2, self).__init__(message) - self.cause = cause - - class ProxyError(Exception): def __init__(self, code, message, headers=None): super(ProxyError, self).__init__(message) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 32d596ad..c107cbed 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -7,7 +7,7 @@ from netlib import tcp from ..protocol.handle import protocol_handler from .. import protocol2 -from .primitives import ProxyServerError, Log, ProxyError, ProxyError2 +from .primitives import ProxyServerError, Log, ProxyError from .connection import ClientConnection, ServerConnection @@ -79,12 +79,12 @@ class ConnectionHandler2: self.config, self.channel ) - root_layer = protocol2.ReverseProxy(root_context, ("localhost", 5000), True, True) + root_layer = protocol2.Socks5IncomingLayer(root_context) try: for message in root_layer(): print("Root layer receveived: %s" % message) - except ProxyError2 as e: + except protocol2.ProtocolException as e: self.log(e, "info") except Exception: self.log(traceback.format_exc(), "error") -- cgit v1.2.3 From aef3b626a70de5f385c8f5496c2e49575b5c3e1c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 11 Aug 2015 20:27:34 +0200 Subject: wip commit --- libmproxy/exceptions.py | 10 +++- libmproxy/protocol2/__init__.py | 12 +++-- libmproxy/protocol2/auto.py | 20 ------- libmproxy/protocol2/http.py | 90 +++++++++++++++++++++++++++++++ libmproxy/protocol2/http_protocol_mock.py | 13 +++++ libmproxy/protocol2/http_proxy.py | 23 ++++++++ libmproxy/protocol2/layer.py | 17 +----- libmproxy/protocol2/root_context.py | 32 +++++++++++ libmproxy/protocol2/socks.py | 22 -------- libmproxy/protocol2/socks_proxy.py | 22 ++++++++ libmproxy/protocol2/tls.py | 4 +- libmproxy/protocol2/transparent_proxy.py | 3 +- libmproxy/protocol2/upstream_proxy.py | 18 ------- libmproxy/proxy/server.py | 5 +- 14 files changed, 202 insertions(+), 89 deletions(-) delete mode 100644 libmproxy/protocol2/auto.py create mode 100644 libmproxy/protocol2/http.py create mode 100644 libmproxy/protocol2/http_protocol_mock.py create mode 100644 libmproxy/protocol2/http_proxy.py create mode 100644 libmproxy/protocol2/root_context.py delete mode 100644 libmproxy/protocol2/socks.py create mode 100644 libmproxy/protocol2/socks_proxy.py delete mode 100644 libmproxy/protocol2/upstream_proxy.py diff --git a/libmproxy/exceptions.py b/libmproxy/exceptions.py index 4d98c024..3825c409 100644 --- a/libmproxy/exceptions.py +++ b/libmproxy/exceptions.py @@ -18,5 +18,13 @@ class ProtocolException(ProxyException): pass +class HttpException(ProtocolException): + pass + + +class InvalidCredentials(HttpException): + pass + + class ServerException(ProxyException): - pass \ No newline at end of file + pass diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index 0d232b13..e3f06ad7 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -1,8 +1,10 @@ from __future__ import (absolute_import, print_function, division) -from .layer import RootContext -from .socks import Socks5IncomingLayer +from .root_context import RootContext +from .socks_proxy import Socks5Proxy from .reverse_proxy import ReverseProxy -from .upstream_proxy import UpstreamProxy +from .http_proxy import HttpProxy, HttpUpstreamProxy from .rawtcp import TcpLayer -from .auto import AutoLayer -__all__ = ["Socks5IncomingLayer", "TcpLayer", "AutoLayer", "RootContext", "ReverseProxy", "UpstreamProxy"] + +__all__ = [ + "Socks5Proxy", "TcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" +] diff --git a/libmproxy/protocol2/auto.py b/libmproxy/protocol2/auto.py deleted file mode 100644 index 4a930720..00000000 --- a/libmproxy/protocol2/auto.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import (absolute_import, print_function, division) -from .layer import Layer - - -class AutoLayer(Layer): - def __call__(self): - d = self.client_conn.rfile.peek(1) - - if not d: - return - # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello - if d[0] == "\x16": - layer = TlsLayer(self, True, True) - else: - layer = TcpLayer(self) - for m in layer(): - yield m - -from .rawtcp import TcpLayer -from .tls import TlsLayer diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py new file mode 100644 index 00000000..54cc9dbc --- /dev/null +++ b/libmproxy/protocol2/http.py @@ -0,0 +1,90 @@ +from __future__ import (absolute_import, print_function, division) + +from .layer import Layer, ServerConnectionMixin +from libmproxy import version +from libmproxy.exceptions import InvalidCredentials +from libmproxy.protocol.http import HTTPFlow +from libmproxy.protocol.http_wrappers import HTTPResponse +from libmproxy.protocol2.http_protocol_mock import HTTP1 +from netlib import tcp +from netlib.http import status_codes +from netlib import odict + + +def send_http_error_response(status_code, message, headers=odict.ODictCaseless()): + response = status_codes.RESPONSES.get(status_code, "Unknown") + body = """ + + + %d %s + + %s + + """.strip() % (status_code, response, message) + + headers["Server"] = [version.NAMEVERSION] + headers["Connection"] = ["close"] + headers["Content-Length"] = [len(body)] + headers["Content-Type"] = ["text/html"] + + resp = HTTPResponse( + (1, 1), # if HTTP/2 is used, this value is ignored anyway + status_code, + response, + headers, + body, + ) + + protocol = self.c.client_conn.protocol or http1.HTTP1Protocol(self.c.client_conn) + self.c.client_conn.send(protocol.assemble(resp)) + +class HttpLayer(Layer, ServerConnectionMixin): + """ + HTTP 1 Layer + """ + + def __init__(self, ctx): + super(HttpLayer, self).__init__(ctx) + self.skip_authentication = False + + def __call__(self): + while True: + flow = HTTPFlow(self.client_conn, self.server_conn) + try: + request = HTTP1.read_request( + self.client_conn, + body_size_limit=self.c.config.body_size_limit + ) + except tcp.NetLibError: + # don't throw an error for disconnects that happen + # before/between requests. + return + + self.c.log("request", "debug", [repr(request)]) + + self.check_authentication(request) + + if self.mode == "regular" and request.form_in == "authority": + raise NotImplementedError + + + + ret = self.process_request(flow, request) + if ret is True: + continue + if ret is False: + return + + def check_authentication(self, request): + if self.config.authenticator: + if self.config.authenticator.authenticate(request.headers): + self.config.authenticator.clean(request.headers) + else: + self.send_error() + raise InvalidCredentials("Proxy Authentication Required") + raise http.HttpAuthenticationError( + self.c.config.authenticator.auth_challenge_headers()) + return request.headers + + def send_error(self, code, message, headers): + pass \ No newline at end of file diff --git a/libmproxy/protocol2/http_protocol_mock.py b/libmproxy/protocol2/http_protocol_mock.py new file mode 100644 index 00000000..962a76d6 --- /dev/null +++ b/libmproxy/protocol2/http_protocol_mock.py @@ -0,0 +1,13 @@ +""" +Temporary mock to sort out API discrepancies +""" +from netlib.http.http1 import HTTP1Protocol + + +class HTTP1(object): + @staticmethod + def read_request(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection).read_request(*args, **kwargs) \ No newline at end of file diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py new file mode 100644 index 00000000..6b3b6a82 --- /dev/null +++ b/libmproxy/protocol2/http_proxy.py @@ -0,0 +1,23 @@ +from __future__ import (absolute_import, print_function, division) + +from .layer import Layer, ServerConnectionMixin +from .http import HttpLayer + + +class HttpProxy(Layer): + def __call__(self): + layer = HttpLayer(self) + for message in layer(): + yield message + + +class HttpUpstreamProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address): + super(HttpUpstreamProxy, self).__init__(ctx) + self.server_address = server_address + + def __call__(self): + layer = HttpLayer(self) + for message in layer(): + if not self._handle_server_message(message): + yield message diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 8aede22e..0ae64c43 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -41,21 +41,6 @@ from .messages import Connect, Reconnect, ChangeServer from ..exceptions import ProtocolException -class RootContext(object): - """ - The outmost context provided to the root layer. - As a consequence, every layer has .client_conn, .channel and .config. - """ - - def __init__(self, client_conn, config, channel): - self.client_conn = client_conn # Client Connection - self.channel = channel # provides .ask() method to communicate with FlowMaster - self.config = config # Proxy Configuration - - def next_layer(self): - print(type(self)) - - class _LayerCodeCompletion(object): """ Dummy class that provides type hinting in PyCharm, which simplifies development a lot. @@ -208,4 +193,4 @@ def yield_from_callback(fun): self.yield_from_callback = None - return wrapper + return wrapper \ No newline at end of file diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py new file mode 100644 index 00000000..cbe596aa --- /dev/null +++ b/libmproxy/protocol2/root_context.py @@ -0,0 +1,32 @@ +from .rawtcp import TcpLayer +from .tls import TlsLayer + + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel, .next_layer() and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def next_layer(self, top_layer): + """ + This function determines the next layer in the protocol stack. + :param top_layer: the current top layer + :return: The next layer. + """ + + d = top_layer.client_conn.rfile.peek(1) + + if not d: + return + # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + if d[0] == "\x16": + layer = TlsLayer(top_layer, True, True) + else: + layer = TcpLayer(top_layer) + return layer diff --git a/libmproxy/protocol2/socks.py b/libmproxy/protocol2/socks.py deleted file mode 100644 index 1222ef5c..00000000 --- a/libmproxy/protocol2/socks.py +++ /dev/null @@ -1,22 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from ..exceptions import ProtocolException -from ..proxy import ProxyError, Socks5ProxyMode -from .layer import Layer, ServerConnectionMixin -from .auto import AutoLayer - -class Socks5IncomingLayer(Layer, ServerConnectionMixin): - def __call__(self): - try: - s5mode = Socks5ProxyMode(self.config.ssl_ports) - address = s5mode.get_upstream_server(self.client_conn)[2:] - except ProxyError as e: - # TODO: Unmonkeypatch - raise ProtocolException(str(e), e) - - self.server_address = address - - layer = AutoLayer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py new file mode 100644 index 00000000..c89477ca --- /dev/null +++ b/libmproxy/protocol2/socks_proxy.py @@ -0,0 +1,22 @@ +from __future__ import (absolute_import, print_function, division) + +from ..exceptions import ProtocolException +from ..proxy import ProxyError, Socks5ProxyMode +from .layer import Layer, ServerConnectionMixin + + +class Socks5Proxy(Layer, ServerConnectionMixin): + def __call__(self): + try: + s5mode = Socks5ProxyMode(self.config.ssl_ports) + address = s5mode.get_upstream_server(self.client_conn)[2:] + except ProxyError as e: + # TODO: Unmonkeypatch + raise ProtocolException(str(e), e) + + self.server_address = address + + layer = self.ctx.next_layer(self) + for message in layer(): + if not self._handle_server_message(message): + yield message diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 2362b2b2..999cbea6 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -5,7 +5,6 @@ from netlib import tcp from ..exceptions import ProtocolException from .layer import Layer, yield_from_callback from .messages import Connect, Reconnect, ChangeServer -from .auto import AutoLayer class TlsLayer(Layer): @@ -55,8 +54,7 @@ class TlsLayer(Layer): for m in self._establish_tls_with_client(): yield m - self.next_layer() - layer = AutoLayer(self) + layer = self.ctx.next_layer(self) for message in layer(): if message != Connect or not self._connected: yield message diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py index 078954c2..f073e2f8 100644 --- a/libmproxy/protocol2/transparent_proxy.py +++ b/libmproxy/protocol2/transparent_proxy.py @@ -3,7 +3,6 @@ from __future__ import (absolute_import, print_function, division) from ..exceptions import ProtocolException from .. import platform from .layer import Layer, ServerConnectionMixin -from .auto import AutoLayer class TransparentProxy(Layer, ServerConnectionMixin): @@ -18,7 +17,7 @@ class TransparentProxy(Layer, ServerConnectionMixin): except Exception as e: raise ProtocolException("Transparent mode failure: %s" % repr(e), e) - layer = AutoLayer(self) + layer = self.ctx.next_layer(self) for message in layer(): if not self._handle_server_message(message): yield message diff --git a/libmproxy/protocol2/upstream_proxy.py b/libmproxy/protocol2/upstream_proxy.py deleted file mode 100644 index bd920309..00000000 --- a/libmproxy/protocol2/upstream_proxy.py +++ /dev/null @@ -1,18 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from .layer import Layer, ServerConnectionMixin -#from .http import HttpLayer - - -class UpstreamProxy(Layer, ServerConnectionMixin): - - def __init__(self, ctx, server_address): - super(UpstreamProxy, self).__init__(ctx) - self.server_address = server_address - - def __call__(self): - #layer = HttpLayer(self) - layer = None - for message in layer(): - if not self._handle_server_message(message): - yield message diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index c107cbed..6a7048e0 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -7,6 +7,7 @@ from netlib import tcp from ..protocol.handle import protocol_handler from .. import protocol2 +from ..exceptions import ProtocolException from .primitives import ProxyServerError, Log, ProxyError from .connection import ClientConnection, ServerConnection @@ -79,12 +80,12 @@ class ConnectionHandler2: self.config, self.channel ) - root_layer = protocol2.Socks5IncomingLayer(root_context) + root_layer = protocol2.Socks5Proxy(root_context) try: for message in root_layer(): print("Root layer receveived: %s" % message) - except protocol2.ProtocolException as e: + except ProtocolException as e: self.log(e, "info") except Exception: self.log(traceback.format_exc(), "error") -- cgit v1.2.3 From 808218f4bc64be8de065604f6509eb75d98fde88 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 14 Aug 2015 10:41:11 +0200 Subject: more work on http layer --- libmproxy/protocol2/__init__.py | 4 +- libmproxy/protocol2/http.py | 174 ++++++++++++++++++++++++++++++----- libmproxy/protocol2/http_proxy.py | 5 +- libmproxy/protocol2/layer.py | 11 ++- libmproxy/protocol2/rawtcp.py | 3 +- libmproxy/protocol2/reverse_proxy.py | 5 +- libmproxy/protocol2/root_context.py | 29 +++++- libmproxy/protocol2/tls.py | 11 ++- 8 files changed, 200 insertions(+), 42 deletions(-) diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index e3f06ad7..d5dafaae 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -3,8 +3,8 @@ from .root_context import RootContext from .socks_proxy import Socks5Proxy from .reverse_proxy import ReverseProxy from .http_proxy import HttpProxy, HttpUpstreamProxy -from .rawtcp import TcpLayer +from .rawtcp import RawTcpLayer __all__ = [ - "Socks5Proxy", "TcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" + "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" ] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 54cc9dbc..44ebf6a8 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -1,17 +1,22 @@ from __future__ import (absolute_import, print_function, division) +from .. import version +from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer, ServerConnectionMixin -from libmproxy import version -from libmproxy.exceptions import InvalidCredentials +from .messages import ChangeServer, Connect, Reconnect +from .http_proxy import HttpProxy, HttpUpstreamProxy +from libmproxy.protocol import KILL + from libmproxy.protocol.http import HTTPFlow -from libmproxy.protocol.http_wrappers import HTTPResponse +from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from libmproxy.protocol2.http_protocol_mock import HTTP1 +from libmproxy.protocol2.tls import TlsLayer from netlib import tcp from netlib.http import status_codes from netlib import odict -def send_http_error_response(status_code, message, headers=odict.ODictCaseless()): +def make_error_response(status_code, message, headers=None): response = status_codes.RESPONSES.get(status_code, "Unknown") body = """ @@ -22,21 +27,40 @@ def send_http_error_response(status_code, message, headers=odict.ODictCaseless() """.strip() % (status_code, response, message) + if not headers: + headers = odict.ODictCaseless() headers["Server"] = [version.NAMEVERSION] headers["Connection"] = ["close"] headers["Content-Length"] = [len(body)] headers["Content-Type"] = ["text/html"] - resp = HTTPResponse( - (1, 1), # if HTTP/2 is used, this value is ignored anyway + return HTTPResponse( + (1, 1), # FIXME: Should be a string. status_code, response, headers, body, ) - protocol = self.c.client_conn.protocol or http1.HTTP1Protocol(self.c.client_conn) - self.c.client_conn.send(protocol.assemble(resp)) +def make_connect_request(address): + return HTTPRequest( + "authority", "CONNECT", None, address.host, address.port, None, (1,1), + odict.ODictCaseless(), "" + ) + +def make_connect_response(httpversion): + headers = odict.ODictCaseless([ + ["Content-Length", "0"], + ["Proxy-Agent", version.NAMEVERSION] + ]) + return HTTPResponse( + httpversion, + 200, + "Connection established", + headers, + "", + ) + class HttpLayer(Layer, ServerConnectionMixin): """ @@ -45,11 +69,16 @@ class HttpLayer(Layer, ServerConnectionMixin): def __init__(self, ctx): super(HttpLayer, self).__init__(ctx) - self.skip_authentication = False + if any(isinstance(l, HttpProxy) for l in self.layers): + self.mode = "regular" + elif any(isinstance(l, HttpUpstreamProxy) for l in self.layers): + self.mode = "upstream" + else: + # also includes socks or reverse mode, which are handled similarly on this layer. + self.mode = "transparent" def __call__(self): while True: - flow = HTTPFlow(self.client_conn, self.server_conn) try: request = HTTP1.read_request( self.client_conn, @@ -62,29 +91,126 @@ class HttpLayer(Layer, ServerConnectionMixin): self.c.log("request", "debug", [repr(request)]) - self.check_authentication(request) + # Handle Proxy Authentication + self.authenticate(request) + # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.form_in == "authority": - raise NotImplementedError - + self.server_address = (request.host, request.port) + self.send_to_client(make_connect_response(request.httpversion)) + layer = self.ctx.next_layer(self) + for message in layer(): + if not self._handle_server_message(message): + yield message + return + # Make sure that the incoming request matches our expectations + self.validate_request(request) - ret = self.process_request(flow, request) - if ret is True: - continue - if ret is False: + flow = HTTPFlow(self.client_conn, self.server_conn) + flow.request = request + if not self.process_request_hook(flow): + self.log("Connection killed", "info") return - def check_authentication(self, request): + if not flow.response: + self.establish_server_connection(flow) + + def process_request_hook(self, flow): + # Determine .scheme, .host and .port attributes for inline scripts. + # For absolute-form requests, they are directly given in the request. + # For authority-form requests, we only need to determine the request scheme. + # For relative-form requests, we need to determine host and port as + # well. + if self.mode == "regular": + pass # only absolute-form at this point, nothing to do here. + elif self.mode == "upstream": + if flow.request.form_in == "authority": + flow.request.scheme = "http" # pseudo value + else: + flow.request.host = self.ctx.server_address.host + flow.request.port = self.ctx.server_address.port + flow.request.scheme = self.server_conn.tls_established + + # TODO: Expose ChangeServer functionality to inline scripts somehow? (yield_from_callback?) + request_reply = self.c.channel.ask("request", flow) + if request_reply is None or request_reply == KILL: + return False + if isinstance(request_reply, HTTPResponse): + flow.response = request_reply + return + + def establish_server_connection(self, flow): + + address = tcp.Address((flow.request.host, flow.request.port)) + tls = (flow.request.scheme == "https") + if self.mode == "regular" or self.mode == "transparent": + # If there's an existing connection that doesn't match our expectations, kill it. + if self.server_address != address or tls != self.server_address.ssl_established: + yield ChangeServer(address, tls, address.host) + # Establish connection is neccessary. + if not self.server_conn: + yield Connect() + + # ChangeServer is not guaranteed to work with TLS: + # If there's not TlsLayer below which could catch the exception, + # TLS will not be established. + if tls and not self.server_conn.tls_established: + raise ProtocolException("Cannot upgrade to SSL, no TLS layer on the protocol stack.") + + else: + if tls: + raise HttpException("Cannot change scheme in upstream proxy mode.") + """ + # This is a very ugly (untested) workaround to solve a very ugly problem. + # FIXME: Check if connected first. + if self.server_conn.tls_established and not ssl: + yield Reconnect() + elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: + if self.server_conn.tls_established: + yield Reconnect() + + self.send_to_server(make_connect_request(address)) + tls_layer = TlsLayer(self, False, True) + tls_layer._establish_tls_with_server() + """ + + def validate_request(self, request): + if request.form_in == "absolute" and request.scheme != "http": + self.send_resplonse(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + raise HttpException("Invalid request scheme: %s" % request.scheme) + + expected_request_forms = { + "regular": ("absolute",), # an authority request would already be handled. + "upstream": ("authority", "absolute"), + "transparent": ("regular",) + } + + allowed_request_forms = expected_request_forms[self.mode] + if request.form_in not in allowed_request_forms: + err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( + " or ".join(allowed_request_forms), request.form_in + ) + self.send_to_client(make_error_response(400, err_message)) + raise HttpException(err_message) + + def authenticate(self, request): if self.config.authenticator: if self.config.authenticator.authenticate(request.headers): self.config.authenticator.clean(request.headers) else: - self.send_error() + self.send_to_client(make_error_response( + 407, + "Proxy Authentication Required", + self.config.authenticator.auth_challenge_headers() + )) raise InvalidCredentials("Proxy Authentication Required") - raise http.HttpAuthenticationError( - self.c.config.authenticator.auth_challenge_headers()) - return request.headers - def send_error(self, code, message, headers): - pass \ No newline at end of file + def send_to_server(self, message): + self.server_conn.wfile.wrie(message) + + def send_to_client(self, message): + # FIXME + # - possibly do some http2 stuff here + # - fix message assembly. + self.client_conn.wfile.write(message) diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 6b3b6a82..51d3763c 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -4,11 +4,12 @@ from .layer import Layer, ServerConnectionMixin from .http import HttpLayer -class HttpProxy(Layer): +class HttpProxy(Layer, ServerConnectionMixin): def __call__(self): layer = HttpLayer(self) for message in layer(): - yield message + if not self._handle_server_message(message): + yield message class HttpUpstreamProxy(Layer, ServerConnectionMixin): diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 0ae64c43..e9f5c667 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -16,9 +16,7 @@ Regular proxy, CONNECT request with WebSockets over SSL: Automated protocol detection by peeking into the buffer: TransparentModeLayer - AutoLayer SslLayer - AutoLayer Http2Layer Communication between layers is done as follows: @@ -91,6 +89,13 @@ class Layer(_LayerCodeCompletion): full_msg = "\n".join(full_msg) self.channel.tell("log", Log(full_msg, level)) + @property + def layers(self): + return [self] + self.ctx.layers + + def __repr__(self): + return "%s\r\n %s" % (self.__class__.name__, repr(self.ctx)) + class ServerConnectionMixin(object): """ @@ -133,6 +138,8 @@ class ServerConnectionMixin(object): self.server_conn = None def _connect(self): + if not self.server_address: + raise ProtocolException("Cannot connect to server, no server address given.") self.log("serverconnect", "debug", [repr(self.server_address)]) self.server_conn = ServerConnection(self.server_address) try: diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index 608a53e3..167c8c79 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -1,4 +1,5 @@ from __future__ import (absolute_import, print_function, division) + import OpenSSL from ..exceptions import ProtocolException from ..protocol.tcp import TCPHandler @@ -6,7 +7,7 @@ from .layer import Layer from .messages import Connect -class TcpLayer(Layer): +class RawTcpLayer(Layer): def __call__(self): yield Connect() tcp_handler = TCPHandler(self) diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index cb6d1d78..bb414ec3 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -13,7 +13,10 @@ class ReverseProxy(Layer, ServerConnectionMixin): self._server_tls = server_tls def __call__(self): - layer = TlsLayer(self, self._client_tls, self._server_tls) + if self._client_tls or self._server_tls: + layer = TlsLayer(self, self._client_tls, self._server_tls) + else: + layer = self.ctx.next_layer(self) for message in layer(): if not self._handle_server_message(message): yield message diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index cbe596aa..3b341778 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -1,4 +1,6 @@ -from .rawtcp import TcpLayer +from __future__ import (absolute_import, print_function, division) + +from .rawtcp import RawTcpLayer from .tls import TlsLayer @@ -20,13 +22,30 @@ class RootContext(object): :return: The next layer. """ - d = top_layer.client_conn.rfile.peek(1) + d = top_layer.client_conn.rfile.peek(3) + + # TODO: Handle ignore and tcp passthrough + + # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + is_tls_client_hello = ( + len(d) == 3 and + d[0] == '\x16' and + d[1] == '\x03' and + d[2] in ('\x00', '\x01', '\x02', '\x03') + ) if not d: return - # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello - if d[0] == "\x16": + + if is_tls_client_hello: layer = TlsLayer(top_layer, True, True) else: - layer = TcpLayer(top_layer) + layer = RawTcpLayer(top_layer) return layer + + @property + def layers(self): + return [] + + def __repr__(self): + return "RootContext" diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 999cbea6..988304aa 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -1,4 +1,5 @@ from __future__ import (absolute_import, print_function, division) + import traceback from netlib import tcp @@ -99,7 +100,7 @@ class TlsLayer(Layer): if server_err and not self.client_sni: raise server_err - def handle_sni(self, connection): + def __handle_sni(self, connection): """ This callback gets called during the TLS handshake with the client. The client has just sent the Sever Name Indication (SNI). @@ -119,7 +120,7 @@ class TlsLayer(Layer): if self.client_sni: # Now, change client context to reflect possibly changed certificate: - cert, key, chain_file = self.find_cert() + cert, key, chain_file = self._find_cert() new_context = self.client_conn.create_ssl_context( cert, key, method=self.config.openssl_method_client, @@ -137,13 +138,13 @@ class TlsLayer(Layer): @yield_from_callback def _establish_tls_with_client(self): self.log("Establish TLS with client", "debug") - cert, key, chain_file = self.find_cert() + cert, key, chain_file = self._find_cert() try: self.client_conn.convert_to_ssl( cert, key, method=self.config.openssl_method_client, options=self.config.openssl_options_client, - handle_sni=self.handle_sni, + handle_sni=self.__handle_sni, cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, chain_file=chain_file @@ -182,7 +183,7 @@ class TlsLayer(Layer): except tcp.NetLibError as e: raise ProtocolException(repr(e), e) - def find_cert(self): + def _find_cert(self): host = self.server_conn.address.host sans = set() # Incorporate upstream certificate -- cgit v1.2.3 From 747699b126ab5788aca4541c9c9b4608611e7efa Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 14 Aug 2015 16:49:52 +0200 Subject: more work on http protocol --- libmproxy/protocol2/__init__.py | 3 +- libmproxy/protocol2/http.py | 144 +++++++++++++++++++++++++++--- libmproxy/protocol2/http_protocol_mock.py | 40 ++++++++- libmproxy/protocol2/layer.py | 5 +- libmproxy/protocol2/messages.py | 6 ++ libmproxy/proxy/server.py | 4 + 6 files changed, 187 insertions(+), 15 deletions(-) diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index d5dafaae..cf6032da 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -4,7 +4,8 @@ from .socks_proxy import Socks5Proxy from .reverse_proxy import ReverseProxy from .http_proxy import HttpProxy, HttpUpstreamProxy from .rawtcp import RawTcpLayer +from . import messages __all__ = [ - "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" + "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "messages" ] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 44ebf6a8..1e774648 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -3,7 +3,8 @@ from __future__ import (absolute_import, print_function, division) from .. import version from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer, ServerConnectionMixin -from .messages import ChangeServer, Connect, Reconnect +from libmproxy import utils +from .messages import ChangeServer, Connect, Reconnect, Kill from .http_proxy import HttpProxy, HttpUpstreamProxy from libmproxy.protocol import KILL @@ -12,7 +13,8 @@ from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from libmproxy.protocol2.http_protocol_mock import HTTP1 from libmproxy.protocol2.tls import TlsLayer from netlib import tcp -from netlib.http import status_codes +from netlib.http import status_codes, http1 +from netlib.http.semantics import CONTENT_MISSING from netlib import odict @@ -42,12 +44,14 @@ def make_error_response(status_code, message, headers=None): body, ) + def make_connect_request(address): return HTTPRequest( - "authority", "CONNECT", None, address.host, address.port, None, (1,1), + "authority", "CONNECT", None, address.host, address.port, None, (1, 1), odict.ODictCaseless(), "" ) + def make_connect_response(httpversion): headers = odict.ODictCaseless([ ["Content-Length", "0"], @@ -82,14 +86,14 @@ class HttpLayer(Layer, ServerConnectionMixin): try: request = HTTP1.read_request( self.client_conn, - body_size_limit=self.c.config.body_size_limit + body_size_limit=self.config.body_size_limit ) except tcp.NetLibError: # don't throw an error for disconnects that happen # before/between requests. return - self.c.log("request", "debug", [repr(request)]) + self.log("request", "debug", [repr(request)]) # Handle Proxy Authentication self.authenticate(request) @@ -109,12 +113,128 @@ class HttpLayer(Layer, ServerConnectionMixin): flow = HTTPFlow(self.client_conn, self.server_conn) flow.request = request - if not self.process_request_hook(flow): - self.log("Connection killed", "info") - return + for message in self.process_request_hook(flow): + yield message if not flow.response: - self.establish_server_connection(flow) + for message in self.establish_server_connection(flow): + yield message + for message in self.get_response_from_server(flow): + yield message + + self.send_response_to_client(flow) + + if self.check_close_connection(flow): + return + + if flow.request.form_in == "authority" and flow.response.code == 200: + raise NotImplementedError("Upstream mode CONNECT not implemented") + + def check_close_connection(self, flow): + """ + Checks if the connection should be closed depending on the HTTP + semantics. Returns True, if so. + """ + + # TODO: add logic for HTTP/2 + + close_connection = ( + http1.HTTP1Protocol.connection_close( + flow.request.httpversion, + flow.request.headers + ) or http1.HTTP1Protocol.connection_close( + flow.response.httpversion, + flow.response.headers + ) or http1.HTTP1Protocol.expected_http_body_size( + flow.response.headers, + False, + flow.request.method, + flow.response.code) == -1 + ) + if flow.request.form_in == "authority" and flow.response.code == 200: + # Workaround for + # https://github.com/mitmproxy/mitmproxy/issues/313: Some + # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 + # and no Content-Length header + + return False + return close_connection + + def send_response_to_client(self, flow): + if not flow.response.stream: + # no streaming: + # we already received the full response from the server and can + # send it to the client straight away. + self.send_to_client(flow.response) + else: + # streaming: + # First send the headers and then transfer the response + # incrementally: + h = HTTP1._assemble_response_first_line(flow.response) + self.send_to_client(h + "\r\n") + h = HTTP1._assemble_response_headers(flow.response, preserve_transfer_encoding=True) + self.send_to_client(h + "\r\n") + + chunks = HTTP1.read_http_body_chunked( + flow.response.headers, + self.config.body_size_limit, + flow.request.method, + flow.response.code, + False, + 4096 + ) + + if callable(flow.response.stream): + chunks = flow.response.stream(chunks) + + for chunk in chunks: + for part in chunk: + self.send_to_client(part) + self.client_conn.wfile.flush() + + flow.response.timestamp_end = utils.timestamp() + + def get_response_from_server(self, flow): + + self.send_to_server(flow.request) + + flow.response = HTTP1.read_response( + self.server_conn.protocol, + flow.request.method, + body_size_limit=self.config.body_size_limit, + include_body=False, + ) + + # call the appropriate script hook - this is an opportunity for an + # inline script to set flow.stream = True + flow = self.channel.ask("responseheaders", flow) + if flow is None or flow == KILL: + yield Kill() + + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = HTTP1.read_http_body( + flow.response.headers, + self.config.body_size_limit, + flow.request.method, + flow.response.code, + False + ) + flow.response.timestamp_end = utils.timestamp() + + # no further manipulation of self.server_conn beyond this point + # we can safely set it as the final attribute value here. + flow.server_conn = self.server_conn + + self.log( + "response", + "debug", + [repr(flow.response)] + ) + response_reply = self.channel.ask("response", flow) + if response_reply is None or response_reply == KILL: + yield Kill() def process_request_hook(self, flow): # Determine .scheme, .host and .port attributes for inline scripts. @@ -133,9 +253,9 @@ class HttpLayer(Layer, ServerConnectionMixin): flow.request.scheme = self.server_conn.tls_established # TODO: Expose ChangeServer functionality to inline scripts somehow? (yield_from_callback?) - request_reply = self.c.channel.ask("request", flow) + request_reply = self.channel.ask("request", flow) if request_reply is None or request_reply == KILL: - return False + yield Kill() if isinstance(request_reply, HTTPResponse): flow.response = request_reply return @@ -181,7 +301,7 @@ class HttpLayer(Layer, ServerConnectionMixin): raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { - "regular": ("absolute",), # an authority request would already be handled. + "regular": ("absolute",), # an authority request would already be handled. "upstream": ("authority", "absolute"), "transparent": ("regular",) } diff --git a/libmproxy/protocol2/http_protocol_mock.py b/libmproxy/protocol2/http_protocol_mock.py index 962a76d6..5fdb9f2b 100644 --- a/libmproxy/protocol2/http_protocol_mock.py +++ b/libmproxy/protocol2/http_protocol_mock.py @@ -10,4 +10,42 @@ class HTTP1(object): """ :type connection: object """ - return HTTP1Protocol(connection).read_request(*args, **kwargs) \ No newline at end of file + return HTTP1Protocol(connection).read_request(*args, **kwargs) + + @staticmethod + def read_response(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection).read_response(*args, **kwargs) + + @staticmethod + def read_http_body(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection).read_http_body(*args, **kwargs) + + + @staticmethod + def _assemble_response_first_line(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection)._assemble_response_first_line(*args, **kwargs) + + + @staticmethod + def _assemble_response_headers(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection)._assemble_response_headers(*args, **kwargs) + + + @staticmethod + def read_http_body_chunked(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection).read_http_body_chunked(*args, **kwargs) \ No newline at end of file diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index e9f5c667..2775845e 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -35,7 +35,7 @@ import threading from netlib import tcp from ..proxy import Log from ..proxy.connection import ServerConnection -from .messages import Connect, Reconnect, ChangeServer +from .messages import Connect, Reconnect, ChangeServer, Kill from ..exceptions import ProtocolException @@ -116,6 +116,9 @@ class ServerConnectionMixin(object): return True elif message == ChangeServer: raise NotImplementedError + elif message == Kill: + self._disconnect() + return False @property diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index 3f53fbd4..f6b584a1 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -41,3 +41,9 @@ class ChangeServer(_Message): # We can express this neatly as the "nth-server-providing-layer" # ServerConnection could get a `via` attribute. self.depth = depth + + +class Kill(_Message): + """ + Kill a connection. + """ \ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 6a7048e0..defcd464 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -84,6 +84,10 @@ class ConnectionHandler2: try: for message in root_layer(): + if message == protocol2.messages.Kill: + self.log("Connection killed", "info") + break + print("Root layer receveived: %s" % message) except ProtocolException as e: self.log(e, "info") -- cgit v1.2.3 From 0dd243c5e42950de9c8b1193ba9dbdd2d0414a45 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 15 Aug 2015 16:26:12 +0200 Subject: various fixes --- libmproxy/protocol2/http.py | 22 +++++++++++++++------- libmproxy/protocol2/http_protocol_mock.py | 25 ++++++++++++------------- libmproxy/protocol2/layer.py | 11 ++++++++--- libmproxy/proxy/server.py | 2 +- 4 files changed, 36 insertions(+), 24 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 1e774648..7adeb419 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -5,7 +5,6 @@ from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer, ServerConnectionMixin from libmproxy import utils from .messages import ChangeServer, Connect, Reconnect, Kill -from .http_proxy import HttpProxy, HttpUpstreamProxy from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow @@ -66,13 +65,18 @@ def make_connect_response(httpversion): ) -class HttpLayer(Layer, ServerConnectionMixin): +class HttpLayer(Layer): + """ HTTP 1 Layer """ def __init__(self, ctx): super(HttpLayer, self).__init__(ctx) + + # FIXME: Imports + from .http_proxy import HttpProxy, HttpUpstreamProxy + if any(isinstance(l, HttpProxy) for l in self.layers): self.mode = "regular" elif any(isinstance(l, HttpUpstreamProxy) for l in self.layers): @@ -199,7 +203,7 @@ class HttpLayer(Layer, ServerConnectionMixin): self.send_to_server(flow.request) flow.response = HTTP1.read_response( - self.server_conn.protocol, + self.server_conn, flow.request.method, body_size_limit=self.config.body_size_limit, include_body=False, @@ -215,6 +219,7 @@ class HttpLayer(Layer, ServerConnectionMixin): flow.response.content = CONTENT_MISSING else: flow.response.content = HTTP1.read_http_body( + self.server_conn, flow.response.headers, self.config.body_size_limit, flow.request.method, @@ -303,7 +308,7 @@ class HttpLayer(Layer, ServerConnectionMixin): expected_request_forms = { "regular": ("absolute",), # an authority request would already be handled. "upstream": ("authority", "absolute"), - "transparent": ("regular",) + "transparent": ("relative",) } allowed_request_forms = expected_request_forms[self.mode] @@ -314,6 +319,9 @@ class HttpLayer(Layer, ServerConnectionMixin): self.send_to_client(make_error_response(400, err_message)) raise HttpException(err_message) + if self.mode == "regular": + request.form_out = "relative" + def authenticate(self, request): if self.config.authenticator: if self.config.authenticator.authenticate(request.headers): @@ -327,10 +335,10 @@ class HttpLayer(Layer, ServerConnectionMixin): raise InvalidCredentials("Proxy Authentication Required") def send_to_server(self, message): - self.server_conn.wfile.wrie(message) + self.server_conn.send(HTTP1.assemble(message)) + def send_to_client(self, message): # FIXME # - possibly do some http2 stuff here - # - fix message assembly. - self.client_conn.wfile.write(message) + self.client_conn.send(HTTP1.assemble(message)) diff --git a/libmproxy/protocol2/http_protocol_mock.py b/libmproxy/protocol2/http_protocol_mock.py index 5fdb9f2b..22f3dc14 100644 --- a/libmproxy/protocol2/http_protocol_mock.py +++ b/libmproxy/protocol2/http_protocol_mock.py @@ -1,6 +1,7 @@ """ Temporary mock to sort out API discrepancies """ +from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from netlib.http.http1 import HTTP1Protocol @@ -10,14 +11,14 @@ class HTTP1(object): """ :type connection: object """ - return HTTP1Protocol(connection).read_request(*args, **kwargs) + return HTTPRequest.wrap(HTTP1Protocol(connection).read_request(*args, **kwargs)) @staticmethod def read_response(connection, *args, **kwargs): """ :type connection: object """ - return HTTP1Protocol(connection).read_response(*args, **kwargs) + return HTTPResponse.wrap(HTTP1Protocol(connection).read_response(*args, **kwargs)) @staticmethod def read_http_body(connection, *args, **kwargs): @@ -28,19 +29,13 @@ class HTTP1(object): @staticmethod - def _assemble_response_first_line(connection, *args, **kwargs): - """ - :type connection: object - """ - return HTTP1Protocol(connection)._assemble_response_first_line(*args, **kwargs) + def _assemble_response_first_line(*args, **kwargs): + return HTTP1Protocol()._assemble_response_first_line(*args, **kwargs) @staticmethod - def _assemble_response_headers(connection, *args, **kwargs): - """ - :type connection: object - """ - return HTTP1Protocol(connection)._assemble_response_headers(*args, **kwargs) + def _assemble_response_headers(*args, **kwargs): + return HTTP1Protocol()._assemble_response_headers(*args, **kwargs) @staticmethod @@ -48,4 +43,8 @@ class HTTP1(object): """ :type connection: object """ - return HTTP1Protocol(connection).read_http_body_chunked(*args, **kwargs) \ No newline at end of file + return HTTP1Protocol(connection).read_http_body_chunked(*args, **kwargs) + + @staticmethod + def assemble(*args, **kwargs): + return HTTP1Protocol().assemble(*args, **kwargs) \ No newline at end of file diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 2775845e..f2d6b3fb 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -45,6 +45,7 @@ class _LayerCodeCompletion(object): """ def __init__(self): + super(_LayerCodeCompletion, self).__init__() if True: return self.config = None @@ -94,7 +95,7 @@ class Layer(_LayerCodeCompletion): return [self] + self.ctx.layers def __repr__(self): - return "%s\r\n %s" % (self.__class__.name__, repr(self.ctx)) + return type(self).__name__ class ServerConnectionMixin(object): @@ -103,6 +104,7 @@ class ServerConnectionMixin(object): """ def __init__(self): + super(ServerConnectionMixin, self).__init__() self._server_address = None self.server_conn = None @@ -114,8 +116,11 @@ class ServerConnectionMixin(object): elif message == Connect: self._connect() return True - elif message == ChangeServer: - raise NotImplementedError + elif message == ChangeServer and message.depth == 1: + if self.server_conn: + self._disconnect() + self.server_address = message.address + return True elif message == Kill: self._disconnect() diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index defcd464..ffca55ee 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -80,7 +80,7 @@ class ConnectionHandler2: self.config, self.channel ) - root_layer = protocol2.Socks5Proxy(root_context) + root_layer = protocol2.HttpProxy(root_context) try: for message in root_layer(): -- cgit v1.2.3 From a9dd82c986be54d82f6ce9c7b65473f2b052cbe8 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 15 Aug 2015 17:43:46 +0200 Subject: add ALPN to proxy connections --- libmproxy/protocol2/http_proxy.py | 3 ++- libmproxy/protocol2/tls.py | 8 +++++++- libmproxy/proxy/connection.py | 21 ++++++++++----------- 3 files changed, 19 insertions(+), 13 deletions(-) diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 51d3763c..b85a65eb 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -1,7 +1,6 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin -from .http import HttpLayer class HttpProxy(Layer, ServerConnectionMixin): @@ -22,3 +21,5 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + +from .http import HttpLayer diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 988304aa..9572912f 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -1,7 +1,9 @@ from __future__ import (absolute_import, print_function, division) import traceback + from netlib import tcp +import netlib.http.http2 from ..exceptions import ProtocolException from .layer import Layer, yield_from_callback @@ -147,7 +149,8 @@ class TlsLayer(Layer): handle_sni=self.__handle_sni, cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, - chain_file=chain_file + chain_file=chain_file, + alpn_select=netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2, # TODO: check if server is capable of h2 first ) except tcp.NetLibError as e: raise ProtocolException(repr(e), e) @@ -164,6 +167,9 @@ class TlsLayer(Layer): ca_path=self.config.openssl_trusted_cadir_server, ca_pemfile=self.config.openssl_trusted_ca_server, cipher_list=self.config.ciphers_server, + alpn_protos=[ + netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1, + netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], # TODO: read this from client_conn first ) tls_cert_err = self.server_conn.ssl_verification_error if tls_cert_err is not None: diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index 49210e47..d2b956f3 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -1,6 +1,8 @@ from __future__ import absolute_import + import copy import os + from netlib import tcp, certutils from .. import stateobject, utils @@ -75,14 +77,14 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): return f def convert_to_ssl(self, *args, **kwargs): - # TODO: read ALPN from server and select same proto for client conn - # alpn_select = 'h2' - # def alpn_select_callback(conn_, options): - # if alpn_select in options: - # return bytes(alpn_select) - # else: # pragma no cover - # return options[0] - # tcp.BaseHandler.convert_to_ssl(self, alpn_select=alpn_select_callback, *args, **kwargs) + if 'alpn_select' in kwargs: + alpn_select = kwargs['alpn_select'] + def alpn_select_callback(conn_, options): + if alpn_select in options: + return bytes(alpn_select) + else: # pragma no cover + return options[0] + kwargs['alpn_select'] = alpn_select_callback tcp.BaseHandler.convert_to_ssl(self, *args, **kwargs) self.timestamp_ssl_setup = utils.timestamp() @@ -184,9 +186,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): if os.path.exists(path): clientcert = path - # TODO: read ALPN from client and use same list for server conn - # self.convert_to_ssl(cert=clientcert, sni=sni, alpn_protos=[netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], **kwargs) - self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) self.sni = sni self.timestamp_ssl_setup = utils.timestamp() -- cgit v1.2.3 From 2a15479cdbda07a4a99f56f6090e479decbeb17c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 15 Aug 2015 20:20:46 +0200 Subject: fix bugs, make https work --- libmproxy/protocol2/http.py | 24 +++++++----------------- libmproxy/protocol2/http_proxy.py | 4 ++-- libmproxy/protocol2/layer.py | 4 ++-- libmproxy/protocol2/messages.py | 2 +- libmproxy/protocol2/root_context.py | 10 ++++++---- libmproxy/protocol2/tls.py | 34 +++++++++++++++++++--------------- 6 files changed, 37 insertions(+), 41 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 7adeb419..f629a6b0 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -4,7 +4,7 @@ from .. import version from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer, ServerConnectionMixin from libmproxy import utils -from .messages import ChangeServer, Connect, Reconnect, Kill +from .messages import SetServer, Connect, Reconnect, Kill from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow @@ -71,19 +71,9 @@ class HttpLayer(Layer): HTTP 1 Layer """ - def __init__(self, ctx): + def __init__(self, ctx, mode): super(HttpLayer, self).__init__(ctx) - - # FIXME: Imports - from .http_proxy import HttpProxy, HttpUpstreamProxy - - if any(isinstance(l, HttpProxy) for l in self.layers): - self.mode = "regular" - elif any(isinstance(l, HttpUpstreamProxy) for l in self.layers): - self.mode = "upstream" - else: - # also includes socks or reverse mode, which are handled similarly on this layer. - self.mode = "transparent" + self.mode = mode def __call__(self): while True: @@ -104,7 +94,7 @@ class HttpLayer(Layer): # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.form_in == "authority": - self.server_address = (request.host, request.port) + yield SetServer((request.host, request.port), False, None) self.send_to_client(make_connect_response(request.httpversion)) layer = self.ctx.next_layer(self) for message in layer(): @@ -255,7 +245,7 @@ class HttpLayer(Layer): else: flow.request.host = self.ctx.server_address.host flow.request.port = self.ctx.server_address.port - flow.request.scheme = self.server_conn.tls_established + flow.request.scheme = "https" if self.server_conn.tls_established else "http" # TODO: Expose ChangeServer functionality to inline scripts somehow? (yield_from_callback?) request_reply = self.channel.ask("request", flow) @@ -271,8 +261,8 @@ class HttpLayer(Layer): tls = (flow.request.scheme == "https") if self.mode == "regular" or self.mode == "transparent": # If there's an existing connection that doesn't match our expectations, kill it. - if self.server_address != address or tls != self.server_address.ssl_established: - yield ChangeServer(address, tls, address.host) + if self.server_address != address or tls != self.server_conn.ssl_established: + yield SetServer(address, tls, address.host) # Establish connection is neccessary. if not self.server_conn: yield Connect() diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 51d3763c..8ac7ea8e 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -6,7 +6,7 @@ from .http import HttpLayer class HttpProxy(Layer, ServerConnectionMixin): def __call__(self): - layer = HttpLayer(self) + layer = HttpLayer(self, "regular") for message in layer(): if not self._handle_server_message(message): yield message @@ -18,7 +18,7 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): self.server_address = server_address def __call__(self): - layer = HttpLayer(self) + layer = HttpLayer(self, "upstream") for message in layer(): if not self._handle_server_message(message): yield message diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index f2d6b3fb..8e985d4d 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -35,7 +35,7 @@ import threading from netlib import tcp from ..proxy import Log from ..proxy.connection import ServerConnection -from .messages import Connect, Reconnect, ChangeServer, Kill +from .messages import Connect, Reconnect, SetServer, Kill from ..exceptions import ProtocolException @@ -116,7 +116,7 @@ class ServerConnectionMixin(object): elif message == Connect: self._connect() return True - elif message == ChangeServer and message.depth == 1: + elif message == SetServer and message.depth == 1: if self.server_conn: self._disconnect() self.server_address = message.address diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index f6b584a1..17e12f11 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -27,7 +27,7 @@ class Reconnect(_Message): """ -class ChangeServer(_Message): +class SetServer(_Message): """ Change the upstream server. """ diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 3b341778..bda8b12b 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -2,7 +2,7 @@ from __future__ import (absolute_import, print_function, division) from .rawtcp import RawTcpLayer from .tls import TlsLayer - +from .http import HttpLayer class RootContext(object): """ @@ -38,10 +38,12 @@ class RootContext(object): return if is_tls_client_hello: - layer = TlsLayer(top_layer, True, True) + return TlsLayer(top_layer, True, True) + elif isinstance(top_layer, TlsLayer) and isinstance(top_layer.ctx, HttpLayer): + return HttpLayer(top_layer, "transparent") else: - layer = RawTcpLayer(top_layer) - return layer + return RawTcpLayer(top_layer) + @property def layers(self): diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 988304aa..55cc9794 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -5,7 +5,7 @@ from netlib import tcp from ..exceptions import ProtocolException from .layer import Layer, yield_from_callback -from .messages import Connect, Reconnect, ChangeServer +from .messages import Connect, Reconnect, SetServer class TlsLayer(Layer): @@ -13,7 +13,6 @@ class TlsLayer(Layer): super(TlsLayer, self).__init__(ctx) self._client_tls = client_tls self._server_tls = server_tls - self._connected = False self.client_sni = None self._sni_from_server_change = None @@ -44,9 +43,6 @@ class TlsLayer(Layer): client_tls_requires_server_cert = ( self._client_tls and self._server_tls and not self.config.no_upstream_cert ) - lazy_server_tls = ( - self._server_tls and not client_tls_requires_server_cert - ) if client_tls_requires_server_cert: for m in self._establish_tls_with_client_and_server(): @@ -56,18 +52,27 @@ class TlsLayer(Layer): yield m layer = self.ctx.next_layer(self) + for message in layer(): - if message != Connect or not self._connected: + self.log("TlsLayer: %s" % message,"debug") + if not (message == Connect and self._connected): yield message - if message == Connect: - if lazy_server_tls: - self._establish_tls_with_server() - if message == ChangeServer and message.depth == 1: - self._server_tls = message.server_tls - self._sni_from_server_change = message.sni - if message == Reconnect or message == ChangeServer: - if self._server_tls: + + if message == Connect or message == Reconnect: + if self._server_tls and not self._server_tls_established: self._establish_tls_with_server() + if message == SetServer and message.depth == 1: + if message.server_tls is not None: + self._sni_from_server_change = message.sni + self._server_tls = message.server_tls + + @property + def _server_tls_established(self): + return self.server_conn and self.server_conn.tls_established + + @property + def _connected(self): + return bool(self.server_conn) @property def sni_for_upstream_connection(self): @@ -83,7 +88,6 @@ class TlsLayer(Layer): # First, try to connect to the server. yield Connect() - self._connected = True server_err = None try: self._establish_tls_with_server() -- cgit v1.2.3 From 1e40d34e942382bbb11234e0e9232794b3bf6acf Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 15 Aug 2015 17:43:46 +0200 Subject: add ALPN to proxy connections --- libmproxy/protocol2/http_proxy.py | 3 ++- libmproxy/protocol2/layer.py | 2 +- libmproxy/protocol2/tls.py | 33 +++++++++++++++++++++++++++++++-- libmproxy/proxy/connection.py | 14 ++------------ 4 files changed, 36 insertions(+), 16 deletions(-) diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 8ac7ea8e..b4c506cb 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -1,7 +1,6 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin -from .http import HttpLayer class HttpProxy(Layer, ServerConnectionMixin): @@ -22,3 +21,5 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + +from .http import HttpLayer diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 8e985d4d..de519baa 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -208,4 +208,4 @@ def yield_from_callback(fun): self.yield_from_callback = None - return wrapper \ No newline at end of file + return wrapper diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 55cc9794..fcc12f18 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -1,7 +1,9 @@ from __future__ import (absolute_import, print_function, division) import traceback + from netlib import tcp +import netlib.http.http2 from ..exceptions import ProtocolException from .layer import Layer, yield_from_callback @@ -15,6 +17,9 @@ class TlsLayer(Layer): self._server_tls = server_tls self.client_sni = None self._sni_from_server_change = None + self.client_alpn_protos = None + + # foo alpn protos = [netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1, netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], # TODO: read this from client_conn first def __call__(self): """ @@ -131,7 +136,8 @@ class TlsLayer(Layer): options=self.config.openssl_options_client, cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, - chain_file=chain_file + chain_file=chain_file, + alpn_select_callback=self.__handle_alpn_select, ) connection.set_context(new_context) # An unhandled exception in this method will core dump PyOpenSSL, so @@ -139,10 +145,30 @@ class TlsLayer(Layer): except: # pragma: no cover self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") + def __handle_alpn_select(self, conn_, options): + # TODO: change to something meaningful? + alpn_preference = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 + alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 + ### + + if self.client_alpn_protos != options: + # Perform reconnect + if self._server_tls: + self.yield_from_callback(Reconnect()) + + self.client_alpn_protos = options + print("foo: %s" % options) + + if alpn_preference in options: + return bytes(alpn_preference) + else: # pragma no cover + return options[0] + @yield_from_callback def _establish_tls_with_client(self): self.log("Establish TLS with client", "debug") cert, key, chain_file = self._find_cert() + try: self.client_conn.convert_to_ssl( cert, key, @@ -151,9 +177,11 @@ class TlsLayer(Layer): handle_sni=self.__handle_sni, cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, - chain_file=chain_file + chain_file=chain_file, + alpn_select_callback=self.__handle_alpn_select, ) except tcp.NetLibError as e: + print("alpn: %s" % self.client_alpn_protos) raise ProtocolException(repr(e), e) def _establish_tls_with_server(self): @@ -168,6 +196,7 @@ class TlsLayer(Layer): ca_path=self.config.openssl_trusted_cadir_server, ca_pemfile=self.config.openssl_trusted_ca_server, cipher_list=self.config.ciphers_server, + alpn_protos=self.client_alpn_protos, ) tls_cert_err = self.server_conn.ssl_verification_error if tls_cert_err is not None: diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index 49210e47..f33e84cd 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -1,6 +1,8 @@ from __future__ import absolute_import + import copy import os + from netlib import tcp, certutils from .. import stateobject, utils @@ -75,15 +77,6 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): return f def convert_to_ssl(self, *args, **kwargs): - # TODO: read ALPN from server and select same proto for client conn - # alpn_select = 'h2' - # def alpn_select_callback(conn_, options): - # if alpn_select in options: - # return bytes(alpn_select) - # else: # pragma no cover - # return options[0] - # tcp.BaseHandler.convert_to_ssl(self, alpn_select=alpn_select_callback, *args, **kwargs) - tcp.BaseHandler.convert_to_ssl(self, *args, **kwargs) self.timestamp_ssl_setup = utils.timestamp() @@ -184,9 +177,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): if os.path.exists(path): clientcert = path - # TODO: read ALPN from client and use same list for server conn - # self.convert_to_ssl(cert=clientcert, sni=sni, alpn_protos=[netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], **kwargs) - self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) self.sni = sni self.timestamp_ssl_setup = utils.timestamp() -- cgit v1.2.3 From 4c31ffd90fcc273f798b9a5be96c811fbedb5e2e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 16 Aug 2015 12:43:15 +0200 Subject: minor fixes --- libmproxy/protocol2/http.py | 95 ++++++++++++++++++++------------------- libmproxy/protocol2/http_proxy.py | 5 +-- libmproxy/protocol2/layer.py | 1 - 3 files changed, 51 insertions(+), 50 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index f629a6b0..7cc27652 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -12,9 +12,10 @@ from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from libmproxy.protocol2.http_protocol_mock import HTTP1 from libmproxy.protocol2.tls import TlsLayer from netlib import tcp -from netlib.http import status_codes, http1 +from netlib.http import status_codes, http1, HttpErrorConnClosed from netlib.http.semantics import CONTENT_MISSING from netlib import odict +from netlib.tcp import NetLibError def make_error_response(status_code, message, headers=None): @@ -66,7 +67,6 @@ def make_connect_response(httpversion): class HttpLayer(Layer): - """ HTTP 1 Layer """ @@ -78,51 +78,55 @@ class HttpLayer(Layer): def __call__(self): while True: try: - request = HTTP1.read_request( - self.client_conn, - body_size_limit=self.config.body_size_limit - ) - except tcp.NetLibError: - # don't throw an error for disconnects that happen - # before/between requests. - return - - self.log("request", "debug", [repr(request)]) - - # Handle Proxy Authentication - self.authenticate(request) - - # Regular Proxy Mode: Handle CONNECT - if self.mode == "regular" and request.form_in == "authority": - yield SetServer((request.host, request.port), False, None) - self.send_to_client(make_connect_response(request.httpversion)) - layer = self.ctx.next_layer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message - return - - # Make sure that the incoming request matches our expectations - self.validate_request(request) - - flow = HTTPFlow(self.client_conn, self.server_conn) - flow.request = request - for message in self.process_request_hook(flow): - yield message - - if not flow.response: - for message in self.establish_server_connection(flow): - yield message - for message in self.get_response_from_server(flow): + try: + request = HTTP1.read_request( + self.client_conn, + body_size_limit=self.config.body_size_limit + ) + except tcp.NetLibError: + # don't throw an error for disconnects that happen + # before/between requests. + return + + self.log("request", "debug", [repr(request)]) + + # Handle Proxy Authentication + self.authenticate(request) + + # Regular Proxy Mode: Handle CONNECT + if self.mode == "regular" and request.form_in == "authority": + yield SetServer((request.host, request.port), False, None) + self.send_to_client(make_connect_response(request.httpversion)) + layer = self.ctx.next_layer(self) + for message in layer(): + if not self._handle_server_message(message): + yield message + return + + # Make sure that the incoming request matches our expectations + self.validate_request(request) + + flow = HTTPFlow(self.client_conn, self.server_conn) + flow.request = request + for message in self.process_request_hook(flow): yield message - self.send_response_to_client(flow) + if not flow.response: + for message in self.establish_server_connection(flow): + yield message + for message in self.get_response_from_server(flow): + yield message + + self.send_response_to_client(flow) - if self.check_close_connection(flow): - return + if self.check_close_connection(flow): + return - if flow.request.form_in == "authority" and flow.response.code == 200: - raise NotImplementedError("Upstream mode CONNECT not implemented") + if flow.request.form_in == "authority" and flow.response.code == 200: + raise NotImplementedError("Upstream mode CONNECT not implemented") + except (HttpErrorConnClosed, NetLibError) as e: + make_error_response(502, repr(e)) + raise ProtocolException(repr(e), e) def check_close_connection(self, flow): """ @@ -144,7 +148,7 @@ class HttpLayer(Layer): False, flow.request.method, flow.response.code) == -1 - ) + ) if flow.request.form_in == "authority" and flow.response.code == 200: # Workaround for # https://github.com/mitmproxy/mitmproxy/issues/313: Some @@ -189,7 +193,7 @@ class HttpLayer(Layer): flow.response.timestamp_end = utils.timestamp() def get_response_from_server(self, flow): - + # TODO: Add second attempt. self.send_to_server(flow.request) flow.response = HTTP1.read_response( @@ -327,7 +331,6 @@ class HttpLayer(Layer): def send_to_server(self, message): self.server_conn.send(HTTP1.assemble(message)) - def send_to_client(self, message): # FIXME # - possibly do some http2 stuff here diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index b4c506cb..ca70b012 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -1,6 +1,7 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin +from .http import HttpLayer class HttpProxy(Layer, ServerConnectionMixin): @@ -20,6 +21,4 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): layer = HttpLayer(self, "upstream") for message in layer(): if not self._handle_server_message(message): - yield message - -from .http import HttpLayer + yield message \ No newline at end of file diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 8e985d4d..ca297c0e 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -200,7 +200,6 @@ def yield_from_callback(fun): if msg is True: break elif isinstance(msg, Exception): - # TODO: Include func name? raise ProtocolException("Error in %s: %s" % (fun.__name__, repr(msg)), msg) else: yield msg -- cgit v1.2.3 From c04fa1b233224d28e85be34ab5b6a8718497488c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 16 Aug 2015 12:52:34 +0200 Subject: minor fixes --- libmproxy/protocol2/http_protocol_mock.py | 4 ++-- libmproxy/protocol2/http_proxy.py | 1 + 2 files changed, 3 insertions(+), 2 deletions(-) diff --git a/libmproxy/protocol2/http_protocol_mock.py b/libmproxy/protocol2/http_protocol_mock.py index 22f3dc14..dd3643f6 100644 --- a/libmproxy/protocol2/http_protocol_mock.py +++ b/libmproxy/protocol2/http_protocol_mock.py @@ -11,14 +11,14 @@ class HTTP1(object): """ :type connection: object """ - return HTTPRequest.wrap(HTTP1Protocol(connection).read_request(*args, **kwargs)) + return HTTPRequest.from_protocol(HTTP1Protocol(connection), *args, **kwargs) @staticmethod def read_response(connection, *args, **kwargs): """ :type connection: object """ - return HTTPResponse.wrap(HTTP1Protocol(connection).read_response(*args, **kwargs)) + return HTTPResponse.from_protocol(HTTP1Protocol(connection), *args, **kwargs) @staticmethod def read_http_body(connection, *args, **kwargs): diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 9488e367..ca70b012 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -1,6 +1,7 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin +from .http import HttpLayer class HttpProxy(Layer, ServerConnectionMixin): -- cgit v1.2.3 From 38c456bb627c4570e0ed983229ec8ef2f120a4b6 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 16 Aug 2015 15:19:11 +0200 Subject: implement Http1 and Http2 protocols as layers --- libmproxy/protocol2/http.py | 51 +++++++++++++++++++++++-------- libmproxy/protocol2/http_protocol_mock.py | 50 ------------------------------ libmproxy/protocol2/http_proxy.py | 8 ++--- libmproxy/protocol2/layer.py | 1 + libmproxy/protocol2/root_context.py | 13 ++++++-- libmproxy/protocol2/tls.py | 1 - 6 files changed, 54 insertions(+), 70 deletions(-) delete mode 100644 libmproxy/protocol2/http_protocol_mock.py diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 7cc27652..cabec806 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -9,15 +9,42 @@ from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest -from libmproxy.protocol2.http_protocol_mock import HTTP1 from libmproxy.protocol2.tls import TlsLayer from netlib import tcp from netlib.http import status_codes, http1, HttpErrorConnClosed from netlib.http.semantics import CONTENT_MISSING from netlib import odict from netlib.tcp import NetLibError +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol + +class Http1Layer(Layer): + def __init__(self, ctx, mode): + super(Http1Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP1Protocol(self.client_conn) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def __call__(self): + from .http import HttpLayer + layer = HttpLayer(self, self.mode) + for message in layer(): + yield message +class Http2Layer(Layer): + def __init__(self, ctx, mode): + super(Http2Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + + def __call__(self): + from .http import HttpLayer + layer = HttpLayer(self, self.mode) + for message in layer(): + yield message + def make_error_response(status_code, message, headers=None): response = status_codes.RESPONSES.get(status_code, "Unknown") body = """ @@ -79,8 +106,8 @@ class HttpLayer(Layer): while True: try: try: - request = HTTP1.read_request( - self.client_conn, + request = HTTPRequest.from_protocol( + self.client_protocol, body_size_limit=self.config.body_size_limit ) except tcp.NetLibError: @@ -168,12 +195,12 @@ class HttpLayer(Layer): # streaming: # First send the headers and then transfer the response # incrementally: - h = HTTP1._assemble_response_first_line(flow.response) + h = self.client_protocol._assemble_response_first_line(flow.response) self.send_to_client(h + "\r\n") - h = HTTP1._assemble_response_headers(flow.response, preserve_transfer_encoding=True) + h = self.client_protocol._assemble_response_headers(flow.response, preserve_transfer_encoding=True) self.send_to_client(h + "\r\n") - chunks = HTTP1.read_http_body_chunked( + chunks = self.client_protocol.read_http_body_chunked( flow.response.headers, self.config.body_size_limit, flow.request.method, @@ -196,8 +223,8 @@ class HttpLayer(Layer): # TODO: Add second attempt. self.send_to_server(flow.request) - flow.response = HTTP1.read_response( - self.server_conn, + flow.response = HTTPResponse.from_protocol( + self.server_protocol, flow.request.method, body_size_limit=self.config.body_size_limit, include_body=False, @@ -211,8 +238,8 @@ class HttpLayer(Layer): if flow.response.stream: flow.response.content = CONTENT_MISSING - else: - flow.response.content = HTTP1.read_http_body( + elif isinstance(self.server_protocol, http1.HTTP1Protocol): + flow.response.content = self.server_protocol.read_http_body( self.server_conn, flow.response.headers, self.config.body_size_limit, @@ -329,9 +356,9 @@ class HttpLayer(Layer): raise InvalidCredentials("Proxy Authentication Required") def send_to_server(self, message): - self.server_conn.send(HTTP1.assemble(message)) + self.server_conn.send(self.server_protocol.assemble(message)) def send_to_client(self, message): # FIXME # - possibly do some http2 stuff here - self.client_conn.send(HTTP1.assemble(message)) + self.client_conn.send(self.client_protocol.assemble(message)) diff --git a/libmproxy/protocol2/http_protocol_mock.py b/libmproxy/protocol2/http_protocol_mock.py deleted file mode 100644 index dd3643f6..00000000 --- a/libmproxy/protocol2/http_protocol_mock.py +++ /dev/null @@ -1,50 +0,0 @@ -""" -Temporary mock to sort out API discrepancies -""" -from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest -from netlib.http.http1 import HTTP1Protocol - - -class HTTP1(object): - @staticmethod - def read_request(connection, *args, **kwargs): - """ - :type connection: object - """ - return HTTPRequest.from_protocol(HTTP1Protocol(connection), *args, **kwargs) - - @staticmethod - def read_response(connection, *args, **kwargs): - """ - :type connection: object - """ - return HTTPResponse.from_protocol(HTTP1Protocol(connection), *args, **kwargs) - - @staticmethod - def read_http_body(connection, *args, **kwargs): - """ - :type connection: object - """ - return HTTP1Protocol(connection).read_http_body(*args, **kwargs) - - - @staticmethod - def _assemble_response_first_line(*args, **kwargs): - return HTTP1Protocol()._assemble_response_first_line(*args, **kwargs) - - - @staticmethod - def _assemble_response_headers(*args, **kwargs): - return HTTP1Protocol()._assemble_response_headers(*args, **kwargs) - - - @staticmethod - def read_http_body_chunked(connection, *args, **kwargs): - """ - :type connection: object - """ - return HTTP1Protocol(connection).read_http_body_chunked(*args, **kwargs) - - @staticmethod - def assemble(*args, **kwargs): - return HTTP1Protocol().assemble(*args, **kwargs) \ No newline at end of file diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index ca70b012..7f5957ac 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -1,12 +1,12 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin -from .http import HttpLayer +from .http import Http1Layer, HttpLayer class HttpProxy(Layer, ServerConnectionMixin): def __call__(self): - layer = HttpLayer(self, "regular") + layer = Http1Layer(self, "regular") for message in layer(): if not self._handle_server_message(message): yield message @@ -18,7 +18,7 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): self.server_address = server_address def __call__(self): - layer = HttpLayer(self, "upstream") + layer = Http1Layer(self, "upstream") for message in layer(): if not self._handle_server_message(message): - yield message \ No newline at end of file + yield message diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index c1648a62..31b74552 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -64,6 +64,7 @@ class Layer(_LayerCodeCompletion): """ super(Layer, self).__init__() self.ctx = ctx + print("%s -> %s" % (repr(ctx), repr(self))) def __call__(self): """ diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index bda8b12b..a68560c2 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -2,7 +2,7 @@ from __future__ import (absolute_import, print_function, division) from .rawtcp import RawTcpLayer from .tls import TlsLayer -from .http import HttpLayer +from .http import Http1Layer, Http2Layer, HttpLayer class RootContext(object): """ @@ -34,13 +34,20 @@ class RootContext(object): d[2] in ('\x00', '\x01', '\x02', '\x03') ) + # TODO: build is_http2_magic check here, maybe this is an easy way to detect h2c + if not d: return if is_tls_client_hello: return TlsLayer(top_layer, True, True) - elif isinstance(top_layer, TlsLayer) and isinstance(top_layer.ctx, HttpLayer): - return HttpLayer(top_layer, "transparent") + elif isinstance(top_layer, TlsLayer): + if top_layer.client_conn.get_alpn_proto_negotiated() == 'h2': + return Http2Layer(top_layer, 'regular') # TODO: regular correct here? + else: + return Http1Layer(top_layer, 'regular') # TODO: regular correct here? + elif isinstance(top_layer, TlsLayer) and isinstance(top_layer.ctx, Http1Layer): + return Http1Layer(top_layer, "transparent") else: return RawTcpLayer(top_layer) diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index fcc12f18..8e367728 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -157,7 +157,6 @@ class TlsLayer(Layer): self.yield_from_callback(Reconnect()) self.client_alpn_protos = options - print("foo: %s" % options) if alpn_preference in options: return bytes(alpn_preference) -- cgit v1.2.3 From a2b85048892626e6834df06e9022498814724636 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 16 Aug 2015 23:25:02 +0200 Subject: improve protocol handling --- libmproxy/protocol2/http.py | 86 +++++++++++++++++++++++++------- libmproxy/protocol2/http_proxy.py | 3 +- libmproxy/protocol2/layer.py | 47 ++++++++--------- libmproxy/protocol2/messages.py | 3 +- libmproxy/protocol2/reverse_proxy.py | 3 +- libmproxy/protocol2/root_context.py | 5 +- libmproxy/protocol2/socks_proxy.py | 4 +- libmproxy/protocol2/tls.py | 3 +- libmproxy/protocol2/transparent_proxy.py | 2 +- libmproxy/proxy/connection.py | 7 ++- libmproxy/proxy/server.py | 11 ++-- 11 files changed, 114 insertions(+), 60 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 7cc27652..90784666 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -2,20 +2,20 @@ from __future__ import (absolute_import, print_function, division) from .. import version from ..exceptions import InvalidCredentials, HttpException, ProtocolException -from .layer import Layer, ServerConnectionMixin +from .layer import Layer from libmproxy import utils +from libmproxy.proxy.connection import ServerConnection from .messages import SetServer, Connect, Reconnect, Kill from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from libmproxy.protocol2.http_protocol_mock import HTTP1 -from libmproxy.protocol2.tls import TlsLayer from netlib import tcp from netlib.http import status_codes, http1, HttpErrorConnClosed from netlib.http.semantics import CONTENT_MISSING from netlib import odict -from netlib.tcp import NetLibError +from netlib.tcp import NetLibError, Address def make_error_response(status_code, message, headers=None): @@ -46,6 +46,7 @@ def make_error_response(status_code, message, headers=None): def make_connect_request(address): + address = Address.wrap(address) return HTTPRequest( "authority", "CONNECT", None, address.host, address.port, None, (1, 1), odict.ODictCaseless(), "" @@ -66,6 +67,22 @@ def make_connect_response(httpversion): ) +class ConnectServerConnection(object): + """ + "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. + """ + def __init__(self, address, ctx): + self.address = tcp.Address.wrap(address) + self._ctx = ctx + + @property + def via(self): + return self._ctx.server_conn + + def __getattr__(self, item): + return getattr(self.via, item) + + class HttpLayer(Layer): """ HTTP 1 Layer @@ -95,12 +112,8 @@ class HttpLayer(Layer): # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.form_in == "authority": - yield SetServer((request.host, request.port), False, None) - self.send_to_client(make_connect_response(request.httpversion)) - layer = self.ctx.next_layer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message + for message in self.handle_regular_mode_connect(request): + yield message return # Make sure that the incoming request matches our expectations @@ -122,12 +135,50 @@ class HttpLayer(Layer): if self.check_close_connection(flow): return + # Upstream Proxy Mode: Handle CONNECT if flow.request.form_in == "authority" and flow.response.code == 200: - raise NotImplementedError("Upstream mode CONNECT not implemented") + for message in self.handle_upstream_mode_connect(flow.request.copy()): + yield message + return + except (HttpErrorConnClosed, NetLibError) as e: make_error_response(502, repr(e)) raise ProtocolException(repr(e), e) + def handle_regular_mode_connect(self, request): + yield SetServer((request.host, request.port), False, None) + self.send_to_client(make_connect_response(request.httpversion)) + layer = self.ctx.next_layer(self) + for message in layer(): + yield message + + def handle_upstream_mode_connect(self, connect_request): + layer = self.ctx.next_layer(self) + self.server_conn = ConnectServerConnection((connect_request.host, connect_request.port), self.ctx) + + for message in layer(): + if message == Connect: + if not self.server_conn: + yield message + self.send_to_server(connect_request) + else: + pass # swallow the message + elif message == Reconnect: + yield message + self.send_to_server(connect_request) + elif message == SetServer: + if message.depth == 1: + if self.ctx.server_conn: + yield Reconnect() + connect_request.host = message.address.host + connect_request.port = message.address.port + self.server_conn.address = message.address + else: + message.depth -= 1 + yield message + else: + yield message + def check_close_connection(self, flow): """ Checks if the connection should be closed depending on the HTTP @@ -247,11 +298,11 @@ class HttpLayer(Layer): if flow.request.form_in == "authority": flow.request.scheme = "http" # pseudo value else: - flow.request.host = self.ctx.server_address.host - flow.request.port = self.ctx.server_address.port + flow.request.host = self.ctx.server_conn.address.host + flow.request.port = self.ctx.server_conn.address.port flow.request.scheme = "https" if self.server_conn.tls_established else "http" - # TODO: Expose ChangeServer functionality to inline scripts somehow? (yield_from_callback?) + # TODO: Expose SetServer functionality to inline scripts somehow? (yield_from_callback?) request_reply = self.channel.ask("request", flow) if request_reply is None or request_reply == KILL: yield Kill() @@ -265,25 +316,26 @@ class HttpLayer(Layer): tls = (flow.request.scheme == "https") if self.mode == "regular" or self.mode == "transparent": # If there's an existing connection that doesn't match our expectations, kill it. - if self.server_address != address or tls != self.server_conn.ssl_established: + if address != self.server_conn.address or tls != self.server_conn.ssl_established: yield SetServer(address, tls, address.host) # Establish connection is neccessary. if not self.server_conn: yield Connect() - # ChangeServer is not guaranteed to work with TLS: + # SetServer is not guaranteed to work with TLS: # If there's not TlsLayer below which could catch the exception, # TLS will not be established. if tls and not self.server_conn.tls_established: raise ProtocolException("Cannot upgrade to SSL, no TLS layer on the protocol stack.") else: + if not self.server_conn: + yield Connect() if tls: raise HttpException("Cannot change scheme in upstream proxy mode.") """ # This is a very ugly (untested) workaround to solve a very ugly problem. - # FIXME: Check if connected first. - if self.server_conn.tls_established and not ssl: + if self.server_conn and self.server_conn.tls_established and not ssl: yield Reconnect() elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: if self.server_conn.tls_established: diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index ca70b012..3cc7fee2 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -14,8 +14,7 @@ class HttpProxy(Layer, ServerConnectionMixin): class HttpUpstreamProxy(Layer, ServerConnectionMixin): def __init__(self, ctx, server_address): - super(HttpUpstreamProxy, self).__init__(ctx) - self.server_address = server_address + super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) def __call__(self): layer = HttpLayer(self, "upstream") diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index c1648a62..67f3d549 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -44,8 +44,8 @@ class _LayerCodeCompletion(object): Dummy class that provides type hinting in PyCharm, which simplifies development a lot. """ - def __init__(self): - super(_LayerCodeCompletion, self).__init__() + def __init__(self, *args, **kwargs): + super(_LayerCodeCompletion, self).__init__(*args, **kwargs) if True: return self.config = None @@ -57,12 +57,12 @@ class _LayerCodeCompletion(object): class Layer(_LayerCodeCompletion): - def __init__(self, ctx): + def __init__(self, ctx, *args, **kwargs): """ Args: ctx: The (read-only) higher layer. """ - super(Layer, self).__init__() + super(Layer, self).__init__(*args, **kwargs) self.ctx = ctx def __call__(self): @@ -103,10 +103,9 @@ class ServerConnectionMixin(object): Mixin that provides a layer with the capabilities to manage a server connection. """ - def __init__(self): + def __init__(self, server_address=None): super(ServerConnectionMixin, self).__init__() - self._server_address = None - self.server_conn = None + self.server_conn = ServerConnection(server_address) def _handle_server_message(self, message): if message == Reconnect: @@ -116,44 +115,38 @@ class ServerConnectionMixin(object): elif message == Connect: self._connect() return True - elif message == SetServer and message.depth == 1: - if self.server_conn: - self._disconnect() - self.server_address = message.address - return True + elif message == SetServer: + if message.depth == 1: + if self.server_conn: + self._disconnect() + self.log("Set new server address: " + repr(message.address), "debug") + self.server_conn.address = message.address + return True + else: + message.depth -= 1 elif message == Kill: self._disconnect() return False - @property - def server_address(self): - return self._server_address - - @server_address.setter - def server_address(self, address): - self._server_address = tcp.Address.wrap(address) - self.log("Set new server address: " + repr(self.server_address), "debug") - def _disconnect(self): """ Deletes (and closes) an existing server connection. """ - self.log("serverdisconnect", "debug", [repr(self.server_address)]) + self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) self.server_conn.finish() self.server_conn.close() # self.channel.tell("serverdisconnect", self) - self.server_conn = None + self.server_conn = ServerConnection(None) def _connect(self): - if not self.server_address: + if not self.server_conn.address: raise ProtocolException("Cannot connect to server, no server address given.") - self.log("serverconnect", "debug", [repr(self.server_address)]) - self.server_conn = ServerConnection(self.server_address) + self.log("serverconnect", "debug", [repr(self.server_conn.address)]) try: self.server_conn.connect() except tcp.NetLibError as e: - raise ProtocolException("Server connection to '%s' failed: %s" % (self.server_address, e), e) + raise ProtocolException("Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) def yield_from_callback(fun): diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index 17e12f11..f5907537 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -2,6 +2,7 @@ This module contains all valid messages layers can send to the underlying layers. """ from __future__ import (absolute_import, print_function, division) +from netlib.tcp import Address class _Message(object): @@ -33,7 +34,7 @@ class SetServer(_Message): """ def __init__(self, address, server_tls, sni, depth=1): - self.address = address + self.address = Address.wrap(address) self.server_tls = server_tls self.sni = sni diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index bb414ec3..2ee3d9d8 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -7,8 +7,7 @@ from .tls import TlsLayer class ReverseProxy(Layer, ServerConnectionMixin): def __init__(self, ctx, server_address, client_tls, server_tls): - super(ReverseProxy, self).__init__(ctx) - self.server_address = server_address + super(ReverseProxy, self).__init__(ctx, server_address=server_address) self._client_tls = client_tls self._server_tls = server_tls diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index bda8b12b..f369fd48 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -1,9 +1,11 @@ from __future__ import (absolute_import, print_function, division) +from .messages import Kill from .rawtcp import RawTcpLayer from .tls import TlsLayer from .http import HttpLayer + class RootContext(object): """ The outmost context provided to the root layer. @@ -35,7 +37,7 @@ class RootContext(object): ) if not d: - return + return iter([]) if is_tls_client_hello: return TlsLayer(top_layer, True, True) @@ -44,7 +46,6 @@ class RootContext(object): else: return RawTcpLayer(top_layer) - @property def layers(self): return [] diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py index c89477ca..c6126a42 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/protocol2/socks_proxy.py @@ -5,7 +5,7 @@ from ..proxy import ProxyError, Socks5ProxyMode from .layer import Layer, ServerConnectionMixin -class Socks5Proxy(Layer, ServerConnectionMixin): +class Socks5Proxy(ServerConnectionMixin, Layer): def __call__(self): try: s5mode = Socks5ProxyMode(self.config.ssl_ports) @@ -14,7 +14,7 @@ class Socks5Proxy(Layer, ServerConnectionMixin): # TODO: Unmonkeypatch raise ProtocolException(str(e), e) - self.server_address = address + self.server_conn.address = address layer = self.ctx.next_layer(self) for message in layer(): diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index fcc12f18..12c67f4e 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -220,7 +220,7 @@ class TlsLayer(Layer): host = self.server_conn.address.host sans = set() # Incorporate upstream certificate - if self.server_conn.tls_established and (not self.config.no_upstream_cert): + if self.server_conn and self.server_conn.tls_established and (not self.config.no_upstream_cert): upstream_cert = self.server_conn.cert sans.update(upstream_cert.altnames) if upstream_cert.cn: @@ -232,4 +232,5 @@ class TlsLayer(Layer): if self._sni_from_server_change: sans.add(self._sni_from_server_change) + sans.discard(host) return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py index f073e2f8..4ed4c14b 100644 --- a/libmproxy/protocol2/transparent_proxy.py +++ b/libmproxy/protocol2/transparent_proxy.py @@ -13,7 +13,7 @@ class TransparentProxy(Layer, ServerConnectionMixin): def __call__(self): try: - self.server_address = self.resolver.original_addr(self.client_conn.connection) + self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) except Exception as e: raise ProtocolException("Transparent mode failure: %s" % repr(e), e) diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index f33e84cd..f92b53aa 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -96,6 +96,9 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.timestamp_ssl_setup = None self.protocol = None + def __nonzero__(self): + return bool(self.connection) + def __repr__(self): if self.ssl_established and self.sni: ssl = "[ssl: {0}] ".format(self.sni) @@ -132,8 +135,8 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): d.update( address={"address": self.address(), "use_ipv6": self.address.use_ipv6}, - source_address= ({"address": self.source_address(), - "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), + source_address=({"address": self.source_address(), + "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), cert=self.cert.to_pem() if self.cert else None ) return d diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index ffca55ee..e23a7d72 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -80,7 +80,12 @@ class ConnectionHandler2: self.config, self.channel ) - root_layer = protocol2.HttpProxy(root_context) + + # FIXME: properly parse config + if self.config.mode == "upstream": + root_layer = protocol2.HttpUpstreamProxy(root_context, ("localhost", 8081)) + else: + root_layer = protocol2.HttpProxy(root_context) try: for message in root_layer(): @@ -302,7 +307,7 @@ class ConnectionHandler: if ssl_cert_err is not None: self.log( "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), + (ssl_cert_err['depth'], ssl_cert_err['errno']), "error") self.log("Ignoring server verification error, continuing with connection", "error") except tcp.NetLibError as v: @@ -318,7 +323,7 @@ class ConnectionHandler: if ssl_cert_err is not None: self.log( "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), + (ssl_cert_err['depth'], ssl_cert_err['errno']), "error") self.log("Aborting connection attempt", "error") raise e -- cgit v1.2.3 From 96de7ad562da9b5110059988b851c66b51874510 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 18 Aug 2015 14:15:08 +0200 Subject: various fixes --- libmproxy/filt.py | 4 +- libmproxy/protocol/http.py | 1 + libmproxy/protocol2/http.py | 64 ++++++++++++++++++++++++-------- libmproxy/protocol2/http_proxy.py | 5 ++- libmproxy/protocol2/layer.py | 2 + libmproxy/protocol2/reverse_proxy.py | 2 + libmproxy/protocol2/root_context.py | 10 +++-- libmproxy/protocol2/socks_proxy.py | 2 + libmproxy/protocol2/transparent_proxy.py | 2 + libmproxy/proxy/connection.py | 10 +++-- test/test_protocol_http.py | 2 +- 11 files changed, 78 insertions(+), 26 deletions(-) diff --git a/libmproxy/filt.py b/libmproxy/filt.py index bd17a807..25747bc6 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -246,14 +246,14 @@ class FSrc(_Rex): help = "Match source address" def __call__(self, f): - return f.client_conn and re.search(self.expr, repr(f.client_conn.address)) + return f.client_conn.address and re.search(self.expr, repr(f.client_conn.address)) class FDst(_Rex): code = "dst" help = "Match destination address" def __call__(self, f): - return f.server_conn and re.search(self.expr, repr(f.server_conn.address)) + return f.server_conn.address and re.search(self.expr, repr(f.server_conn.address)) class _Int(_Action): def __init__(self, num): diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4c15c80d..4472cb2a 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -729,6 +729,7 @@ class RequestReplayThread(threading.Thread): if not self.flow.response: # In all modes, we directly connect to the server displayed if self.config.mode == "upstream": + # FIXME server_address = self.config.mode.get_upstream_server( self.flow.client_conn )[2:] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index eadde3b3..53f40a72 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -10,13 +10,14 @@ from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from netlib import tcp -from netlib.http import status_codes, http1, HttpErrorConnClosed +from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError from netlib.http.semantics import CONTENT_MISSING from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol + # TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. @@ -31,6 +32,7 @@ class Http1Layer(Layer): layer = HttpLayer(self, self.mode) for message in layer(): yield message + self.server_protocol = HTTP1Protocol(self.server_conn) class Http2Layer(Layer): @@ -41,10 +43,10 @@ class Http2Layer(Layer): self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) def __call__(self): - # FIXME: Handle Reconnect etc. layer = HttpLayer(self, self.mode) for message in layer(): yield message + self.server_protocol = HTTP1Protocol(self.server_conn) def make_error_response(status_code, message, headers=None): @@ -100,6 +102,7 @@ class ConnectServerConnection(object): """ "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. """ + def __init__(self, address, ctx): self.address = tcp.Address.wrap(address) self._ctx = ctx @@ -124,6 +127,8 @@ class HttpLayer(Layer): def __call__(self): while True: try: + flow = HTTPFlow(self.client_conn, self.server_conn, live=True) + try: request = HTTPRequest.from_protocol( self.client_protocol, @@ -148,7 +153,6 @@ class HttpLayer(Layer): # Make sure that the incoming request matches our expectations self.validate_request(request) - flow = HTTPFlow(self.client_conn, self.server_conn) flow.request = request for message in self.process_request_hook(flow): yield message @@ -164,15 +168,22 @@ class HttpLayer(Layer): if self.check_close_connection(flow): return + # TODO: Implement HTTP Upgrade + # Upstream Proxy Mode: Handle CONNECT if flow.request.form_in == "authority" and flow.response.code == 200: for message in self.handle_upstream_mode_connect(flow.request.copy()): yield message return - except (HttpErrorConnClosed, NetLibError) as e: - make_error_response(502, repr(e)) + except (HttpErrorConnClosed, NetLibError, HttpError) as e: + self.send_to_client(make_error_response( + getattr(e, "code", 502), + repr(e) + )) raise ProtocolException(repr(e), e) + finally: + flow.live = False def handle_regular_mode_connect(self, request): yield SetServer((request.host, request.port), False, None) @@ -267,21 +278,43 @@ class HttpLayer(Layer): for chunk in chunks: for part in chunk: + # TODO: That's going to fail. self.send_to_client(part) self.client_conn.wfile.flush() flow.response.timestamp_end = utils.timestamp() def get_response_from_server(self, flow): - # TODO: Add second attempt. - self.send_to_server(flow.request) - - flow.response = HTTPResponse.from_protocol( - self.server_protocol, - flow.request.method, - body_size_limit=self.config.body_size_limit, - include_body=False, - ) + def get_response(): + self.send_to_server(flow.request) + # Only get the headers at first... + flow.response = HTTPResponse.from_protocol( + self.server_protocol, + flow.request.method, + body_size_limit=self.config.body_size_limit, + include_body=False, + ) + + try: + get_response() + except (tcp.NetLibError, HttpErrorConnClosed) as v: + self.log( + "server communication error: %s" % repr(v), + level="debug" + ) + # In any case, we try to reconnect at least once. This is + # necessary because it might be possible that we already + # initiated an upstream connection after clientconnect that + # has already been expired, e.g consider the following event + # log: + # > clientconnect (transparent mode destination known) + # > serverconnect (required for client tls handshake) + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + yield Reconnect() + get_response() # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True @@ -293,7 +326,6 @@ class HttpLayer(Layer): flow.response.content = CONTENT_MISSING else: flow.response.content = self.server_protocol.read_http_body( - self.server_conn, flow.response.headers, self.config.body_size_limit, flow.request.method, @@ -405,7 +437,7 @@ class HttpLayer(Layer): self.send_to_client(make_error_response( 407, "Proxy Authentication Required", - self.config.authenticator.auth_challenge_headers() + odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()]) )) raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 19b5f7ef..652aa473 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -10,7 +10,8 @@ class HttpProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message - + if self.server_conn: + self._disconnect() class HttpUpstreamProxy(Layer, ServerConnectionMixin): def __init__(self, ctx, server_address): @@ -21,3 +22,5 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + if self.server_conn: + self._disconnect() diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 67f3d549..eb41bab7 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -109,7 +109,9 @@ class ServerConnectionMixin(object): def _handle_server_message(self, message): if message == Reconnect: + address = self.server_conn.address self._disconnect() + self.server_conn.address = address self._connect() return True elif message == Connect: diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index 2ee3d9d8..767107ad 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -19,3 +19,5 @@ class ReverseProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 6ba6ca9a..f8a645b0 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -1,4 +1,5 @@ from __future__ import (absolute_import, print_function, division) +import string from .messages import Kill from .rawtcp import RawTcpLayer @@ -36,6 +37,8 @@ class RootContext(object): d[2] in ('\x00', '\x01', '\x02', '\x03') ) + is_ascii = all(x in string.ascii_uppercase for x in d) + # TODO: build is_http2_magic check here, maybe this is an easy way to detect h2c if not d: @@ -43,10 +46,11 @@ class RootContext(object): if is_tls_client_hello: return TlsLayer(top_layer, True, True) - elif isinstance(top_layer, TlsLayer) and top_layer.client_conn.get_alpn_proto_negotiated() == 'h2': + elif isinstance(top_layer, TlsLayer) and is_ascii: + if top_layer.client_conn.get_alpn_proto_negotiated() == 'h2': return Http2Layer(top_layer, 'transparent') - elif isinstance(top_layer, TlsLayer) and isinstance(top_layer.ctx, Http1Layer): - return Http1Layer(top_layer, "transparent") + else: + return Http1Layer(top_layer, "transparent") else: return RawTcpLayer(top_layer) diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py index c6126a42..5bb8e5f8 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/protocol2/socks_proxy.py @@ -20,3 +20,5 @@ class Socks5Proxy(ServerConnectionMixin, Layer): for message in layer(): if not self._handle_server_message(message): yield message + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py index 4ed4c14b..28ad3726 100644 --- a/libmproxy/protocol2/transparent_proxy.py +++ b/libmproxy/protocol2/transparent_proxy.py @@ -21,3 +21,5 @@ class TransparentProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index f92b53aa..c9b57998 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -27,6 +27,9 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): self.timestamp_ssl_setup = None self.protocol = None + def __nonzero__(self): + return bool(self.connection) and not self.finished + def __repr__(self): return "".format( ssl="[ssl] " if self.ssl_established else "", @@ -89,7 +92,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def __init__(self, address): tcp.TCPClient.__init__(self, address) - self.state = [] # a list containing (conntype, state) tuples + self.via = None self.timestamp_start = None self.timestamp_end = None self.timestamp_tcp_setup = None @@ -97,7 +100,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.protocol = None def __nonzero__(self): - return bool(self.connection) + return bool(self.connection) and not self.finished def __repr__(self): if self.ssl_established and self.sni: @@ -117,7 +120,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): return self.ssl_established _stateobject_attributes = dict( - state=list, timestamp_start=float, timestamp_end=float, timestamp_tcp_setup=float, @@ -187,3 +189,5 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def finish(self): tcp.TCPClient.finish(self) self.timestamp_end = utils.timestamp() + +ServerConnection._stateobject_attributes["via"] = ServerConnection \ No newline at end of file diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index 2da54093..c6a9159c 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -56,7 +56,7 @@ class TestInvalidRequests(tservers.HTTPProxTest): p = self.pathoc() r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port)) assert r.status_code == 400 - assert "Must not CONNECT on already encrypted connection" in r.body + assert "Invalid HTTP request form" in r.body def test_relative_request(self): p = self.pathoc_raw() -- cgit v1.2.3 From ab1549e0eff98588211346aada44549311f04938 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 18 Aug 2015 15:59:44 +0200 Subject: yield -> callbacks --- libmproxy/protocol2/__init__.py | 3 +- libmproxy/protocol2/http.py | 133 ++++++++++++++++++------------- libmproxy/protocol2/http_proxy.py | 20 ++--- libmproxy/protocol2/layer.py | 96 +++++----------------- libmproxy/protocol2/messages.py | 4 - libmproxy/protocol2/rawtcp.py | 3 +- libmproxy/protocol2/reverse_proxy.py | 11 +-- libmproxy/protocol2/root_context.py | 2 +- libmproxy/protocol2/socks_proxy.py | 15 ++-- libmproxy/protocol2/tls.py | 54 ++++++------- libmproxy/protocol2/transparent_proxy.py | 10 +-- libmproxy/proxy/server.py | 10 +-- 12 files changed, 156 insertions(+), 205 deletions(-) diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index cf6032da..d5dafaae 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -4,8 +4,7 @@ from .socks_proxy import Socks5Proxy from .reverse_proxy import ReverseProxy from .http_proxy import HttpProxy, HttpUpstreamProxy from .rawtcp import RawTcpLayer -from . import messages __all__ = [ - "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "messages" + "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" ] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 53f40a72..db5aabaf 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -4,7 +4,7 @@ from .. import version from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer from libmproxy import utils -from .messages import SetServer, Connect, Reconnect, Kill +from libmproxy.protocol2.layer import Kill from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow @@ -28,12 +28,21 @@ class Http1Layer(Layer): self.client_protocol = HTTP1Protocol(self.client_conn) self.server_protocol = HTTP1Protocol(self.server_conn) + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP1Protocol(self.server_conn) + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP1Protocol(self.server_conn) + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP1Protocol(self.server_conn) + def __call__(self): layer = HttpLayer(self, self.mode) - for message in layer(): - yield message - self.server_protocol = HTTP1Protocol(self.server_conn) - + layer() class Http2Layer(Layer): def __init__(self, ctx, mode): @@ -42,11 +51,21 @@ class Http2Layer(Layer): self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP2Protocol(self.server_conn) + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP2Protocol(self.server_conn) + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP2Protocol(self.server_conn) + def __call__(self): layer = HttpLayer(self, self.mode) - for message in layer(): - yield message - self.server_protocol = HTTP1Protocol(self.server_conn) + layer() def make_error_response(status_code, message, headers=None): @@ -115,6 +134,37 @@ class ConnectServerConnection(object): return getattr(self.via, item) +class UpstreamConnectLayer(Layer): + def __init__(self, ctx, connect_request): + super(UpstreamConnectLayer, self).__init__(ctx) + self.connect_request = connect_request + self.server_conn = ConnectServerConnection((connect_request.host, connect_request.port), self.ctx) + + def __call__(self): + layer = self.ctx.next_layer(self) + layer() + + def connect(self): + if not self.server_conn: + self.ctx.connect() + self.send_to_server(self.connect_request) + else: + pass # swallow the message + + def reconnect(self): + self.ctx.reconnect() + self.send_to_server(self.connect_request) + + def set_server(self, address, server_tls, sni, depth=1): + if depth == 1: + if self.ctx.server_conn: + self.ctx.reconnect() + self.connect_request.host = address.host + self.connect_request.port = address.port + self.server_conn.address = address + else: + self.ctx.set_server(address, server_tls, sni, depth-1) + class HttpLayer(Layer): """ HTTP 1 Layer @@ -146,22 +196,18 @@ class HttpLayer(Layer): # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.form_in == "authority": - for message in self.handle_regular_mode_connect(request): - yield message + self.handle_regular_mode_connect(request) return # Make sure that the incoming request matches our expectations self.validate_request(request) flow.request = request - for message in self.process_request_hook(flow): - yield message + self.process_request_hook(flow) if not flow.response: - for message in self.establish_server_connection(flow): - yield message - for message in self.get_response_from_server(flow): - yield message + self.establish_server_connection(flow) + self.get_response_from_server(flow) self.send_response_to_client(flow) @@ -172,8 +218,7 @@ class HttpLayer(Layer): # Upstream Proxy Mode: Handle CONNECT if flow.request.form_in == "authority" and flow.response.code == 200: - for message in self.handle_upstream_mode_connect(flow.request.copy()): - yield message + self.handle_upstream_mode_connect(flow.request.copy()) return except (HttpErrorConnClosed, NetLibError, HttpError) as e: @@ -186,38 +231,14 @@ class HttpLayer(Layer): flow.live = False def handle_regular_mode_connect(self, request): - yield SetServer((request.host, request.port), False, None) + self.set_server((request.host, request.port), False, None) self.send_to_client(make_connect_response(request.httpversion)) layer = self.ctx.next_layer(self) - for message in layer(): - yield message + layer() def handle_upstream_mode_connect(self, connect_request): - layer = self.ctx.next_layer(self) - self.server_conn = ConnectServerConnection((connect_request.host, connect_request.port), self.ctx) - - for message in layer(): - if message == Connect: - if not self.server_conn: - yield message - self.send_to_server(connect_request) - else: - pass # swallow the message - elif message == Reconnect: - yield message - self.send_to_server(connect_request) - elif message == SetServer: - if message.depth == 1: - if self.ctx.server_conn: - yield Reconnect() - connect_request.host = message.address.host - connect_request.port = message.address.port - self.server_conn.address = message.address - else: - message.depth -= 1 - yield message - else: - yield message + layer = UpstreamConnectLayer(self, connect_request) + layer() def check_close_connection(self, flow): """ @@ -313,14 +334,14 @@ class HttpLayer(Layer): # > server detects timeout, disconnects # > read (100-n)% of large request # > send large request upstream - yield Reconnect() + self.reconnect() get_response() # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True flow = self.channel.ask("responseheaders", flow) if flow is None or flow == KILL: - yield Kill() + raise Kill() if flow.response.stream and isinstance(self.server_protocol, http1.HTTP1Protocol): flow.response.content = CONTENT_MISSING @@ -345,7 +366,7 @@ class HttpLayer(Layer): ) response_reply = self.channel.ask("response", flow) if response_reply is None or response_reply == KILL: - yield Kill() + raise Kill() def process_request_hook(self, flow): # Determine .scheme, .host and .port attributes for inline scripts. @@ -363,10 +384,10 @@ class HttpLayer(Layer): flow.request.port = self.ctx.server_conn.address.port flow.request.scheme = "https" if self.server_conn.tls_established else "http" - # TODO: Expose SetServer functionality to inline scripts somehow? (yield_from_callback?) + # TODO: Expose .set_server functionality to inline scripts request_reply = self.channel.ask("request", flow) if request_reply is None or request_reply == KILL: - yield Kill() + raise Kill() if isinstance(request_reply, HTTPResponse): flow.response = request_reply return @@ -378,10 +399,10 @@ class HttpLayer(Layer): if self.mode == "regular" or self.mode == "transparent": # If there's an existing connection that doesn't match our expectations, kill it. if address != self.server_conn.address or tls != self.server_conn.ssl_established: - yield SetServer(address, tls, address.host) + self.set_server(address, tls, address.host) # Establish connection is neccessary. if not self.server_conn: - yield Connect() + self.connect() # SetServer is not guaranteed to work with TLS: # If there's not TlsLayer below which could catch the exception, @@ -391,16 +412,16 @@ class HttpLayer(Layer): else: if not self.server_conn: - yield Connect() + self.connect() if tls: raise HttpException("Cannot change scheme in upstream proxy mode.") """ # This is a very ugly (untested) workaround to solve a very ugly problem. if self.server_conn and self.server_conn.tls_established and not ssl: - yield Reconnect() + self.reconnect() elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: if self.server_conn.tls_established: - yield Reconnect() + self.reconnect() self.send_to_server(make_connect_request(address)) tls_layer = TlsLayer(self, False, True) diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 652aa473..c24af6cf 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -7,11 +7,11 @@ from .http import Http1Layer class HttpProxy(Layer, ServerConnectionMixin): def __call__(self): layer = Http1Layer(self, "regular") - for message in layer(): - if not self._handle_server_message(message): - yield message - if self.server_conn: - self._disconnect() + try: + layer() + finally: + if self.server_conn: + self._disconnect() class HttpUpstreamProxy(Layer, ServerConnectionMixin): def __init__(self, ctx, server_address): @@ -19,8 +19,8 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): def __call__(self): layer = Http1Layer(self, "upstream") - for message in layer(): - if not self._handle_server_message(message): - yield message - if self.server_conn: - self._disconnect() + try: + layer() + finally: + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index eb41bab7..7cb76591 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -21,7 +21,7 @@ Automated protocol detection by peeking into the buffer: Communication between layers is done as follows: - lower layers provide context information to higher layers - - higher layers can "yield" commands to lower layers, + - higher layers can call functions provided by lower layers, which are propagated until they reach a suitable layer. Further goals: @@ -35,7 +35,6 @@ import threading from netlib import tcp from ..proxy import Log from ..proxy.connection import ServerConnection -from .messages import Connect, Reconnect, SetServer, Kill from ..exceptions import ProtocolException @@ -69,7 +68,7 @@ class Layer(_LayerCodeCompletion): """ Logic of the layer. Raises: - ProxyError2 in case of protocol exceptions. + ProtocolException in case of protocol exceptions. """ raise NotImplementedError @@ -107,29 +106,20 @@ class ServerConnectionMixin(object): super(ServerConnectionMixin, self).__init__() self.server_conn = ServerConnection(server_address) - def _handle_server_message(self, message): - if message == Reconnect: - address = self.server_conn.address - self._disconnect() + def reconnect(self): + address = self.server_conn.address + self._disconnect() + self.server_conn.address = address + self.connect() + + def set_server(self, address, server_tls, sni, depth=1): + if depth == 1: + if self.server_conn: + self._disconnect() + self.log("Set new server address: " + repr(address), "debug") self.server_conn.address = address - self._connect() - return True - elif message == Connect: - self._connect() - return True - elif message == SetServer: - if message.depth == 1: - if self.server_conn: - self._disconnect() - self.log("Set new server address: " + repr(message.address), "debug") - self.server_conn.address = message.address - return True - else: - message.depth -= 1 - elif message == Kill: - self._disconnect() - - return False + else: + self.ctx.set_server(address, server_tls, sni, depth-1) def _disconnect(self): """ @@ -141,7 +131,7 @@ class ServerConnectionMixin(object): # self.channel.tell("serverdisconnect", self) self.server_conn = ServerConnection(None) - def _connect(self): + def connect(self): if not self.server_conn.address: raise ProtocolException("Cannot connect to server, no server address given.") self.log("serverconnect", "debug", [repr(self.server_conn.address)]) @@ -151,55 +141,7 @@ class ServerConnectionMixin(object): raise ProtocolException("Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) -def yield_from_callback(fun): +class Kill(Exception): """ - Decorator which makes it possible to yield from callbacks in the original thread. - As a use case, take the pyOpenSSL handle_sni callback: If we receive a new SNI from the client, - we need to reconnect to the server with the new SNI. Reconnecting would normally be done using "yield Reconnect()", - but we're in a pyOpenSSL callback here, outside of the main program flow. With this decorator, it looks as follows: - - def handle_sni(self): - # ... - self.yield_from_callback(Reconnect()) - - @yield_from_callback - def establish_ssl_with_client(): - self.client_conn.convert_to_ssl(...) - - for message in self.establish_ssl_with_client(): # will yield Reconnect at some point - yield message - - - Limitations: - - You cannot yield True. - """ - yield_queue = Queue.Queue() - - def do_yield(msg): - yield_queue.put(msg) - yield_queue.get() - - def wrapper(self, *args, **kwargs): - self.yield_from_callback = do_yield - - def run(): - try: - fun(self, *args, **kwargs) - yield_queue.put(True) - except Exception as e: - yield_queue.put(e) - - threading.Thread(target=run, name="YieldFromCallbackThread").start() - while True: - msg = yield_queue.get() - if msg is True: - break - elif isinstance(msg, Exception): - raise ProtocolException("Error in %s: %s" % (fun.__name__, repr(msg)), msg) - else: - yield msg - yield_queue.put(None) - - self.yield_from_callback = None - - return wrapper + Kill a connection. + """ \ No newline at end of file diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index f5907537..de049486 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -44,7 +44,3 @@ class SetServer(_Message): self.depth = depth -class Kill(_Message): - """ - Kill a connection. - """ \ No newline at end of file diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index 167c8c79..6819ad6e 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -4,12 +4,11 @@ import OpenSSL from ..exceptions import ProtocolException from ..protocol.tcp import TCPHandler from .layer import Layer -from .messages import Connect class RawTcpLayer(Layer): def __call__(self): - yield Connect() + self.connect() tcp_handler = TCPHandler(self) try: tcp_handler.handle_messages() diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index 767107ad..9d5a4beb 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -16,8 +16,9 @@ class ReverseProxy(Layer, ServerConnectionMixin): layer = TlsLayer(self, self._client_tls, self._server_tls) else: layer = self.ctx.next_layer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message - if self.server_conn: - self._disconnect() \ No newline at end of file + + try: + layer() + finally: + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index f8a645b0..f0e5b9a7 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -1,7 +1,7 @@ from __future__ import (absolute_import, print_function, division) import string -from .messages import Kill +from libmproxy.protocol2.layer import Kill from .rawtcp import RawTcpLayer from .tls import TlsLayer from .http import Http1Layer, Http2Layer, HttpLayer diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py index 5bb8e5f8..18b363d5 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/protocol2/socks_proxy.py @@ -5,7 +5,7 @@ from ..proxy import ProxyError, Socks5ProxyMode from .layer import Layer, ServerConnectionMixin -class Socks5Proxy(ServerConnectionMixin, Layer): +class Socks5Proxy(Layer, ServerConnectionMixin): def __call__(self): try: s5mode = Socks5ProxyMode(self.config.ssl_ports) @@ -16,9 +16,12 @@ class Socks5Proxy(ServerConnectionMixin, Layer): self.server_conn.address = address + # TODO: Kill event + layer = self.ctx.next_layer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message - if self.server_conn: - self._disconnect() \ No newline at end of file + + try: + layer() + finally: + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 970abe62..28480388 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -6,8 +6,7 @@ from netlib import tcp import netlib.http.http2 from ..exceptions import ProtocolException -from .layer import Layer, yield_from_callback -from .messages import Connect, Reconnect, SetServer +from .layer import Layer class TlsLayer(Layer): @@ -50,35 +49,34 @@ class TlsLayer(Layer): ) if client_tls_requires_server_cert: - for m in self._establish_tls_with_client_and_server(): - yield m + self._establish_tls_with_client_and_server() elif self._client_tls: - for m in self._establish_tls_with_client(): - yield m + self._establish_tls_with_client() layer = self.ctx.next_layer(self) + layer() - for message in layer(): - self.log("TlsLayer: %s" % message,"debug") - if not (message == Connect and self._connected): - yield message + def connect(self): + if not self.server_conn: + self.ctx.connect() + if self._server_tls and not self._server_tls_established: + self._establish_tls_with_server() + + def reconnect(self): + self.ctx.reconnect() + if self._server_tls and not self._server_tls_established: + self._establish_tls_with_server() - if message == Connect or message == Reconnect: - if self._server_tls and not self._server_tls_established: - self._establish_tls_with_server() - if message == SetServer and message.depth == 1: - if message.server_tls is not None: - self._sni_from_server_change = message.sni - self._server_tls = message.server_tls + def set_server(self, address, server_tls, sni, depth=1): + self.ctx.set_server(address, server_tls, sni, depth) + if server_tls is not None: + self._sni_from_server_change = sni + self._server_tls = server_tls @property def _server_tls_established(self): return self.server_conn and self.server_conn.tls_established - @property - def _connected(self): - return bool(self.server_conn) - @property def sni_for_upstream_connection(self): if self._sni_from_server_change is False: @@ -92,19 +90,14 @@ class TlsLayer(Layer): """ # First, try to connect to the server. - yield Connect() + self.ctx.connect() server_err = None try: self._establish_tls_with_server() except ProtocolException as e: server_err = e - for message in self._establish_tls_with_client(): - if message == Reconnect: - yield message - self._establish_tls_with_server() - else: - raise RuntimeError("Unexpected Message: %s" % message) + self._establish_tls_with_client() if server_err and not self.client_sni: raise server_err @@ -125,7 +118,7 @@ class TlsLayer(Layer): if old_upstream_sni != self.sni_for_upstream_connection: # Perform reconnect if self.server_conn and self._server_tls: - self.yield_from_callback(Reconnect()) + self.reconnect() if self.client_sni: # Now, change client context to reflect possibly changed certificate: @@ -156,7 +149,7 @@ class TlsLayer(Layer): # Perform reconnect # TODO: Avoid double reconnect. if self.server_conn and self._server_tls: - self.yield_from_callback(Reconnect()) + self.reconnect() self.client_alpn_protos = options @@ -165,7 +158,6 @@ class TlsLayer(Layer): else: # pragma no cover return options[0] - @yield_from_callback def _establish_tls_with_client(self): self.log("Establish TLS with client", "debug") cert, key, chain_file = self._find_cert() diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py index 28ad3726..9263dbde 100644 --- a/libmproxy/protocol2/transparent_proxy.py +++ b/libmproxy/protocol2/transparent_proxy.py @@ -18,8 +18,8 @@ class TransparentProxy(Layer, ServerConnectionMixin): raise ProtocolException("Transparent mode failure: %s" % repr(e), e) layer = self.ctx.next_layer(self) - for message in layer(): - if not self._handle_server_message(message): - yield message - if self.server_conn: - self._disconnect() \ No newline at end of file + try: + layer() + finally: + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index e23a7d72..9957caa0 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -3,6 +3,7 @@ from __future__ import absolute_import, print_function import traceback import sys import socket +from libmproxy.protocol2.layer import Kill from netlib import tcp from ..protocol.handle import protocol_handler @@ -88,12 +89,9 @@ class ConnectionHandler2: root_layer = protocol2.HttpProxy(root_context) try: - for message in root_layer(): - if message == protocol2.messages.Kill: - self.log("Connection killed", "info") - break - - print("Root layer receveived: %s" % message) + root_layer() + except Kill as e: + self.log("Connection killed", "info") except ProtocolException as e: self.log(e, "info") except Exception: -- cgit v1.2.3 From 9bae97eb17ed66a33b5b988c6857ca6c9fae8e22 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 18 Aug 2015 13:43:26 +0200 Subject: http2: fix connection preface and wrappers --- libmproxy/protocol/http_wrappers.py | 36 ++++++++++-------------------------- libmproxy/protocol2/http.py | 16 +++++++--------- 2 files changed, 17 insertions(+), 35 deletions(-) diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index ed5759ea..e41d65d6 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -247,24 +247,11 @@ class HTTPRequest(MessageMixin, semantics.Request): include_body = include_body, body_size_limit = body_size_limit, ) - - return HTTPRequest( - req.form_in, - req.method, - req.scheme, - req.host, - req.port, - req.path, - req.httpversion, - req.headers, - req.body, - req.timestamp_start, - req.timestamp_end, - ) + return self.wrap(req) @classmethod def wrap(self, request): - return HTTPRequest( + req = HTTPRequest( form_in=request.form_in, method=request.method, scheme=request.scheme, @@ -278,6 +265,9 @@ class HTTPRequest(MessageMixin, semantics.Request): timestamp_end=request.timestamp_end, form_out=(request.form_out if hasattr(request, 'form_out') else None), ) + if hasattr(request, 'stream_id'): + req.stream_id = request.stream_id + return req def __hash__(self): return id(self) @@ -371,20 +361,11 @@ class HTTPResponse(MessageMixin, semantics.Response): body_size_limit, include_body=include_body ) - - return HTTPResponse( - resp.httpversion, - resp.status_code, - resp.msg, - resp.headers, - resp.body, - resp.timestamp_start, - resp.timestamp_end, - ) + return self.wrap(resp) @classmethod def wrap(self, response): - return HTTPResponse( + resp = HTTPResponse( httpversion=response.httpversion, status_code=response.status_code, msg=response.msg, @@ -393,6 +374,9 @@ class HTTPResponse(MessageMixin, semantics.Response): timestamp_start=response.timestamp_start, timestamp_end=response.timestamp_end, ) + if hasattr(response, 'stream_id'): + resp.stream_id = response.stream_id + return resp def _refresh_cookie(self, c, delta): """ diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index db5aabaf..e73bbb61 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -10,7 +10,7 @@ from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from netlib import tcp -from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError +from netlib.http import status_codes, http1, http2, HttpErrorConnClosed, HttpError from netlib.http.semantics import CONTENT_MISSING from netlib import odict from netlib.tcp import NetLibError, Address @@ -64,6 +64,7 @@ class Http2Layer(Layer): self.server_protocol = HTTP2Protocol(self.server_conn) def __call__(self): + self.server_protocol.perform_connection_preface() layer = HttpLayer(self, self.mode) layer() @@ -166,10 +167,6 @@ class UpstreamConnectLayer(Layer): self.ctx.set_server(address, server_tls, sni, depth-1) class HttpLayer(Layer): - """ - HTTP 1 Layer - """ - def __init__(self, ctx, mode): super(HttpLayer, self).__init__(ctx) self.mode = mode @@ -337,15 +334,18 @@ class HttpLayer(Layer): self.reconnect() get_response() + if isinstance(self.server_protocol, http2.HTTP2Protocol): + flow.response.stream_id = flow.request.stream_id + # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True flow = self.channel.ask("responseheaders", flow) if flow is None or flow == KILL: raise Kill() - if flow.response.stream and isinstance(self.server_protocol, http1.HTTP1Protocol): + if flow.response.stream: flow.response.content = CONTENT_MISSING - else: + elif isinstance(self.server_protocol, http1.HTTP1Protocol): flow.response.content = self.server_protocol.read_http_body( flow.response.headers, self.config.body_size_limit, @@ -466,6 +466,4 @@ class HttpLayer(Layer): self.server_conn.send(self.server_protocol.assemble(message)) def send_to_client(self, message): - # FIXME - # - possibly do some http2 stuff here self.client_conn.send(self.client_protocol.assemble(message)) -- cgit v1.2.3 From c9fa8491ccc015ddff09ce15a5d718d6b58b515c Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 19 Aug 2015 15:23:52 +0200 Subject: improve next_layer detection --- libmproxy/protocol2/root_context.py | 31 +++++++++++++++++++------------ libmproxy/proxy/connection.py | 2 +- 2 files changed, 20 insertions(+), 13 deletions(-) diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index f0e5b9a7..9b18f0aa 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -6,6 +6,7 @@ from .rawtcp import RawTcpLayer from .tls import TlsLayer from .http import Http1Layer, Http2Layer, HttpLayer +from netlib.http.http2 import HTTP2Protocol class RootContext(object): """ @@ -25,11 +26,11 @@ class RootContext(object): :return: The next layer. """ - d = top_layer.client_conn.rfile.peek(3) - # TODO: Handle ignore and tcp passthrough - # TLS ClientHello magic, see http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + d = top_layer.client_conn.rfile.peek(3) is_tls_client_hello = ( len(d) == 3 and d[0] == '\x16' and @@ -37,20 +38,26 @@ class RootContext(object): d[2] in ('\x00', '\x01', '\x02', '\x03') ) - is_ascii = all(x in string.ascii_uppercase for x in d) + d = top_layer.client_conn.rfile.peek(3) + is_ascii = ( + len(d) == 3 and + all(x in string.ascii_uppercase for x in d) + ) - # TODO: build is_http2_magic check here, maybe this is an easy way to detect h2c + d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) + is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) - if not d: - return iter([]) + is_alpn_h2_negotiated = ( + isinstance(top_layer, TlsLayer) and + top_layer.client_conn.get_alpn_proto_negotiated() == HTTP2Protocol.ALPN_PROTO_H2 + ) if is_tls_client_hello: return TlsLayer(top_layer, True, True) - elif isinstance(top_layer, TlsLayer) and is_ascii: - if top_layer.client_conn.get_alpn_proto_negotiated() == 'h2': - return Http2Layer(top_layer, 'transparent') - else: - return Http1Layer(top_layer, "transparent") + elif is_alpn_h2_negotiated or is_http2_magic: + return Http2Layer(top_layer, 'transparent') + elif is_ascii: + return Http1Layer(top_layer, 'transparent') else: return RawTcpLayer(top_layer) diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index c9b57998..c329ed64 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -190,4 +190,4 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): tcp.TCPClient.finish(self) self.timestamp_end = utils.timestamp() -ServerConnection._stateobject_attributes["via"] = ServerConnection \ No newline at end of file +ServerConnection._stateobject_attributes["via"] = ServerConnection -- cgit v1.2.3 From 97bfd1d856b26216fffbf84b9e75e49b41fc6fb2 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 19 Aug 2015 16:36:22 +0200 Subject: move send method to lower layers --- libmproxy/protocol2/http.py | 21 +++++++++++++++------ 1 file changed, 15 insertions(+), 6 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index e73bbb61..b8b1e8a5 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -28,6 +28,12 @@ class Http1Layer(Layer): self.client_protocol = HTTP1Protocol(self.client_conn) self.server_protocol = HTTP1Protocol(self.server_conn) + def send_to_client(self, message): + self.client_conn.send(self.client_protocol.assemble(message)) + + def send_to_server(self, message): + self.server_conn.send(self.server_protocol.assemble(message)) + def connect(self): self.ctx.connect() self.server_protocol = HTTP1Protocol(self.server_conn) @@ -51,6 +57,14 @@ class Http2Layer(Layer): self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + def send_to_client(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.client_conn.send(self.client_protocol.assemble(message)) + + def send_to_server(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.server_conn.send(self.server_protocol.assemble(message)) + def connect(self): self.ctx.connect() self.server_protocol = HTTP2Protocol(self.server_conn) @@ -166,6 +180,7 @@ class UpstreamConnectLayer(Layer): else: self.ctx.set_server(address, server_tls, sni, depth-1) + class HttpLayer(Layer): def __init__(self, ctx, mode): super(HttpLayer, self).__init__(ctx) @@ -461,9 +476,3 @@ class HttpLayer(Layer): odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()]) )) raise InvalidCredentials("Proxy Authentication Required") - - def send_to_server(self, message): - self.server_conn.send(self.server_protocol.assemble(message)) - - def send_to_client(self, message): - self.client_conn.send(self.client_protocol.assemble(message)) -- cgit v1.2.3 From f2ace5493b1fc6eceff46cff3fb1238559e9aef2 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 19 Aug 2015 18:09:45 +0200 Subject: move read methods to lower HTTP layer --- libmproxy/protocol2/http.py | 53 +++++++++++++++++++++++++++++++-------------- 1 file changed, 37 insertions(+), 16 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index b8b1e8a5..5170660f 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -28,6 +28,20 @@ class Http1Layer(Layer): self.client_protocol = HTTP1Protocol(self.client_conn) self.server_protocol = HTTP1Protocol(self.server_conn) + def read_from_client(self): + return HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + + def read_from_server(self, method): + return HTTPResponse.from_protocol( + self.server_protocol, + method, + body_size_limit=self.config.body_size_limit, + include_body=False, + ) + def send_to_client(self, message): self.client_conn.send(self.client_protocol.assemble(message)) @@ -57,6 +71,20 @@ class Http2Layer(Layer): self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + def read_from_client(self): + return HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + + def read_from_server(self, method): + return HTTPResponse.from_protocol( + self.server_protocol, + method, + body_size_limit=self.config.body_size_limit, + include_body=False, + ) + def send_to_client(self, message): # TODO: implement flow control and WINDOW_UPDATE frames self.client_conn.send(self.client_protocol.assemble(message)) @@ -67,15 +95,18 @@ class Http2Layer(Layer): def connect(self): self.ctx.connect() - self.server_protocol = HTTP2Protocol(self.server_conn) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.server_protocol.perform_connection_preface() def reconnect(self): self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.server_protocol.perform_connection_preface() def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP2Protocol(self.server_conn) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.server_protocol.perform_connection_preface() def __call__(self): self.server_protocol.perform_connection_preface() @@ -192,10 +223,7 @@ class HttpLayer(Layer): flow = HTTPFlow(self.client_conn, self.server_conn, live=True) try: - request = HTTPRequest.from_protocol( - self.client_protocol, - body_size_limit=self.config.body_size_limit - ) + request = self.read_from_client() except tcp.NetLibError: # don't throw an error for disconnects that happen # before/between requests. @@ -320,13 +348,7 @@ class HttpLayer(Layer): def get_response_from_server(self, flow): def get_response(): self.send_to_server(flow.request) - # Only get the headers at first... - flow.response = HTTPResponse.from_protocol( - self.server_protocol, - flow.request.method, - body_size_limit=self.config.body_size_limit, - include_body=False, - ) + flow.response = self.read_from_server(flow.request.method) try: get_response() @@ -408,9 +430,9 @@ class HttpLayer(Layer): return def establish_server_connection(self, flow): - address = tcp.Address((flow.request.host, flow.request.port)) tls = (flow.request.scheme == "https") + if self.mode == "regular" or self.mode == "transparent": # If there's an existing connection that doesn't match our expectations, kill it. if address != self.server_conn.address or tls != self.server_conn.ssl_established: @@ -424,7 +446,6 @@ class HttpLayer(Layer): # TLS will not be established. if tls and not self.server_conn.tls_established: raise ProtocolException("Cannot upgrade to SSL, no TLS layer on the protocol stack.") - else: if not self.server_conn: self.connect() -- cgit v1.2.3 From 4339b8e7fa1140b9138a023e7e61d78cefe6bb02 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 19 Aug 2015 21:09:48 +0200 Subject: http2: use callback for handle unexpected frames --- libmproxy/protocol2/http.py | 13 ++++++++----- 1 file changed, 8 insertions(+), 5 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 5170660f..e227f0ba 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -68,8 +68,8 @@ class Http2Layer(Layer): def __init__(self, ctx, mode): super(Http2Layer, self).__init__(ctx) self.mode = mode - self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) def read_from_client(self): return HTTPRequest.from_protocol( @@ -95,17 +95,17 @@ class Http2Layer(Layer): def connect(self): self.ctx.connect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol.perform_connection_preface() def reconnect(self): self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol.perform_connection_preface() def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol.perform_connection_preface() def __call__(self): @@ -113,6 +113,9 @@ class Http2Layer(Layer): layer = HttpLayer(self, self.mode) layer() + def handle_unexpected_frame(self, frm): + print(frm.human_readable()) + def make_error_response(status_code, message, headers=None): response = status_codes.RESPONSES.get(status_code, "Unknown") -- cgit v1.2.3 From 5746472426d3928497e9c8f85664a46598a044af Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 20 Aug 2015 19:53:17 +0200 Subject: fix typo --- libmproxy/protocol2/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index e227f0ba..e5a434f2 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -469,7 +469,7 @@ class HttpLayer(Layer): def validate_request(self, request): if request.form_in == "absolute" and request.scheme != "http": - self.send_resplonse(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + self.send_response(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { -- cgit v1.2.3 From 55cfd259dc8264ca9bca166b50b0d3dc7ab71451 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 20 Aug 2015 20:31:01 +0200 Subject: http2: simplify protocol-related code --- libmproxy/protocol2/http.py | 17 ++++++++--------- 1 file changed, 8 insertions(+), 9 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index e5a434f2..a2dfc428 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -34,10 +34,10 @@ class Http1Layer(Layer): body_size_limit=self.config.body_size_limit ) - def read_from_server(self, method): + def read_from_server(self, request): return HTTPResponse.from_protocol( self.server_protocol, - method, + request.method, body_size_limit=self.config.body_size_limit, include_body=False, ) @@ -77,13 +77,15 @@ class Http2Layer(Layer): body_size_limit=self.config.body_size_limit ) - def read_from_server(self, method): - return HTTPResponse.from_protocol( + def read_from_server(self, request): + response = HTTPResponse.from_protocol( self.server_protocol, - method, + request.method, body_size_limit=self.config.body_size_limit, include_body=False, ) + response.stream_id = request.stream_id + return response def send_to_client(self, message): # TODO: implement flow control and WINDOW_UPDATE frames @@ -351,7 +353,7 @@ class HttpLayer(Layer): def get_response_from_server(self, flow): def get_response(): self.send_to_server(flow.request) - flow.response = self.read_from_server(flow.request.method) + flow.response = self.read_from_server(flow.request) try: get_response() @@ -374,9 +376,6 @@ class HttpLayer(Layer): self.reconnect() get_response() - if isinstance(self.server_protocol, http2.HTTP2Protocol): - flow.response.stream_id = flow.request.stream_id - # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True flow = self.channel.ask("responseheaders", flow) -- cgit v1.2.3 From 05d26545e4c65e8fd2242142833a40a96ce5fb81 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 21 Aug 2015 10:26:28 +0200 Subject: adapt netlib changes --- libmproxy/protocol/http.py | 6 +++--- libmproxy/protocol/http_wrappers.py | 4 ++-- libmproxy/protocol2/http.py | 4 ++-- test/test_protocol_http.py | 9 +++++---- 4 files changed, 12 insertions(+), 11 deletions(-) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4472cb2a..1b168569 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -41,7 +41,7 @@ def send_connect_request(conn, host, port, update_state=True): protocol = http1.HTTP1Protocol(conn) conn.send(protocol.assemble(upstream_request)) - resp = HTTPResponse.from_protocol(protocol, upstream_request.method) + resp = HTTPResponse.from_protocol(protocol, upstream_request) if resp.status_code != 200: raise proxy.ProxyError(resp.status_code, "Cannot establish SSL " + @@ -177,7 +177,7 @@ class HTTPHandler(ProtocolHandler): # Only get the headers at first... flow.response = HTTPResponse.from_protocol( self.c.server_conn.protocol, - flow.request.method, + flow.request, body_size_limit=self.c.config.body_size_limit, include_body=False, ) @@ -760,7 +760,7 @@ class RequestReplayThread(threading.Thread): self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) self.flow.response = HTTPResponse.from_protocol( self.flow.server_conn.protocol, - r.method, + r, body_size_limit=self.config.body_size_limit, ) if self.channel: diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index e41d65d6..f91b936c 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -352,12 +352,12 @@ class HTTPResponse(MessageMixin, semantics.Response): def from_protocol( self, protocol, - request_method, + request, include_body=True, body_size_limit=None ): resp = protocol.read_response( - request_method, + request, body_size_limit, include_body=include_body ) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index a2dfc428..f093f7c5 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -37,7 +37,7 @@ class Http1Layer(Layer): def read_from_server(self, request): return HTTPResponse.from_protocol( self.server_protocol, - request.method, + request, body_size_limit=self.config.body_size_limit, include_body=False, ) @@ -80,7 +80,7 @@ class Http2Layer(Layer): def read_from_server(self, request): response = HTTPResponse.from_protocol( self.server_protocol, - request.method, + request, body_size_limit=self.config.body_size_limit, include_body=False, ) diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index c6a9159c..940d6c7a 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -4,6 +4,7 @@ from cStringIO import StringIO from mock import MagicMock from libmproxy.protocol.http import * +import netlib.http from netlib import odict from netlib.http import http1 from netlib.http.semantics import CONTENT_MISSING @@ -27,19 +28,19 @@ class TestHTTPResponse: "\r\n" protocol = mock_protocol(s) - r = HTTPResponse.from_protocol(protocol, "GET") + r = HTTPResponse.from_protocol(protocol, netlib.http.EmptyRequest(method="GET")) assert r.status_code == 200 assert r.content == "content" - assert HTTPResponse.from_protocol(protocol, "GET").status_code == 204 + assert HTTPResponse.from_protocol(protocol, netlib.http.EmptyRequest(method="GET")).status_code == 204 protocol = mock_protocol(s) # HEAD must not have content by spec. We should leave it on the pipe. - r = HTTPResponse.from_protocol(protocol, "HEAD") + r = HTTPResponse.from_protocol(protocol, netlib.http.EmptyRequest(method="HEAD")) assert r.status_code == 200 assert r.content == "" tutils.raises( "Invalid server response: 'content", - HTTPResponse.from_protocol, protocol, "GET" + HTTPResponse.from_protocol, protocol, netlib.http.EmptyRequest(method="GET") ) -- cgit v1.2.3 From f1f34e7713295adb8f54b13ec50d74d43cd42841 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 24 Aug 2015 16:52:03 +0200 Subject: fix bugs, fix tests --- libmproxy/protocol2/http.py | 9 +++-- libmproxy/protocol2/root_context.py | 2 +- libmproxy/protocol2/tls.py | 79 ++++++++++++++++++++----------------- test/test_proxy.py | 2 +- test/test_server.py | 22 ++++------- 5 files changed, 58 insertions(+), 56 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index f093f7c5..973f169c 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -266,12 +266,15 @@ class HttpLayer(Layer): self.handle_upstream_mode_connect(flow.request.copy()) return - except (HttpErrorConnClosed, NetLibError, HttpError) as e: + except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: self.send_to_client(make_error_response( getattr(e, "code", 502), repr(e) )) - raise ProtocolException(repr(e), e) + if isinstance(e, ProtocolException): + raise e + else: + raise ProtocolException(repr(e), e) finally: flow.live = False @@ -468,7 +471,7 @@ class HttpLayer(Layer): def validate_request(self, request): if request.form_in == "absolute" and request.scheme != "http": - self.send_response(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + self.send_to_client(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 9b18f0aa..d0c62be4 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -41,7 +41,7 @@ class RootContext(object): d = top_layer.client_conn.rfile.peek(3) is_ascii = ( len(d) == 3 and - all(x in string.ascii_uppercase for x in d) + all(x in string.ascii_letters for x in d) # better be safe here and don't expect uppercase... ) d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 28480388..98c5d603 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -17,6 +17,7 @@ class TlsLayer(Layer): self.client_sni = None self._sni_from_server_change = None self.client_alpn_protos = None + self.__server_tls_exception = None # foo alpn protos = [netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1, netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], # TODO: read this from client_conn first @@ -107,49 +108,48 @@ class TlsLayer(Layer): This callback gets called during the TLS handshake with the client. The client has just sent the Sever Name Indication (SNI). """ - try: - old_upstream_sni = self.sni_for_upstream_connection - - sn = connection.get_servername() - if not sn: - return - self.client_sni = sn.decode("utf8").encode("idna") - - if old_upstream_sni != self.sni_for_upstream_connection: - # Perform reconnect - if self.server_conn and self._server_tls: - self.reconnect() - - if self.client_sni: - # Now, change client context to reflect possibly changed certificate: - cert, key, chain_file = self._find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file, - alpn_select_callback=self.__handle_alpn_select, - ) - connection.set_context(new_context) - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except: # pragma: no cover - self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") + old_upstream_sni = self.sni_for_upstream_connection + + sn = connection.get_servername() + if not sn: + return + + self.client_sni = sn.decode("utf8").encode("idna") + + server_sni_changed = (old_upstream_sni != self.sni_for_upstream_connection) + server_conn_with_tls_exists = (self.server_conn and self._server_tls) + if server_sni_changed and server_conn_with_tls_exists: + try: + self.reconnect() + except Exception as e: + self.__server_tls_exception = e + + # Now, change client context to reflect possibly changed certificate: + cert, key, chain_file = self._find_cert() + new_context = self.client_conn.create_ssl_context( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file, + alpn_select_callback=self.__handle_alpn_select, + ) + connection.set_context(new_context) def __handle_alpn_select(self, conn_, options): # TODO: change to something meaningful? - alpn_preference = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 + # alpn_preference = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 - ### - # TODO: Not - if self.client_alpn_protos != options: - # Perform reconnect - # TODO: Avoid double reconnect. - if self.server_conn and self._server_tls: + # TODO: Don't reconnect twice? + upstream_alpn_changed = (self.client_alpn_protos != options) + server_conn_with_tls_exists = (self.server_conn and self._server_tls) + if upstream_alpn_changed and server_conn_with_tls_exists: + try: self.reconnect() + except Exception as e: + self.__server_tls_exception = e self.client_alpn_protos = options @@ -177,6 +177,11 @@ class TlsLayer(Layer): print("alpn: %s" % self.client_alpn_protos) raise ProtocolException(repr(e), e) + # Do not raise server tls exceptions immediately. + # We want to try to finish the client handshake so that other layers can send error messages over it. + if self.__server_tls_exception: + raise self.__server_tls_exception + def _establish_tls_with_server(self): self.log("Establish TLS with server", "debug") try: diff --git a/test/test_proxy.py b/test/test_proxy.py index 6ab19e02..b9eec53b 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -36,7 +36,7 @@ class TestServerConnection: sc.send(protocol.assemble(f.request)) protocol = http.http1.HTTP1Protocol(rfile=sc.rfile) - assert protocol.read_response(f.request.method, 1000) + assert protocol.read_response(f.request, 1000) assert self.d.last_log() sc.finish() diff --git a/test/test_server.py b/test/test_server.py index 77ba4576..529024f5 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -319,17 +319,6 @@ class TestHTTPAuth(tservers.HTTPProxTest): assert ret.status_code == 202 -class TestHTTPConnectSSLError(tservers.HTTPProxTest): - certfile = True - - def test_go(self): - self.config.ssl_ports.append(self.proxy.port) - p = self.pathoc_raw() - dst = ("localhost", self.proxy.port) - p.connect(connect_to=dst) - tutils.raises("502 - Bad Gateway", p.http_connect, dst) - - class TestHTTPS(tservers.HTTPProxTest, CommonMixin, TcpMixin): ssl = True ssloptions = pathod.SSLOptions(request_client_cert=True) @@ -390,26 +379,31 @@ class TestHTTPSUpstreamServerVerificationWBadCert(tservers.HTTPProxTest): ("untrusted-cert", tutils.test_data.path("data/untrusted-server.crt")) ]) + def _request(self): + p = self.pathoc() + # We need to make an actual request because the upstream connection is lazy-loaded. + return p.request("get:/p/242") + def test_default_verification_w_bad_cert(self): """Should use no verification.""" self.config.openssl_trusted_ca_server = tutils.test_data.path( "data/trusted-cadir/trusted-ca.pem") - self.pathoc() + assert self._request().status_code == 242 def test_no_verification_w_bad_cert(self): self.config.openssl_verification_mode_server = SSL.VERIFY_NONE self.config.openssl_trusted_ca_server = tutils.test_data.path( "data/trusted-cadir/trusted-ca.pem") - self.pathoc() + assert self._request().status_code == 242 def test_verification_w_bad_cert(self): self.config.openssl_verification_mode_server = SSL.VERIFY_PEER self.config.openssl_trusted_ca_server = tutils.test_data.path( "data/trusted-cadir/trusted-ca.pem") - tutils.raises("SSL handshake error", self.pathoc) + assert self._request().status_code == 502 class TestHTTPSNoCommonName(tservers.HTTPProxTest): -- cgit v1.2.3 From 56a4bc381efc177c08ed7b9e7c845f74120050e4 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 24 Aug 2015 18:17:04 +0200 Subject: request -> request_method --- libmproxy/protocol/http.py | 6 +++--- libmproxy/protocol/http_wrappers.py | 20 ++++++-------------- libmproxy/protocol2/http.py | 19 ++++++++++--------- test/test_protocol_http.py | 8 ++++---- test/test_proxy.py | 2 +- 5 files changed, 24 insertions(+), 31 deletions(-) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 1b168569..4472cb2a 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -41,7 +41,7 @@ def send_connect_request(conn, host, port, update_state=True): protocol = http1.HTTP1Protocol(conn) conn.send(protocol.assemble(upstream_request)) - resp = HTTPResponse.from_protocol(protocol, upstream_request) + resp = HTTPResponse.from_protocol(protocol, upstream_request.method) if resp.status_code != 200: raise proxy.ProxyError(resp.status_code, "Cannot establish SSL " + @@ -177,7 +177,7 @@ class HTTPHandler(ProtocolHandler): # Only get the headers at first... flow.response = HTTPResponse.from_protocol( self.c.server_conn.protocol, - flow.request, + flow.request.method, body_size_limit=self.c.config.body_size_limit, include_body=False, ) @@ -760,7 +760,7 @@ class RequestReplayThread(threading.Thread): self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) self.flow.response = HTTPResponse.from_protocol( self.flow.server_conn.protocol, - r, + r.method, body_size_limit=self.config.body_size_limit, ) if self.channel: diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index f91b936c..b1000a79 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -240,13 +240,10 @@ class HTTPRequest(MessageMixin, semantics.Request): def from_protocol( self, protocol, - include_body=True, - body_size_limit=None, + *args, + **kwargs ): - req = protocol.read_request( - include_body = include_body, - body_size_limit = body_size_limit, - ) + req = protocol.read_request(*args, **kwargs) return self.wrap(req) @classmethod @@ -352,15 +349,10 @@ class HTTPResponse(MessageMixin, semantics.Response): def from_protocol( self, protocol, - request, - include_body=True, - body_size_limit=None + *args, + **kwargs ): - resp = protocol.read_response( - request, - body_size_limit, - include_body=include_body - ) + resp = protocol.read_response(*args, **kwargs) return self.wrap(resp) @classmethod diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 973f169c..2c8c8d27 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -34,10 +34,10 @@ class Http1Layer(Layer): body_size_limit=self.config.body_size_limit ) - def read_from_server(self, request): + def read_from_server(self, request_method): return HTTPResponse.from_protocol( self.server_protocol, - request, + request_method, body_size_limit=self.config.body_size_limit, include_body=False, ) @@ -64,6 +64,7 @@ class Http1Layer(Layer): layer = HttpLayer(self, self.mode) layer() + class Http2Layer(Layer): def __init__(self, ctx, mode): super(Http2Layer, self).__init__(ctx) @@ -72,20 +73,20 @@ class Http2Layer(Layer): self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) def read_from_client(self): - return HTTPRequest.from_protocol( + request = HTTPRequest.from_protocol( self.client_protocol, body_size_limit=self.config.body_size_limit ) + self._stream_id = request.stream_id - def read_from_server(self, request): - response = HTTPResponse.from_protocol( + def read_from_server(self, request_method): + return HTTPResponse.from_protocol( self.server_protocol, - request, + request_method, body_size_limit=self.config.body_size_limit, include_body=False, + stream_id=self._stream_id ) - response.stream_id = request.stream_id - return response def send_to_client(self, message): # TODO: implement flow control and WINDOW_UPDATE frames @@ -356,7 +357,7 @@ class HttpLayer(Layer): def get_response_from_server(self, flow): def get_response(): self.send_to_server(flow.request) - flow.response = self.read_from_server(flow.request) + flow.response = self.read_from_server(flow.request.method) try: get_response() diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index 940d6c7a..cd0f77fa 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -28,19 +28,19 @@ class TestHTTPResponse: "\r\n" protocol = mock_protocol(s) - r = HTTPResponse.from_protocol(protocol, netlib.http.EmptyRequest(method="GET")) + r = HTTPResponse.from_protocol(protocol, "GET") assert r.status_code == 200 assert r.content == "content" - assert HTTPResponse.from_protocol(protocol, netlib.http.EmptyRequest(method="GET")).status_code == 204 + assert HTTPResponse.from_protocol(protocol, "GET").status_code == 204 protocol = mock_protocol(s) # HEAD must not have content by spec. We should leave it on the pipe. - r = HTTPResponse.from_protocol(protocol, netlib.http.EmptyRequest(method="HEAD")) + r = HTTPResponse.from_protocol(protocol, "HEAD") assert r.status_code == 200 assert r.content == "" tutils.raises( "Invalid server response: 'content", - HTTPResponse.from_protocol, protocol, netlib.http.EmptyRequest(method="GET") + HTTPResponse.from_protocol, protocol, "GET" ) diff --git a/test/test_proxy.py b/test/test_proxy.py index b9eec53b..6ab19e02 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -36,7 +36,7 @@ class TestServerConnection: sc.send(protocol.assemble(f.request)) protocol = http.http1.HTTP1Protocol(rfile=sc.rfile) - assert protocol.read_response(f.request, 1000) + assert protocol.read_response(f.request.method, 1000) assert self.d.last_log() sc.finish() -- cgit v1.2.3 From 8ce0de8bed5fbd2c42e7b43ee553e065e1c08a4c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 25 Aug 2015 18:24:17 +0200 Subject: minor fixes --- libmproxy/protocol2/http.py | 11 +++++++---- libmproxy/proxy/server.py | 2 ++ 2 files changed, 9 insertions(+), 4 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 2c8c8d27..5a25c317 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -268,10 +268,13 @@ class HttpLayer(Layer): return except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: - self.send_to_client(make_error_response( - getattr(e, "code", 502), - repr(e) - )) + try: + self.send_to_client(make_error_response( + getattr(e, "code", 502), + repr(e) + )) + except NetLibError: + pass if isinstance(e, ProtocolException): raise e else: diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 9957caa0..19ddb930 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -100,6 +100,8 @@ class ConnectionHandler2: print("mitmproxy has crashed!", file=sys.stderr) print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) + self.log("clientdisconnect", "info") + def finish(self): self.client_conn.finish() -- cgit v1.2.3 From 3fa65c48dd2880a806985e273b3fa280103e2a7b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Aug 2015 05:39:00 +0200 Subject: manually read tls clienthello [wip] --- libmproxy/contrib/tls/__init__.py | 5 + libmproxy/contrib/tls/_constructs.py | 213 ++++++++++++++++++++ libmproxy/contrib/tls/alert_message.py | 64 ++++++ libmproxy/contrib/tls/ciphersuites.py | 343 +++++++++++++++++++++++++++++++++ libmproxy/contrib/tls/exceptions.py | 2 + libmproxy/contrib/tls/hello_message.py | 178 +++++++++++++++++ libmproxy/contrib/tls/message.py | 313 ++++++++++++++++++++++++++++++ libmproxy/contrib/tls/record.py | 110 +++++++++++ libmproxy/contrib/tls/utils.py | 52 +++++ libmproxy/protocol2/tls.py | 44 +++-- setup.py | 4 +- 11 files changed, 1316 insertions(+), 12 deletions(-) create mode 100644 libmproxy/contrib/tls/__init__.py create mode 100644 libmproxy/contrib/tls/_constructs.py create mode 100644 libmproxy/contrib/tls/alert_message.py create mode 100644 libmproxy/contrib/tls/ciphersuites.py create mode 100644 libmproxy/contrib/tls/exceptions.py create mode 100644 libmproxy/contrib/tls/hello_message.py create mode 100644 libmproxy/contrib/tls/message.py create mode 100644 libmproxy/contrib/tls/record.py create mode 100644 libmproxy/contrib/tls/utils.py diff --git a/libmproxy/contrib/tls/__init__.py b/libmproxy/contrib/tls/__init__.py new file mode 100644 index 00000000..4b540884 --- /dev/null +++ b/libmproxy/contrib/tls/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function diff --git a/libmproxy/contrib/tls/_constructs.py b/libmproxy/contrib/tls/_constructs.py new file mode 100644 index 00000000..49661efb --- /dev/null +++ b/libmproxy/contrib/tls/_constructs.py @@ -0,0 +1,213 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from construct import Array, Bytes, Struct, UBInt16, UBInt32, UBInt8, PascalString, Embed, \ + TunnelAdapter, GreedyRange, Switch + +from .utils import UBInt24 + +ProtocolVersion = Struct( + "version", + UBInt8("major"), + UBInt8("minor"), +) + +TLSPlaintext = Struct( + "TLSPlaintext", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + Bytes("fragment", lambda ctx: ctx.length), +) + +TLSCompressed = Struct( + "TLSCompressed", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + 1024 + Bytes("fragment", lambda ctx: ctx.length), +) + +TLSCiphertext = Struct( + "TLSCiphertext", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + 2048 + Bytes("fragment", lambda ctx: ctx.length), +) + +Random = Struct( + "random", + UBInt32("gmt_unix_time"), + Bytes("random_bytes", 28), +) + +SessionID = Struct( + "session_id", + UBInt8("length"), + Bytes("session_id", lambda ctx: ctx.length), +) + +CipherSuites = Struct( + "cipher_suites", + UBInt16("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length // 2, UBInt16("cipher_suites")), +) + +CompressionMethods = Struct( + "compression_methods", + UBInt8("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length, UBInt8("compression_methods")), +) + +ServerName = Struct( + "", + UBInt8("type"), + PascalString("name", length_field=UBInt16("length")), +) + +SNIExtension = Struct( + "", + TunnelAdapter( + PascalString("server_names", length_field=UBInt16("length")), + TunnelAdapter( + PascalString("", length_field=UBInt16("length")), + GreedyRange(ServerName) + ), + ), +) + +ALPNExtension = Struct( + "", + TunnelAdapter( + PascalString("alpn_protocols", length_field=UBInt16("length")), + TunnelAdapter( + PascalString("", length_field=UBInt16("length")), + GreedyRange(PascalString("name")) + ), + ), +) + +UnknownExtension = Struct( + "", + PascalString("bytes", length_field=UBInt16("extensions_length")) +) + +Extension = Struct( + "Extension", + UBInt16("type"), + Embed( + Switch( + "data", lambda ctx: ctx.type, + { + 0x00: SNIExtension, + 0x10: ALPNExtension + }, + default=UnknownExtension + ) + ) +) + +extensions = TunnelAdapter( + PascalString("extensions", length_field=UBInt16("extensions_length")), + GreedyRange(Extension) +) + +ClientHello = Struct( + "ClientHello", + ProtocolVersion, + Random, + SessionID, + CipherSuites, + CompressionMethods, + extensions, +) + +ServerHello = Struct( + "ServerHello", + ProtocolVersion, + Random, + SessionID, + Bytes("cipher_suite", 2), + UBInt8("compression_method"), + extensions, +) + +ClientCertificateType = Struct( + "certificate_types", + UBInt8("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length, UBInt8("certificate_types")), +) + +SignatureAndHashAlgorithm = Struct( + "algorithms", + UBInt8("hash"), + UBInt8("signature"), +) + +SupportedSignatureAlgorithms = Struct( + "supported_signature_algorithms", + UBInt16("supported_signature_algorithms_length"), + # TODO: Reject packets of length 0 + Array( + lambda ctx: ctx.supported_signature_algorithms_length / 2, + SignatureAndHashAlgorithm, + ), +) + +DistinguishedName = Struct( + "certificate_authorities", + UBInt16("length"), + Bytes("certificate_authorities", lambda ctx: ctx.length), +) + +CertificateRequest = Struct( + "CertificateRequest", + ClientCertificateType, + SupportedSignatureAlgorithms, + DistinguishedName, +) + +ServerDHParams = Struct( + "ServerDHParams", + UBInt16("dh_p_length"), + Bytes("dh_p", lambda ctx: ctx.dh_p_length), + UBInt16("dh_g_length"), + Bytes("dh_g", lambda ctx: ctx.dh_g_length), + UBInt16("dh_Ys_length"), + Bytes("dh_Ys", lambda ctx: ctx.dh_Ys_length), +) + +PreMasterSecret = Struct( + "pre_master_secret", + ProtocolVersion, + Bytes("random_bytes", 46), +) + +ASN1Cert = Struct( + "ASN1Cert", + UBInt32("length"), # TODO: Reject packets with length not in 1..2^24-1 + Bytes("asn1_cert", lambda ctx: ctx.length), +) + +Certificate = Struct( + "Certificate", # TODO: Reject packets with length > 2 ** 24 - 1 + UBInt32("certificates_length"), + Bytes("certificates_bytes", lambda ctx: ctx.certificates_length), +) + +Handshake = Struct( + "Handshake", + UBInt8("msg_type"), + UBInt24("length"), # TODO: Reject packets with length > 2 ** 24 + Bytes("body", lambda ctx: ctx.length), +) + +Alert = Struct( + "Alert", + UBInt8("level"), + UBInt8("description"), +) diff --git a/libmproxy/contrib/tls/alert_message.py b/libmproxy/contrib/tls/alert_message.py new file mode 100644 index 00000000..ef02f56d --- /dev/null +++ b/libmproxy/contrib/tls/alert_message.py @@ -0,0 +1,64 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +from characteristic import attributes + +from . import _constructs + + +class AlertLevel(Enum): + WARNING = 1 + FATAL = 2 + + +class AlertDescription(Enum): + CLOSE_NOTIFY = 0 + UNEXPECTED_MESSAGE = 10 + BAD_RECORD_MAC = 20 + DECRYPTION_FAILED_RESERVED = 21 + RECORD_OVERFLOW = 22 + DECOMPRESSION_FAILURE = 30 + HANDSHAKE_FAILURE = 40 + NO_CERTIFICATE_RESERVED = 41 + BAD_CERTIFICATE = 42 + UNSUPPORTED_CERTIFICATE = 43 + CERTIFICATE_REVOKED = 44 + CERTIFICATE_EXPIRED = 45 + CERTIFICATE_UNKNOWN = 46 + ILLEGAL_PARAMETER = 47 + UNKNOWN_CA = 48 + ACCESS_DENIED = 49 + DECODE_ERROR = 50 + DECRYPT_ERROR = 51 + EXPORT_RESTRICTION_RESERVED = 60 + PROTOCOL_VERSION = 70 + INSUFFICIENT_SECURITY = 71 + INTERNAL_ERROR = 80 + USER_CANCELED = 90 + NO_RENEGOTIATION = 100 + UNSUPPORTED_EXTENSION = 110 + + +@attributes(['level', 'description']) +class Alert(object): + """ + An object representing an Alert message. + """ + @classmethod + def from_bytes(cls, bytes): + """ + Parse an ``Alert`` struct. + + :param bytes: the bytes representing the input. + :return: Alert object. + """ + construct = _constructs.Alert.parse(bytes) + return cls( + level=AlertLevel(construct.level), + description=AlertDescription(construct.description) + ) diff --git a/libmproxy/contrib/tls/ciphersuites.py b/libmproxy/contrib/tls/ciphersuites.py new file mode 100644 index 00000000..86298f80 --- /dev/null +++ b/libmproxy/contrib/tls/ciphersuites.py @@ -0,0 +1,343 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +from .exceptions import UnsupportedCipherException + + +class CipherSuites(Enum): + TLS_NULL_WITH_NULL_NULL = 0x0000 + TLS_RSA_WITH_NULL_MD5 = 0x0001 + TLS_RSA_WITH_NULL_SHA = 0x0002 + TLS_RSA_EXPORT_WITH_RC4_40_MD5 = 0x0003 + TLS_RSA_WITH_RC4_128_MD5 = 0x0004 + TLS_RSA_WITH_RC4_128_SHA = 0x0005 + TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5 = 0x0006 + TLS_RSA_WITH_IDEA_CBC_SHA = 0x0007 + TLS_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0008 + TLS_RSA_WITH_DES_CBC_SHA = 0x0009 + TLS_RSA_WITH_3DES_EDE_CBC_SHA = 0x000A + TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x000B + TLS_DH_DSS_WITH_DES_CBC_SHA = 0x000C + TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA = 0x000D + TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x000E + TLS_DH_RSA_WITH_DES_CBC_SHA = 0x000F + TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA = 0x0010 + TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x0011 + TLS_DHE_DSS_WITH_DES_CBC_SHA = 0x0012 + TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA = 0x0013 + TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0014 + TLS_DHE_RSA_WITH_DES_CBC_SHA = 0x0015 + TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA = 0x0016 + TLS_DH_anon_EXPORT_WITH_RC4_40_MD5 = 0x0017 + TLS_DH_anon_WITH_RC4_128_MD5 = 0x0018 + TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA = 0x0019 + TLS_DH_anon_WITH_DES_CBC_SHA = 0x001A + TLS_DH_anon_WITH_3DES_EDE_CBC_SHA = 0x001B + TLS_KRB5_WITH_DES_CBC_SHA = 0x001E + TLS_KRB5_WITH_3DES_EDE_CBC_SHA = 0x001F + TLS_KRB5_WITH_RC4_128_SHA = 0x0020 + TLS_KRB5_WITH_IDEA_CBC_SHA = 0x0021 + TLS_KRB5_WITH_DES_CBC_MD5 = 0x0022 + TLS_KRB5_WITH_3DES_EDE_CBC_MD5 = 0x0023 + TLS_KRB5_WITH_RC4_128_MD5 = 0x0024 + TLS_KRB5_WITH_IDEA_CBC_MD5 = 0x0025 + TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA = 0x0026 + TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA = 0x0027 + TLS_KRB5_EXPORT_WITH_RC4_40_SHA = 0x0028 + TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5 = 0x0029 + TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5 = 0x002A + TLS_KRB5_EXPORT_WITH_RC4_40_MD5 = 0x002B + TLS_PSK_WITH_NULL_SHA = 0x002C + TLS_DHE_PSK_WITH_NULL_SHA = 0x002D + TLS_RSA_PSK_WITH_NULL_SHA = 0x002E + TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F + TLS_DH_DSS_WITH_AES_128_CBC_SHA = 0x0030 + TLS_DH_RSA_WITH_AES_128_CBC_SHA = 0x0031 + TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032 + TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033 + TLS_DH_anon_WITH_AES_128_CBC_SHA = 0x0034 + TLS_RSA_WITH_AES_256_CBC_SHA = 0x0035 + TLS_DH_DSS_WITH_AES_256_CBC_SHA = 0x0036 + TLS_DH_RSA_WITH_AES_256_CBC_SHA = 0x0037 + TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038 + TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039 + TLS_DH_anon_WITH_AES_256_CBC_SHA = 0x003A + TLS_RSA_WITH_NULL_SHA256 = 0x003B + TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x003C + TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D + TLS_DH_DSS_WITH_AES_128_CBC_SHA256 = 0x003E + TLS_DH_RSA_WITH_AES_128_CBC_SHA256 = 0x003F + TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040 + TLS_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0041 + TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA = 0x0042 + TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0043 + TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA = 0x0044 + TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0045 + TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA = 0x0046 + TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067 + TLS_DH_DSS_WITH_AES_256_CBC_SHA256 = 0x0068 + TLS_DH_RSA_WITH_AES_256_CBC_SHA256 = 0x0069 + TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006A + TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B + TLS_DH_anon_WITH_AES_128_CBC_SHA256 = 0x006C + TLS_DH_anon_WITH_AES_256_CBC_SHA256 = 0x006D + TLS_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0084 + TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA = 0x0085 + TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0086 + TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA = 0x0087 + TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0088 + TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA = 0x0089 + TLS_PSK_WITH_RC4_128_SHA = 0x008A + TLS_PSK_WITH_3DES_EDE_CBC_SHA = 0x008B + TLS_PSK_WITH_AES_128_CBC_SHA = 0x008C + TLS_PSK_WITH_AES_256_CBC_SHA = 0x008D + TLS_DHE_PSK_WITH_RC4_128_SHA = 0x008E + TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA = 0x008F + TLS_DHE_PSK_WITH_AES_128_CBC_SHA = 0x0090 + TLS_DHE_PSK_WITH_AES_256_CBC_SHA = 0x0091 + TLS_RSA_PSK_WITH_RC4_128_SHA = 0x0092 + TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA = 0x0093 + TLS_RSA_PSK_WITH_AES_128_CBC_SHA = 0x0094 + TLS_RSA_PSK_WITH_AES_256_CBC_SHA = 0x0095 + TLS_RSA_WITH_SEED_CBC_SHA = 0x0096 + TLS_DH_DSS_WITH_SEED_CBC_SHA = 0x0097 + TLS_DH_RSA_WITH_SEED_CBC_SHA = 0x0098 + TLS_DHE_DSS_WITH_SEED_CBC_SHA = 0x0099 + TLS_DHE_RSA_WITH_SEED_CBC_SHA = 0x009A + TLS_DH_anon_WITH_SEED_CBC_SHA = 0x009B + TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C + TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D + TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E + TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F + TLS_DH_RSA_WITH_AES_128_GCM_SHA256 = 0x00A0 + TLS_DH_RSA_WITH_AES_256_GCM_SHA384 = 0x00A1 + TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 = 0x00A2 + TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 = 0x00A3 + TLS_DH_DSS_WITH_AES_128_GCM_SHA256 = 0x00A4 + TLS_DH_DSS_WITH_AES_256_GCM_SHA384 = 0x00A5 + TLS_DH_anon_WITH_AES_128_GCM_SHA256 = 0x00A6 + TLS_DH_anon_WITH_AES_256_GCM_SHA384 = 0x00A7 + TLS_PSK_WITH_AES_128_GCM_SHA256 = 0x00A8 + TLS_PSK_WITH_AES_256_GCM_SHA384 = 0x00A9 + TLS_DHE_PSK_WITH_AES_128_GCM_SHA256 = 0x00AA + TLS_DHE_PSK_WITH_AES_256_GCM_SHA384 = 0x00AB + TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 = 0x00AC + TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 = 0x00AD + TLS_PSK_WITH_AES_128_CBC_SHA256 = 0x00AE + TLS_PSK_WITH_AES_256_CBC_SHA384 = 0x00AF + TLS_PSK_WITH_NULL_SHA256 = 0x00B0 + TLS_PSK_WITH_NULL_SHA384 = 0x00B1 + TLS_DHE_PSK_WITH_AES_128_CBC_SHA256 = 0x00B2 + TLS_DHE_PSK_WITH_AES_256_CBC_SHA384 = 0x00B3 + TLS_DHE_PSK_WITH_NULL_SHA256 = 0x00B4 + TLS_DHE_PSK_WITH_NULL_SHA384 = 0x00B5 + TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 = 0x00B6 + TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 = 0x00B7 + TLS_RSA_PSK_WITH_NULL_SHA256 = 0x00B8 + TLS_RSA_PSK_WITH_NULL_SHA384 = 0x00B9 + TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BA + TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BB + TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BC + TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BD + TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BE + TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BF + TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C0 + TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C1 + TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C2 + TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C3 + TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C4 + TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C5 + TLS_EMPTY_RENEGOTIATION_INFO_SCSV = 0x00FF + TLS_ECDH_ECDSA_WITH_NULL_SHA = 0xC001 + TLS_ECDH_ECDSA_WITH_RC4_128_SHA = 0xC002 + TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA = 0xC003 + TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA = 0xC004 + TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA = 0xC005 + TLS_ECDHE_ECDSA_WITH_NULL_SHA = 0xC006 + TLS_ECDHE_ECDSA_WITH_RC4_128_SHA = 0xC007 + TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA = 0xC008 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009 + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A + TLS_ECDH_RSA_WITH_NULL_SHA = 0xC00B + TLS_ECDH_RSA_WITH_RC4_128_SHA = 0xC00C + TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA = 0xC00D + TLS_ECDH_RSA_WITH_AES_128_CBC_SHA = 0xC00E + TLS_ECDH_RSA_WITH_AES_256_CBC_SHA = 0xC00F + TLS_ECDHE_RSA_WITH_NULL_SHA = 0xC010 + TLS_ECDHE_RSA_WITH_RC4_128_SHA = 0xC011 + TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA = 0xC012 + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013 + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014 + TLS_ECDH_anon_WITH_NULL_SHA = 0xC015 + TLS_ECDH_anon_WITH_RC4_128_SHA = 0xC016 + TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA = 0xC017 + TLS_ECDH_anon_WITH_AES_128_CBC_SHA = 0xC018 + TLS_ECDH_anon_WITH_AES_256_CBC_SHA = 0xC019 + TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA = 0xC01A + TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA = 0xC01B + TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA = 0xC01C + TLS_SRP_SHA_WITH_AES_128_CBC_SHA = 0xC01D + TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA = 0xC01E + TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA = 0xC01F + TLS_SRP_SHA_WITH_AES_256_CBC_SHA = 0xC020 + TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA = 0xC021 + TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA = 0xC022 + TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023 + TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024 + TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC025 + TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC026 + TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027 + TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028 + TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 = 0xC029 + TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 = 0xC02A + TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B + TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C + TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02D + TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02E + TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F + TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030 + TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 = 0xC031 + TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 = 0xC032 + TLS_ECDHE_PSK_WITH_RC4_128_SHA = 0xC033 + TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA = 0xC034 + TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA = 0xC035 + TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA = 0xC036 + TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 = 0xC037 + TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 = 0xC038 + TLS_ECDHE_PSK_WITH_NULL_SHA = 0xC039 + TLS_ECDHE_PSK_WITH_NULL_SHA256 = 0xC03A + TLS_ECDHE_PSK_WITH_NULL_SHA384 = 0xC03B + TLS_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC03C + TLS_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC03D + TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256 = 0xC03E + TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384 = 0xC03F + TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC040 + TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC041 + TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256 = 0xC042 + TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384 = 0xC043 + TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC044 + TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC045 + TLS_DH_anon_WITH_ARIA_128_CBC_SHA256 = 0xC046 + TLS_DH_anon_WITH_ARIA_256_CBC_SHA384 = 0xC047 + TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256 = 0xC048 + TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384 = 0xC049 + TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256 = 0xC04A + TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384 = 0xC04B + TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC04C + TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC04D + TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC04E + TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC04F + TLS_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC050 + TLS_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC051 + TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC052 + TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC053 + TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC054 + TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC055 + TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256 = 0xC056 + TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384 = 0xC057 + TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256 = 0xC058 + TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384 = 0xC059 + TLS_DH_anon_WITH_ARIA_128_GCM_SHA256 = 0xC05A + TLS_DH_anon_WITH_ARIA_256_GCM_SHA384 = 0xC05B + TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256 = 0xC05C + TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384 = 0xC05D + TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 = 0xC05E + TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 = 0xC05F + TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC060 + TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC061 + TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC062 + TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC063 + TLS_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC064 + TLS_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC065 + TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC066 + TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC067 + TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC068 + TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC069 + TLS_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06A + TLS_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06B + TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06C + TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06D + TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06E + TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06F + TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC070 + TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC071 + TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC072 + TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC073 + TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC074 + TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC075 + TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC076 + TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC077 + TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC078 + TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC079 + TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07A + TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07B + TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07C + TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07D + TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07E + TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07F + TLS_DHE_DSS_WITH_CAMELLIA_128_GCM_SHA256 = 0xC080 + TLS_DHE_DSS_WITH_CAMELLIA_256_GCM_SHA384 = 0xC081 + TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256 = 0xC082 + TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384 = 0xC083 + TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256 = 0xC084 + TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384 = 0xC085 + TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC086 + TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC087 + TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC088 + TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC089 + TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08A + TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08B + TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08C + TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08D + TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08E + TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08F + TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC090 + TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC091 + TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC092 + TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC093 + TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC094 + TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC095 + TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC096 + TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC097 + TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC098 + TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC099 + TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC09A + TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC09B + TLS_RSA_WITH_AES_128_CCM = 0xC09C + TLS_RSA_WITH_AES_256_CCM = 0xC09D + TLS_DHE_RSA_WITH_AES_128_CCM = 0xC09E + TLS_DHE_RSA_WITH_AES_256_CCM = 0xC09F + TLS_RSA_WITH_AES_128_CCM_8 = 0xC0A0 + TLS_RSA_WITH_AES_256_CCM_8 = 0xC0A1 + TLS_DHE_RSA_WITH_AES_128_CCM_8 = 0xC0A2 + TLS_DHE_RSA_WITH_AES_256_CCM_8 = 0xC0A3 + TLS_PSK_WITH_AES_128_CCM = 0xC0A4 + TLS_PSK_WITH_AES_256_CCM = 0xC0A5 + TLS_DHE_PSK_WITH_AES_128_CCM = 0xC0A6 + TLS_DHE_PSK_WITH_AES_256_CCM = 0xC0A7 + TLS_PSK_WITH_AES_128_CCM_8 = 0xC0A8 + TLS_PSK_WITH_AES_256_CCM_8 = 0xC0A9 + TLS_PSK_DHE_WITH_AES_128_CCM_8 = 0xC0AA + TLS_PSK_DHE_WITH_AES_256_CCM_8 = 0xC0AB + TLS_ECDHE_ECDSA_WITH_AES_128_CCM = 0xC0AC + TLS_ECDHE_ECDSA_WITH_AES_256_CCM = 0xC0AD + TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 = 0xC0AE + TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8 = 0xC0AF + TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCC14 + TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCC13 + + +def select_preferred_ciphersuite(client_supported, server_supported): + for i in server_supported: + assert isinstance(i, CipherSuites) + if i in client_supported: + return i + + raise UnsupportedCipherException( + "Client supported ciphersuites are not supported on the server." + ) diff --git a/libmproxy/contrib/tls/exceptions.py b/libmproxy/contrib/tls/exceptions.py new file mode 100644 index 00000000..75b34d11 --- /dev/null +++ b/libmproxy/contrib/tls/exceptions.py @@ -0,0 +1,2 @@ +class UnsupportedCipherException(Exception): + pass diff --git a/libmproxy/contrib/tls/hello_message.py b/libmproxy/contrib/tls/hello_message.py new file mode 100644 index 00000000..23cd872b --- /dev/null +++ b/libmproxy/contrib/tls/hello_message.py @@ -0,0 +1,178 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +from characteristic import attributes + +from construct import Container + +from six import BytesIO + +from . import _constructs + + +@attributes(['major', 'minor']) +class ProtocolVersion(object): + """ + An object representing a ProtocolVersion struct. + """ + + +@attributes(['gmt_unix_time', 'random_bytes']) +class Random(object): + """ + An object representing a Random struct. + """ + + +@attributes(['type', 'data']) +class Extension(object): + """ + An object representing an Extension struct. + """ + def as_bytes(self): + return _constructs.Extension.build(Container( + type=self.type.value, length=len(self.data), data=self.data)) + + +@attributes(['client_version', 'random', 'session_id', 'cipher_suites', + 'compression_methods', 'extensions']) +class ClientHello(object): + """ + An object representing a ClientHello message. + """ + def as_bytes(self): + return _constructs.ClientHello.build( + Container( + version=Container(major=self.client_version.major, + minor=self.client_version.minor), + random=Container( + gmt_unix_time=self.random.gmt_unix_time, + random_bytes=self.random.random_bytes + ), + session_id=Container(length=len(self.session_id), + session_id=self.session_id), + cipher_suites=Container(length=len(self.cipher_suites) * 2, + cipher_suites=self.cipher_suites), + compression_methods=Container( + length=len(self.compression_methods), + compression_methods=self.compression_methods + ), + extensions_length=sum([2 + 2 + len(ext.data) + for ext in self.extensions]), + extensions_bytes=b''.join( + [ext.as_bytes() for ext in self.extensions] + ) + ) + ) + + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``ClientHello`` struct. + + :param bytes: the bytes representing the input. + :return: ClientHello object. + """ + construct = _constructs.ClientHello.parse(bytes) + # XXX Is there a better way in Construct to parse an array of + # variable-length structs? + extensions = [] + extensions_io = BytesIO(construct.extensions_bytes) + while extensions_io.tell() < construct.extensions_length: + extension_construct = _constructs.Extension.parse_stream( + extensions_io) + extensions.append( + Extension(type=ExtensionType(extension_construct.type), + data=extension_construct.data)) + return ClientHello( + client_version=ProtocolVersion( + major=construct.version.major, + minor=construct.version.minor, + ), + random=Random( + gmt_unix_time=construct.random.gmt_unix_time, + random_bytes=construct.random.random_bytes, + ), + session_id=construct.session_id.session_id, + # TODO: cipher suites should be enums + cipher_suites=construct.cipher_suites.cipher_suites, + compression_methods=( + construct.compression_methods.compression_methods + ), + extensions=extensions, + ) + + +class ExtensionType(Enum): + SIGNATURE_ALGORITHMS = 13 + # XXX: See http://tools.ietf.org/html/rfc5246#ref-TLSEXT + + +@attributes(['server_version', 'random', 'session_id', 'cipher_suite', + 'compression_method', 'extensions']) +class ServerHello(object): + """ + An object representing a ServerHello message. + """ + def as_bytes(self): + return _constructs.ServerHello.build( + Container( + version=Container(major=self.server_version.major, + minor=self.server_version.minor), + random=Container( + gmt_unix_time=self.random.gmt_unix_time, + random_bytes=self.random.random_bytes + ), + session_id=Container(length=len(self.session_id), + session_id=self.session_id), + cipher_suite=self.cipher_suite, + compression_method=self.compression_method.value, + extensions_length=sum([2 + 2 + len(ext.data) + for ext in self.extensions]), + extensions_bytes=b''.join( + [ext.as_bytes() for ext in self.extensions] + ) + ) + ) + + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``ServerHello`` struct. + + :param bytes: the bytes representing the input. + :return: ServerHello object. + """ + construct = _constructs.ServerHello.parse(bytes) + # XXX: Find a better way to parse extensions + extensions = [] + extensions_io = BytesIO(construct.extensions_bytes) + while extensions_io.tell() < construct.extensions_length: + extension_construct = _constructs.Extension.parse_stream( + extensions_io) + extensions.append( + Extension(type=ExtensionType(extension_construct.type), + data=extension_construct.data)) + return ServerHello( + server_version=ProtocolVersion( + major=construct.version.major, + minor=construct.version.minor, + ), + random=Random( + gmt_unix_time=construct.random.gmt_unix_time, + random_bytes=construct.random.random_bytes, + ), + session_id=construct.session_id.session_id, + cipher_suite=construct.cipher_suite, + compression_method=CompressionMethod(construct.compression_method), + extensions=extensions, + ) + + +class CompressionMethod(Enum): + NULL = 0 diff --git a/libmproxy/contrib/tls/message.py b/libmproxy/contrib/tls/message.py new file mode 100644 index 00000000..b372859f --- /dev/null +++ b/libmproxy/contrib/tls/message.py @@ -0,0 +1,313 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +from characteristic import attributes + +from construct import Container + +from six import BytesIO + +from . import _constructs + +from .hello_message import ( + ClientHello, ProtocolVersion, ServerHello +) + + +class ClientCertificateType(Enum): + RSA_SIGN = 1 + DSS_SIGN = 2 + RSA_FIXED_DH = 3 + DSS_FIXED_DH = 4 + RSA_EPHEMERAL_DH_RESERVED = 5 + DSS_EPHEMERAL_DH_RESERVED = 6 + FORTEZZA_DMS_RESERVED = 20 + + +class HashAlgorithm(Enum): + NONE = 0 + MD5 = 1 + SHA1 = 2 + SHA224 = 3 + SHA256 = 4 + SHA384 = 5 + SHA512 = 6 + + +class SignatureAlgorithm(Enum): + ANONYMOUS = 0 + RSA = 1 + DSA = 2 + ECDSA = 3 + + +class HandshakeType(Enum): + HELLO_REQUEST = 0 + CLIENT_HELLO = 1 + SERVER_HELLO = 2 + CERTIFICATE = 11 + SERVER_KEY_EXCHANGE = 12 + CERTIFICATE_REQUEST = 13 + SERVER_HELLO_DONE = 14 + CERTIFICATE_VERIFY = 15 + CLIENT_KEY_EXCHANGE = 16 + FINISHED = 20 + + +class HelloRequest(object): + """ + An object representing a HelloRequest struct. + """ + def as_bytes(self): + return b'' + + +class ServerHelloDone(object): + """ + An object representing a ServerHelloDone struct. + """ + def as_bytes(self): + return b'' + + +@attributes(['certificate_types', 'supported_signature_algorithms', + 'certificate_authorities']) +class CertificateRequest(object): + """ + An object representing a CertificateRequest struct. + """ + def as_bytes(self): + return _constructs.CertificateRequest.build(Container( + certificate_types=Container( + length=len(self.certificate_types), + certificate_types=[cert_type.value + for cert_type in self.certificate_types] + ), + supported_signature_algorithms=Container( + supported_signature_algorithms_length=2 * len( + self.supported_signature_algorithms + ), + algorithms=[Container( + hash=algorithm.hash.value, + signature=algorithm.signature.value, + ) + for algorithm in self.supported_signature_algorithms + ] + ), + certificate_authorities=Container( + length=len(self.certificate_authorities), + certificate_authorities=self.certificate_authorities + ) + )) + + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``CertificateRequest`` struct. + + :param bytes: the bytes representing the input. + :return: CertificateRequest object. + """ + construct = _constructs.CertificateRequest.parse(bytes) + return cls( + certificate_types=[ + ClientCertificateType(cert_type) + for cert_type in construct.certificate_types.certificate_types + ], + supported_signature_algorithms=[ + SignatureAndHashAlgorithm( + hash=HashAlgorithm(algorithm.hash), + signature=SignatureAlgorithm(algorithm.signature), + ) + for algorithm in ( + construct.supported_signature_algorithms.algorithms + ) + ], + certificate_authorities=( + construct.certificate_authorities.certificate_authorities + ) + ) + + +@attributes(['hash', 'signature']) +class SignatureAndHashAlgorithm(object): + """ + An object representing a SignatureAndHashAlgorithm struct. + """ + + +@attributes(['dh_p', 'dh_g', 'dh_Ys']) +class ServerDHParams(object): + """ + An object representing a ServerDHParams struct. + """ + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``ServerDHParams`` struct. + + :param bytes: the bytes representing the input. + :return: ServerDHParams object. + """ + construct = _constructs.ServerDHParams.parse(bytes) + return cls( + dh_p=construct.dh_p, + dh_g=construct.dh_g, + dh_Ys=construct.dh_Ys + ) + + +@attributes(['client_version', 'random']) +class PreMasterSecret(object): + """ + An object representing a PreMasterSecret struct. + """ + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``PreMasterSecret`` struct. + + :param bytes: the bytes representing the input. + :return: CertificateRequest object. + """ + construct = _constructs.PreMasterSecret.parse(bytes) + return cls( + client_version=ProtocolVersion( + major=construct.version.major, + minor=construct.version.minor, + ), + random=construct.random_bytes, + ) + + +@attributes(['asn1_cert']) +class ASN1Cert(object): + """ + An object representing ASN.1 Certificate + """ + def as_bytes(self): + return _constructs.ASN1Cert.build(Container( + length=len(self.asn1_cert), + asn1_cert=self.asn1_cert + )) + + +@attributes(['certificate_list']) +class Certificate(object): + """ + An object representing a Certificate struct. + """ + def as_bytes(self): + return _constructs.Certificate.build(Container( + certificates_length=sum([4 + len(asn1cert.asn1_cert) + for asn1cert in self.certificate_list]), + certificates_bytes=b''.join( + [asn1cert.as_bytes() for asn1cert in self.certificate_list] + ) + + )) + + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``Certificate`` struct. + + :param bytes: the bytes representing the input. + :return: Certificate object. + """ + construct = _constructs.Certificate.parse(bytes) + # XXX: Find a better way to parse an array of variable-length objects + certificates = [] + certificates_io = BytesIO(construct.certificates_bytes) + + while certificates_io.tell() < construct.certificates_length: + certificate_construct = _constructs.ASN1Cert.parse_stream( + certificates_io + ) + certificates.append( + ASN1Cert(asn1_cert=certificate_construct.asn1_cert) + ) + return cls( + certificate_list=certificates + ) + + +@attributes(['verify_data']) +class Finished(object): + def as_bytes(self): + return self.verify_data + + +@attributes(['msg_type', 'length', 'body']) +class Handshake(object): + """ + An object representing a Handshake struct. + """ + def as_bytes(self): + if self.msg_type in [ + HandshakeType.SERVER_HELLO, HandshakeType.CLIENT_HELLO, + HandshakeType.CERTIFICATE, HandshakeType.CERTIFICATE_REQUEST, + HandshakeType.HELLO_REQUEST, HandshakeType.SERVER_HELLO_DONE, + HandshakeType.FINISHED + ]: + _body_as_bytes = self.body.as_bytes() + else: + _body_as_bytes = b'' + return _constructs.Handshake.build( + Container( + msg_type=self.msg_type.value, + length=self.length, + body=_body_as_bytes + ) + ) + + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``Handshake`` struct. + + :param bytes: the bytes representing the input. + :return: Handshake object. + """ + construct = _constructs.Handshake.parse(bytes) + return cls( + msg_type=HandshakeType(construct.msg_type), + length=construct.length, + body=cls._get_handshake_message( + HandshakeType(construct.msg_type), construct.body + ), + ) + + @staticmethod + def _get_handshake_message(msg_type, body): + _handshake_message_parser = { + HandshakeType.CLIENT_HELLO: ClientHello.from_bytes, + HandshakeType.SERVER_HELLO: ServerHello.from_bytes, + HandshakeType.CERTIFICATE: Certificate.from_bytes, + # 12: parse_server_key_exchange, + HandshakeType.CERTIFICATE_REQUEST: CertificateRequest.from_bytes, + # 15: parse_certificate_verify, + # 16: parse_client_key_exchange, + } + + try: + if msg_type == HandshakeType.HELLO_REQUEST: + return HelloRequest() + elif msg_type == HandshakeType.SERVER_HELLO_DONE: + return ServerHelloDone() + elif msg_type == HandshakeType.FINISHED: + return Finished(verify_data=body) + elif msg_type in [HandshakeType.SERVER_KEY_EXCHANGE, + HandshakeType.CERTIFICATE_VERIFY, + HandshakeType.CLIENT_KEY_EXCHANGE, + ]: + raise NotImplementedError + else: + return _handshake_message_parser[msg_type](body) + except NotImplementedError: + return None # TODO diff --git a/libmproxy/contrib/tls/record.py b/libmproxy/contrib/tls/record.py new file mode 100644 index 00000000..481c93bc --- /dev/null +++ b/libmproxy/contrib/tls/record.py @@ -0,0 +1,110 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from enum import Enum + +from characteristic import attributes + +from construct import Container + +from . import _constructs + + +@attributes(['major', 'minor']) +class ProtocolVersion(object): + """ + An object representing a ProtocolVersion struct. + """ + + +@attributes(['type', 'version', 'fragment']) +class TLSPlaintext(object): + """ + An object representing a TLSPlaintext struct. + """ + def as_bytes(self): + return _constructs.TLSPlaintext.build( + Container( + type=self.type.value, + version=Container(major=self.version.major, + minor=self.version.minor), + length=len(self.fragment), + fragment=self.fragment + ) + ) + + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``TLSPlaintext`` struct. + + :param bytes: the bytes representing the input. + :return: TLSPlaintext object. + """ + construct = _constructs.TLSPlaintext.parse(bytes) + return cls( + type=ContentType(construct.type), + version=ProtocolVersion( + major=construct.version.major, + minor=construct.version.minor + ), + fragment=construct.fragment + ) + + +@attributes(['type', 'version', 'fragment']) +class TLSCompressed(object): + """ + An object representing a TLSCompressed struct. + """ + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``TLSCompressed`` struct. + + :param bytes: the bytes representing the input. + :return: TLSCompressed object. + """ + construct = _constructs.TLSCompressed.parse(bytes) + return cls( + type=ContentType(construct.type), + version=ProtocolVersion( + major=construct.version.major, + minor=construct.version.minor + ), + fragment=construct.fragment + ) + + +@attributes(['type', 'version', 'fragment']) +class TLSCiphertext(object): + """ + An object representing a TLSCiphertext struct. + """ + @classmethod + def from_bytes(cls, bytes): + """ + Parse a ``TLSCiphertext`` struct. + + :param bytes: the bytes representing the input. + :return: TLSCiphertext object. + """ + construct = _constructs.TLSCiphertext.parse(bytes) + return cls( + type=ContentType(construct.type), + version=ProtocolVersion( + major=construct.version.major, + minor=construct.version.minor + ), + fragment=construct.fragment + ) + + +class ContentType(Enum): + CHANGE_CIPHER_SPEC = 20 + ALERT = 21 + HANDSHAKE = 22 + APPLICATION_DATA = 23 diff --git a/libmproxy/contrib/tls/utils.py b/libmproxy/contrib/tls/utils.py new file mode 100644 index 00000000..a971af49 --- /dev/null +++ b/libmproxy/contrib/tls/utils.py @@ -0,0 +1,52 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import construct + +import six + + +class _UBInt24(construct.Adapter): + def _encode(self, obj, context): + return ( + six.int2byte((obj & 0xFF0000) >> 16) + + six.int2byte((obj & 0x00FF00) >> 8) + + six.int2byte(obj & 0x0000FF) + ) + + def _decode(self, obj, context): + obj = bytearray(obj) + return (obj[0] << 16 | obj[1] << 8 | obj[2]) + + +def UBInt24(name): # noqa + return _UBInt24(construct.Bytes(name, 3)) + + +def LengthPrefixedArray(subcon, length_field=construct.UBInt8("length")): + """ + An array prefixed by a byte length field. + + In contrast to construct.macros.PrefixedArray, + the length field signifies the number of bytes, not the number of elements. + """ + subcon_with_pos = construct.Struct( + subcon.name, + construct.Embed(subcon), + construct.Anchor("__current_pos") + ) + + return construct.Embed( + construct.Struct( + "", + length_field, + construct.Anchor("__start_pos"), + construct.RepeatUntil( + lambda obj, ctx: obj.__current_pos == ctx.__start_pos + getattr(ctx, length_field.name), + subcon_with_pos + ), + ) + ) \ No newline at end of file diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 98c5d603..78372284 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -1,6 +1,6 @@ from __future__ import (absolute_import, print_function, division) -import traceback +from ..contrib.tls._constructs import ClientHello from netlib import tcp import netlib.http.http2 @@ -11,16 +11,16 @@ from .layer import Layer class TlsLayer(Layer): def __init__(self, ctx, client_tls, server_tls): + self.client_sni = None + self.client_alpn_protos = None + super(TlsLayer, self).__init__(ctx) self._client_tls = client_tls self._server_tls = server_tls - self.client_sni = None + self._sni_from_server_change = None - self.client_alpn_protos = None self.__server_tls_exception = None - # foo alpn protos = [netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1, netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], # TODO: read this from client_conn first - def __call__(self): """ The strategy for establishing SSL is as follows: @@ -45,6 +45,28 @@ class TlsLayer(Layer): https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 """ + import struct + + # Read all records that contain the initial Client Hello message. + client_hello = "" + client_hello_size = 1 + offset = 0 + while len(client_hello) < client_hello_size: + record_header = self.client_conn.rfile.peek(offset+5)[offset:] + record_size = struct.unpack("!H", record_header[3:])[0] + 5 + record_body = self.client_conn.rfile.peek(offset+record_size)[offset+5:] + client_hello += record_body + offset += record_size + client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 + + client_hello = ClientHello.parse(client_hello[4:]) + + for extension in client_hello.extensions: + if extension.type == 0x00: + host = extension.server_names[0].name + if extension.type == 0x10: + alpn = extension.alpn_protocols + client_tls_requires_server_cert = ( self._client_tls and self._server_tls and not self.config.no_upstream_cert ) @@ -60,12 +82,12 @@ class TlsLayer(Layer): def connect(self): if not self.server_conn: self.ctx.connect() - if self._server_tls and not self._server_tls_established: + if self._server_tls and not self.server_conn.tls_established: self._establish_tls_with_server() def reconnect(self): self.ctx.reconnect() - if self._server_tls and not self._server_tls_established: + if self._server_tls and not self.server_conn.tls_established: self._establish_tls_with_server() def set_server(self, address, server_tls, sni, depth=1): @@ -74,10 +96,6 @@ class TlsLayer(Layer): self._sni_from_server_change = sni self._server_tls = server_tls - @property - def _server_tls_established(self): - return self.server_conn and self.server_conn.tls_established - @property def sni_for_upstream_connection(self): if self._sni_from_server_change is False: @@ -138,6 +156,10 @@ class TlsLayer(Layer): connection.set_context(new_context) def __handle_alpn_select(self, conn_, options): + """ + Once the client signals the alternate protocols it supports, + we reconnect upstream with the same list and pass the server's choice down to the client. + """ # TODO: change to something meaningful? # alpn_preference = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 diff --git a/setup.py b/setup.py index da080bc1..876b4f9c 100644 --- a/setup.py +++ b/setup.py @@ -20,7 +20,9 @@ deps = { "pyperclip>=1.5.8", "blinker>=1.3", "pyparsing>=1.5.2", - "html2text>=2015.4.14" + "html2text>=2015.4.14", + "construct>=2.5.2", + "six>=1.9.0", } # A script -> additional dependencies dict. scripts = { -- cgit v1.2.3 From 1093d185ec78cdfff4fb425b902a52f61991cf5e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Aug 2015 06:38:03 +0200 Subject: manually read tls clienthello --- libmproxy/contrib/tls/_constructs.py | 4 +- libmproxy/protocol2/root_context.py | 3 +- libmproxy/protocol2/tls.py | 157 ++++++++++++----------------------- 3 files changed, 57 insertions(+), 107 deletions(-) diff --git a/libmproxy/contrib/tls/_constructs.py b/libmproxy/contrib/tls/_constructs.py index 49661efb..a5f8b524 100644 --- a/libmproxy/contrib/tls/_constructs.py +++ b/libmproxy/contrib/tls/_constructs.py @@ -101,7 +101,7 @@ Extension = Struct( UBInt16("type"), Embed( Switch( - "data", lambda ctx: ctx.type, + "", lambda ctx: ctx.type, { 0x00: SNIExtension, 0x10: ALPNExtension @@ -202,7 +202,7 @@ Certificate = Struct( Handshake = Struct( "Handshake", UBInt8("msg_type"), - UBInt24("length"), # TODO: Reject packets with length > 2 ** 24 + UBInt24("length"), Bytes("body", lambda ctx: ctx.length), ) diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index d0c62be4..880bc160 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -47,9 +47,10 @@ class RootContext(object): d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) + alpn_proto_negotiated = top_layer.client_conn.get_alpn_proto_negotiated() is_alpn_h2_negotiated = ( isinstance(top_layer, TlsLayer) and - top_layer.client_conn.get_alpn_proto_negotiated() == HTTP2Protocol.ALPN_PROTO_H2 + alpn_proto_negotiated == HTTP2Protocol.ALPN_PROTO_H2 ) if is_tls_client_hello: diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 78372284..24b8989d 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -1,10 +1,12 @@ from __future__ import (absolute_import, print_function, division) -from ..contrib.tls._constructs import ClientHello +import struct +from construct import ConstructError from netlib import tcp import netlib.http.http2 +from ..contrib.tls._constructs import ClientHello from ..exceptions import ProtocolException from .layer import Layer @@ -12,14 +14,13 @@ from .layer import Layer class TlsLayer(Layer): def __init__(self, ctx, client_tls, server_tls): self.client_sni = None - self.client_alpn_protos = None + self.client_alpn_protocols = None super(TlsLayer, self).__init__(ctx) self._client_tls = client_tls self._server_tls = server_tls self._sni_from_server_change = None - self.__server_tls_exception = None def __call__(self): """ @@ -35,18 +36,31 @@ class TlsLayer(Layer): 3. Pause the client handshake, establish SSL with the server. 4. Finish the client handshake with the certificate from the server. There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( - Thus, we resort to the following workaround when establishing SSL with the server: - 1. Try to establish SSL with the server without SNI. If this fails, we ignore it. - 2. Establish SSL with client. - - If there's a SNI callback, reconnect to the server with SNI. - - If not and the server connect failed, raise the original exception. + Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values. + Further notes: - OpenSSL 1.0.2 introduces a callback that would help here: https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 """ - import struct + client_tls_requires_server_cert = ( + self._client_tls and self._server_tls and not self.config.no_upstream_cert + ) + + self._parse_client_hello() + + if client_tls_requires_server_cert: + self.ctx.connect() + self._establish_tls_with_server() + self._establish_tls_with_client() + elif self._client_tls: + self._establish_tls_with_client() + + layer = self.ctx.next_layer(self) + layer() + + def _get_client_hello(self): # Read all records that contain the initial Client Hello message. client_hello = "" client_hello_size = 1 @@ -58,26 +72,25 @@ class TlsLayer(Layer): client_hello += record_body offset += record_size client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 + return client_hello - client_hello = ClientHello.parse(client_hello[4:]) + def _parse_client_hello(self): + try: + client_hello = ClientHello.parse(self._get_client_hello()[4:]) + except ConstructError as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + return for extension in client_hello.extensions: if extension.type == 0x00: - host = extension.server_names[0].name - if extension.type == 0x10: - alpn = extension.alpn_protocols - - client_tls_requires_server_cert = ( - self._client_tls and self._server_tls and not self.config.no_upstream_cert - ) - - if client_tls_requires_server_cert: - self._establish_tls_with_client_and_server() - elif self._client_tls: - self._establish_tls_with_client() + if len(extension.server_names) != 1 or extension.server_names[0].type != 0: + self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") + self.client_sni = extension.server_names[0].name + elif extension.type == 0x10: + self.client_alpn_protocols = extension.alpn_protocols - layer = self.ctx.next_layer(self) - layer() + print("sni: %s" % self.client_sni) + print("alpn: %s" % self.client_alpn_protocols) def connect(self): if not self.server_conn: @@ -97,88 +110,31 @@ class TlsLayer(Layer): self._server_tls = server_tls @property - def sni_for_upstream_connection(self): + def sni_for_server_connection(self): if self._sni_from_server_change is False: return None else: return self._sni_from_server_change or self.client_sni - def _establish_tls_with_client_and_server(self): - """ - This function deals with the problem that the server may require a SNI value from the client. - """ - - # First, try to connect to the server. - self.ctx.connect() - server_err = None - try: - self._establish_tls_with_server() - except ProtocolException as e: - server_err = e - - self._establish_tls_with_client() - - if server_err and not self.client_sni: - raise server_err - - def __handle_sni(self, connection): - """ - This callback gets called during the TLS handshake with the client. - The client has just sent the Sever Name Indication (SNI). - """ - old_upstream_sni = self.sni_for_upstream_connection - - sn = connection.get_servername() - if not sn: - return - - self.client_sni = sn.decode("utf8").encode("idna") - - server_sni_changed = (old_upstream_sni != self.sni_for_upstream_connection) - server_conn_with_tls_exists = (self.server_conn and self._server_tls) - if server_sni_changed and server_conn_with_tls_exists: - try: - self.reconnect() - except Exception as e: - self.__server_tls_exception = e - - # Now, change client context to reflect possibly changed certificate: - cert, key, chain_file = self._find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file, - alpn_select_callback=self.__handle_alpn_select, - ) - connection.set_context(new_context) + @property + def alpn_for_client_connection(self): + return self.server_conn.get_alpn_proto_negotiated() - def __handle_alpn_select(self, conn_, options): + def __alpn_select_callback(self, conn_, options): """ Once the client signals the alternate protocols it supports, we reconnect upstream with the same list and pass the server's choice down to the client. """ - # TODO: change to something meaningful? - # alpn_preference = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 - alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 - - # TODO: Don't reconnect twice? - upstream_alpn_changed = (self.client_alpn_protos != options) - server_conn_with_tls_exists = (self.server_conn and self._server_tls) - if upstream_alpn_changed and server_conn_with_tls_exists: - try: - self.reconnect() - except Exception as e: - self.__server_tls_exception = e - self.client_alpn_protos = options + # This gets triggered if we haven't established an upstream connection yet. + default_alpn = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 + # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 - if alpn_preference in options: - return bytes(alpn_preference) - else: # pragma no cover - return options[0] + if self.alpn_for_client_connection in options: + return bytes(self.alpn_for_client_connection) + if default_alpn in options: + return bytes(default_alpn) + return options[0] def _establish_tls_with_client(self): self.log("Establish TLS with client", "debug") @@ -189,34 +145,27 @@ class TlsLayer(Layer): cert, key, method=self.config.openssl_method_client, options=self.config.openssl_options_client, - handle_sni=self.__handle_sni, cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, chain_file=chain_file, - alpn_select_callback=self.__handle_alpn_select, + alpn_select_callback=self.__alpn_select_callback, ) except tcp.NetLibError as e: - print("alpn: %s" % self.client_alpn_protos) raise ProtocolException(repr(e), e) - # Do not raise server tls exceptions immediately. - # We want to try to finish the client handshake so that other layers can send error messages over it. - if self.__server_tls_exception: - raise self.__server_tls_exception - def _establish_tls_with_server(self): self.log("Establish TLS with server", "debug") try: self.server_conn.establish_ssl( self.config.clientcerts, - self.sni_for_upstream_connection, + self.sni_for_server_connection, method=self.config.openssl_method_server, options=self.config.openssl_options_server, verify_options=self.config.openssl_verification_mode_server, ca_path=self.config.openssl_trusted_cadir_server, ca_pemfile=self.config.openssl_trusted_ca_server, cipher_list=self.config.ciphers_server, - alpn_protos=self.client_alpn_protos, + alpn_protos=self.client_alpn_protocols, ) tls_cert_err = self.server_conn.ssl_verification_error if tls_cert_err is not None: -- cgit v1.2.3 From 158906444f48e5afc164b641bf2021a0d29267ee Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 26 Aug 2015 09:33:19 +0200 Subject: fix return value and empty requests --- libmproxy/protocol2/http.py | 1 + 1 file changed, 1 insertion(+) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 5a25c317..952f6246 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -78,6 +78,7 @@ class Http2Layer(Layer): body_size_limit=self.config.body_size_limit ) self._stream_id = request.stream_id + return request def read_from_server(self, request_method): return HTTPResponse.from_protocol( -- cgit v1.2.3 From aebe34202553bea24a5d4e99b9f218b58559c0f0 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Aug 2015 14:03:51 +0200 Subject: improve alpn handling --- libmproxy/protocol2/http.py | 9 ++++++--- libmproxy/protocol2/tls.py | 26 ++++++++++++++++++-------- 2 files changed, 24 insertions(+), 11 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 5a25c317..649e7843 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -78,13 +78,14 @@ class Http2Layer(Layer): body_size_limit=self.config.body_size_limit ) self._stream_id = request.stream_id + return request def read_from_server(self, request_method): return HTTPResponse.from_protocol( self.server_protocol, request_method, body_size_limit=self.config.body_size_limit, - include_body=False, + include_body=True, stream_id=self._stream_id ) @@ -389,9 +390,11 @@ class HttpLayer(Layer): if flow is None or flow == KILL: raise Kill() - if flow.response.stream: + if isinstance(self.ctx, Http2Layer): + pass # streaming is not implemented for http2 yet. + elif flow.response.stream: flow.response.content = CONTENT_MISSING - elif isinstance(self.server_protocol, http1.HTTP1Protocol): + else: flow.response.content = self.server_protocol.read_http_body( flow.response.headers, self.config.body_size_limit, diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 24b8989d..96ee643f 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -87,10 +87,9 @@ class TlsLayer(Layer): self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") self.client_sni = extension.server_names[0].name elif extension.type == 0x10: - self.client_alpn_protocols = extension.alpn_protocols + self.client_alpn_protocols = list(extension.alpn_protocols) - print("sni: %s" % self.client_sni) - print("alpn: %s" % self.client_alpn_protocols) + self.log("Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), "debug") def connect(self): if not self.server_conn: @@ -131,10 +130,13 @@ class TlsLayer(Layer): # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 if self.alpn_for_client_connection in options: - return bytes(self.alpn_for_client_connection) - if default_alpn in options: - return bytes(default_alpn) - return options[0] + choice = bytes(self.alpn_for_client_connection) + elif default_alpn in options: + choice = bytes(default_alpn) + else: + choice = options[0] + self.log("ALPN for client: %s" % choice, "debug") + return choice def _establish_tls_with_client(self): self.log("Establish TLS with client", "debug") @@ -156,6 +158,12 @@ class TlsLayer(Layer): def _establish_tls_with_server(self): self.log("Establish TLS with server", "debug") try: + # We only support http/1.1 and h2. + # If the server only supports spdy (next to http/1.1), it may select that + # and mitmproxy would enter TCP passthrough mode, which we want to avoid. + deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") + alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + self.server_conn.establish_ssl( self.config.clientcerts, self.sni_for_server_connection, @@ -165,7 +173,7 @@ class TlsLayer(Layer): ca_path=self.config.openssl_trusted_cadir_server, ca_pemfile=self.config.openssl_trusted_ca_server, cipher_list=self.config.ciphers_server, - alpn_protos=self.client_alpn_protocols, + alpn_protos=alpn, ) tls_cert_err = self.server_conn.ssl_verification_error if tls_cert_err is not None: @@ -185,6 +193,8 @@ class TlsLayer(Layer): except tcp.NetLibError as e: raise ProtocolException(repr(e), e) + self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") + def _find_cert(self): host = self.server_conn.address.host sans = set() -- cgit v1.2.3 From 778644d4b810e87ce20cf9da1dca55913c2ffd07 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Aug 2015 15:12:04 +0200 Subject: http2: fix bugs, chrome works :tada: --- libmproxy/protocol2/http.py | 2 +- libmproxy/protocol2/tls.py | 6 +++--- libmproxy/proxy/config.py | 5 ++++- 3 files changed, 8 insertions(+), 5 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 649e7843..e3878fa6 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -279,7 +279,7 @@ class HttpLayer(Layer): if isinstance(e, ProtocolException): raise e else: - raise ProtocolException(repr(e), e) + raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) finally: flow.live = False diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 96ee643f..ce684eb9 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -153,7 +153,7 @@ class TlsLayer(Layer): alpn_select_callback=self.__alpn_select_callback, ) except tcp.NetLibError as e: - raise ProtocolException(repr(e), e) + raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) def _establish_tls_with_server(self): self.log("Establish TLS with server", "debug") @@ -189,9 +189,9 @@ class TlsLayer(Layer): (tls_cert_err['depth'], tls_cert_err['errno']), "error") self.log("Aborting connection attempt", "error") - raise ProtocolException(repr(e), e) + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) except tcp.NetLibError as e: - raise ProtocolException(repr(e), e) + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index ec91a6e0..4ca15747 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -14,6 +14,9 @@ TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" +# We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. +# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old +DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" class HostMatcher(object): def __init__(self, patterns=[]): @@ -241,7 +244,7 @@ def ssl_option_group(parser): 'Can be passed multiple times.') group.add_argument( "--ciphers-client", action="store", - type=str, dest="ciphers_client", default=None, + type=str, dest="ciphers_client", default=DEFAULT_CLIENT_CIPHERS, help="Set supported ciphers for client connections. (OpenSSL Syntax)" ) group.add_argument( -- cgit v1.2.3 From 2cfc1b1b4030838f6047f18f8014c91926b414d0 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Aug 2015 20:48:59 +0200 Subject: fix non-alpn clients --- libmproxy/protocol2/tls.py | 5 ++++- 1 file changed, 4 insertions(+), 1 deletion(-) diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index ce684eb9..7ef0ad8c 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -162,7 +162,10 @@ class TlsLayer(Layer): # If the server only supports spdy (next to http/1.1), it may select that # and mitmproxy would enter TCP passthrough mode, which we want to avoid. deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") - alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + if self.client_alpn_protocols: + alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + else: + alpn = None self.server_conn.establish_ssl( self.config.clientcerts, -- cgit v1.2.3 From 9c6b3eb58a22817daa576063c3626d7a239e7093 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 26 Aug 2015 22:00:50 +0200 Subject: clean up clienthello parsing --- libmproxy/contrib/README | 4 + libmproxy/contrib/tls/_constructs.py | 6 +- libmproxy/contrib/tls/alert_message.py | 64 ------ libmproxy/contrib/tls/ciphersuites.py | 343 --------------------------------- libmproxy/contrib/tls/exceptions.py | 2 - libmproxy/contrib/tls/hello_message.py | 178 ----------------- libmproxy/contrib/tls/message.py | 313 ------------------------------ libmproxy/contrib/tls/record.py | 110 ----------- libmproxy/contrib/tls/utils.py | 26 --- libmproxy/protocol2/tls.py | 14 +- 10 files changed, 19 insertions(+), 1041 deletions(-) delete mode 100644 libmproxy/contrib/tls/alert_message.py delete mode 100644 libmproxy/contrib/tls/ciphersuites.py delete mode 100644 libmproxy/contrib/tls/exceptions.py delete mode 100644 libmproxy/contrib/tls/hello_message.py delete mode 100644 libmproxy/contrib/tls/message.py delete mode 100644 libmproxy/contrib/tls/record.py diff --git a/libmproxy/contrib/README b/libmproxy/contrib/README index 3b0f7512..e339310a 100644 --- a/libmproxy/contrib/README +++ b/libmproxy/contrib/README @@ -8,3 +8,7 @@ jsbeautifier, git checkout 25/03/12, MIT license wbxml - https://github.com/davidpshaw/PyWBXMLDecoder + +tls, BSD license + - https://github.com/mhils/tls/tree/extension-parsing + - limited to required files. \ No newline at end of file diff --git a/libmproxy/contrib/tls/_constructs.py b/libmproxy/contrib/tls/_constructs.py index a5f8b524..9c57a799 100644 --- a/libmproxy/contrib/tls/_constructs.py +++ b/libmproxy/contrib/tls/_constructs.py @@ -4,8 +4,8 @@ from __future__ import absolute_import, division, print_function -from construct import Array, Bytes, Struct, UBInt16, UBInt32, UBInt8, PascalString, Embed, \ - TunnelAdapter, GreedyRange, Switch +from construct import (Array, Bytes, Struct, UBInt16, UBInt32, UBInt8, PascalString, Embed, TunnelAdapter, GreedyRange, + Switch, OptionalGreedyRange) from .utils import UBInt24 @@ -113,7 +113,7 @@ Extension = Struct( extensions = TunnelAdapter( PascalString("extensions", length_field=UBInt16("extensions_length")), - GreedyRange(Extension) + OptionalGreedyRange(Extension) ) ClientHello = Struct( diff --git a/libmproxy/contrib/tls/alert_message.py b/libmproxy/contrib/tls/alert_message.py deleted file mode 100644 index ef02f56d..00000000 --- a/libmproxy/contrib/tls/alert_message.py +++ /dev/null @@ -1,64 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from __future__ import absolute_import, division, print_function - -from enum import Enum - -from characteristic import attributes - -from . import _constructs - - -class AlertLevel(Enum): - WARNING = 1 - FATAL = 2 - - -class AlertDescription(Enum): - CLOSE_NOTIFY = 0 - UNEXPECTED_MESSAGE = 10 - BAD_RECORD_MAC = 20 - DECRYPTION_FAILED_RESERVED = 21 - RECORD_OVERFLOW = 22 - DECOMPRESSION_FAILURE = 30 - HANDSHAKE_FAILURE = 40 - NO_CERTIFICATE_RESERVED = 41 - BAD_CERTIFICATE = 42 - UNSUPPORTED_CERTIFICATE = 43 - CERTIFICATE_REVOKED = 44 - CERTIFICATE_EXPIRED = 45 - CERTIFICATE_UNKNOWN = 46 - ILLEGAL_PARAMETER = 47 - UNKNOWN_CA = 48 - ACCESS_DENIED = 49 - DECODE_ERROR = 50 - DECRYPT_ERROR = 51 - EXPORT_RESTRICTION_RESERVED = 60 - PROTOCOL_VERSION = 70 - INSUFFICIENT_SECURITY = 71 - INTERNAL_ERROR = 80 - USER_CANCELED = 90 - NO_RENEGOTIATION = 100 - UNSUPPORTED_EXTENSION = 110 - - -@attributes(['level', 'description']) -class Alert(object): - """ - An object representing an Alert message. - """ - @classmethod - def from_bytes(cls, bytes): - """ - Parse an ``Alert`` struct. - - :param bytes: the bytes representing the input. - :return: Alert object. - """ - construct = _constructs.Alert.parse(bytes) - return cls( - level=AlertLevel(construct.level), - description=AlertDescription(construct.description) - ) diff --git a/libmproxy/contrib/tls/ciphersuites.py b/libmproxy/contrib/tls/ciphersuites.py deleted file mode 100644 index 86298f80..00000000 --- a/libmproxy/contrib/tls/ciphersuites.py +++ /dev/null @@ -1,343 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from __future__ import absolute_import, division, print_function - -from enum import Enum - -from .exceptions import UnsupportedCipherException - - -class CipherSuites(Enum): - TLS_NULL_WITH_NULL_NULL = 0x0000 - TLS_RSA_WITH_NULL_MD5 = 0x0001 - TLS_RSA_WITH_NULL_SHA = 0x0002 - TLS_RSA_EXPORT_WITH_RC4_40_MD5 = 0x0003 - TLS_RSA_WITH_RC4_128_MD5 = 0x0004 - TLS_RSA_WITH_RC4_128_SHA = 0x0005 - TLS_RSA_EXPORT_WITH_RC2_CBC_40_MD5 = 0x0006 - TLS_RSA_WITH_IDEA_CBC_SHA = 0x0007 - TLS_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0008 - TLS_RSA_WITH_DES_CBC_SHA = 0x0009 - TLS_RSA_WITH_3DES_EDE_CBC_SHA = 0x000A - TLS_DH_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x000B - TLS_DH_DSS_WITH_DES_CBC_SHA = 0x000C - TLS_DH_DSS_WITH_3DES_EDE_CBC_SHA = 0x000D - TLS_DH_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x000E - TLS_DH_RSA_WITH_DES_CBC_SHA = 0x000F - TLS_DH_RSA_WITH_3DES_EDE_CBC_SHA = 0x0010 - TLS_DHE_DSS_EXPORT_WITH_DES40_CBC_SHA = 0x0011 - TLS_DHE_DSS_WITH_DES_CBC_SHA = 0x0012 - TLS_DHE_DSS_WITH_3DES_EDE_CBC_SHA = 0x0013 - TLS_DHE_RSA_EXPORT_WITH_DES40_CBC_SHA = 0x0014 - TLS_DHE_RSA_WITH_DES_CBC_SHA = 0x0015 - TLS_DHE_RSA_WITH_3DES_EDE_CBC_SHA = 0x0016 - TLS_DH_anon_EXPORT_WITH_RC4_40_MD5 = 0x0017 - TLS_DH_anon_WITH_RC4_128_MD5 = 0x0018 - TLS_DH_anon_EXPORT_WITH_DES40_CBC_SHA = 0x0019 - TLS_DH_anon_WITH_DES_CBC_SHA = 0x001A - TLS_DH_anon_WITH_3DES_EDE_CBC_SHA = 0x001B - TLS_KRB5_WITH_DES_CBC_SHA = 0x001E - TLS_KRB5_WITH_3DES_EDE_CBC_SHA = 0x001F - TLS_KRB5_WITH_RC4_128_SHA = 0x0020 - TLS_KRB5_WITH_IDEA_CBC_SHA = 0x0021 - TLS_KRB5_WITH_DES_CBC_MD5 = 0x0022 - TLS_KRB5_WITH_3DES_EDE_CBC_MD5 = 0x0023 - TLS_KRB5_WITH_RC4_128_MD5 = 0x0024 - TLS_KRB5_WITH_IDEA_CBC_MD5 = 0x0025 - TLS_KRB5_EXPORT_WITH_DES_CBC_40_SHA = 0x0026 - TLS_KRB5_EXPORT_WITH_RC2_CBC_40_SHA = 0x0027 - TLS_KRB5_EXPORT_WITH_RC4_40_SHA = 0x0028 - TLS_KRB5_EXPORT_WITH_DES_CBC_40_MD5 = 0x0029 - TLS_KRB5_EXPORT_WITH_RC2_CBC_40_MD5 = 0x002A - TLS_KRB5_EXPORT_WITH_RC4_40_MD5 = 0x002B - TLS_PSK_WITH_NULL_SHA = 0x002C - TLS_DHE_PSK_WITH_NULL_SHA = 0x002D - TLS_RSA_PSK_WITH_NULL_SHA = 0x002E - TLS_RSA_WITH_AES_128_CBC_SHA = 0x002F - TLS_DH_DSS_WITH_AES_128_CBC_SHA = 0x0030 - TLS_DH_RSA_WITH_AES_128_CBC_SHA = 0x0031 - TLS_DHE_DSS_WITH_AES_128_CBC_SHA = 0x0032 - TLS_DHE_RSA_WITH_AES_128_CBC_SHA = 0x0033 - TLS_DH_anon_WITH_AES_128_CBC_SHA = 0x0034 - TLS_RSA_WITH_AES_256_CBC_SHA = 0x0035 - TLS_DH_DSS_WITH_AES_256_CBC_SHA = 0x0036 - TLS_DH_RSA_WITH_AES_256_CBC_SHA = 0x0037 - TLS_DHE_DSS_WITH_AES_256_CBC_SHA = 0x0038 - TLS_DHE_RSA_WITH_AES_256_CBC_SHA = 0x0039 - TLS_DH_anon_WITH_AES_256_CBC_SHA = 0x003A - TLS_RSA_WITH_NULL_SHA256 = 0x003B - TLS_RSA_WITH_AES_128_CBC_SHA256 = 0x003C - TLS_RSA_WITH_AES_256_CBC_SHA256 = 0x003D - TLS_DH_DSS_WITH_AES_128_CBC_SHA256 = 0x003E - TLS_DH_RSA_WITH_AES_128_CBC_SHA256 = 0x003F - TLS_DHE_DSS_WITH_AES_128_CBC_SHA256 = 0x0040 - TLS_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0041 - TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA = 0x0042 - TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0043 - TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA = 0x0044 - TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA = 0x0045 - TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA = 0x0046 - TLS_DHE_RSA_WITH_AES_128_CBC_SHA256 = 0x0067 - TLS_DH_DSS_WITH_AES_256_CBC_SHA256 = 0x0068 - TLS_DH_RSA_WITH_AES_256_CBC_SHA256 = 0x0069 - TLS_DHE_DSS_WITH_AES_256_CBC_SHA256 = 0x006A - TLS_DHE_RSA_WITH_AES_256_CBC_SHA256 = 0x006B - TLS_DH_anon_WITH_AES_128_CBC_SHA256 = 0x006C - TLS_DH_anon_WITH_AES_256_CBC_SHA256 = 0x006D - TLS_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0084 - TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA = 0x0085 - TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0086 - TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA = 0x0087 - TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA = 0x0088 - TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA = 0x0089 - TLS_PSK_WITH_RC4_128_SHA = 0x008A - TLS_PSK_WITH_3DES_EDE_CBC_SHA = 0x008B - TLS_PSK_WITH_AES_128_CBC_SHA = 0x008C - TLS_PSK_WITH_AES_256_CBC_SHA = 0x008D - TLS_DHE_PSK_WITH_RC4_128_SHA = 0x008E - TLS_DHE_PSK_WITH_3DES_EDE_CBC_SHA = 0x008F - TLS_DHE_PSK_WITH_AES_128_CBC_SHA = 0x0090 - TLS_DHE_PSK_WITH_AES_256_CBC_SHA = 0x0091 - TLS_RSA_PSK_WITH_RC4_128_SHA = 0x0092 - TLS_RSA_PSK_WITH_3DES_EDE_CBC_SHA = 0x0093 - TLS_RSA_PSK_WITH_AES_128_CBC_SHA = 0x0094 - TLS_RSA_PSK_WITH_AES_256_CBC_SHA = 0x0095 - TLS_RSA_WITH_SEED_CBC_SHA = 0x0096 - TLS_DH_DSS_WITH_SEED_CBC_SHA = 0x0097 - TLS_DH_RSA_WITH_SEED_CBC_SHA = 0x0098 - TLS_DHE_DSS_WITH_SEED_CBC_SHA = 0x0099 - TLS_DHE_RSA_WITH_SEED_CBC_SHA = 0x009A - TLS_DH_anon_WITH_SEED_CBC_SHA = 0x009B - TLS_RSA_WITH_AES_128_GCM_SHA256 = 0x009C - TLS_RSA_WITH_AES_256_GCM_SHA384 = 0x009D - TLS_DHE_RSA_WITH_AES_128_GCM_SHA256 = 0x009E - TLS_DHE_RSA_WITH_AES_256_GCM_SHA384 = 0x009F - TLS_DH_RSA_WITH_AES_128_GCM_SHA256 = 0x00A0 - TLS_DH_RSA_WITH_AES_256_GCM_SHA384 = 0x00A1 - TLS_DHE_DSS_WITH_AES_128_GCM_SHA256 = 0x00A2 - TLS_DHE_DSS_WITH_AES_256_GCM_SHA384 = 0x00A3 - TLS_DH_DSS_WITH_AES_128_GCM_SHA256 = 0x00A4 - TLS_DH_DSS_WITH_AES_256_GCM_SHA384 = 0x00A5 - TLS_DH_anon_WITH_AES_128_GCM_SHA256 = 0x00A6 - TLS_DH_anon_WITH_AES_256_GCM_SHA384 = 0x00A7 - TLS_PSK_WITH_AES_128_GCM_SHA256 = 0x00A8 - TLS_PSK_WITH_AES_256_GCM_SHA384 = 0x00A9 - TLS_DHE_PSK_WITH_AES_128_GCM_SHA256 = 0x00AA - TLS_DHE_PSK_WITH_AES_256_GCM_SHA384 = 0x00AB - TLS_RSA_PSK_WITH_AES_128_GCM_SHA256 = 0x00AC - TLS_RSA_PSK_WITH_AES_256_GCM_SHA384 = 0x00AD - TLS_PSK_WITH_AES_128_CBC_SHA256 = 0x00AE - TLS_PSK_WITH_AES_256_CBC_SHA384 = 0x00AF - TLS_PSK_WITH_NULL_SHA256 = 0x00B0 - TLS_PSK_WITH_NULL_SHA384 = 0x00B1 - TLS_DHE_PSK_WITH_AES_128_CBC_SHA256 = 0x00B2 - TLS_DHE_PSK_WITH_AES_256_CBC_SHA384 = 0x00B3 - TLS_DHE_PSK_WITH_NULL_SHA256 = 0x00B4 - TLS_DHE_PSK_WITH_NULL_SHA384 = 0x00B5 - TLS_RSA_PSK_WITH_AES_128_CBC_SHA256 = 0x00B6 - TLS_RSA_PSK_WITH_AES_256_CBC_SHA384 = 0x00B7 - TLS_RSA_PSK_WITH_NULL_SHA256 = 0x00B8 - TLS_RSA_PSK_WITH_NULL_SHA384 = 0x00B9 - TLS_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BA - TLS_DH_DSS_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BB - TLS_DH_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BC - TLS_DHE_DSS_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BD - TLS_DHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BE - TLS_DH_anon_WITH_CAMELLIA_128_CBC_SHA256 = 0x00BF - TLS_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C0 - TLS_DH_DSS_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C1 - TLS_DH_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C2 - TLS_DHE_DSS_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C3 - TLS_DHE_RSA_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C4 - TLS_DH_anon_WITH_CAMELLIA_256_CBC_SHA256 = 0x00C5 - TLS_EMPTY_RENEGOTIATION_INFO_SCSV = 0x00FF - TLS_ECDH_ECDSA_WITH_NULL_SHA = 0xC001 - TLS_ECDH_ECDSA_WITH_RC4_128_SHA = 0xC002 - TLS_ECDH_ECDSA_WITH_3DES_EDE_CBC_SHA = 0xC003 - TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA = 0xC004 - TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA = 0xC005 - TLS_ECDHE_ECDSA_WITH_NULL_SHA = 0xC006 - TLS_ECDHE_ECDSA_WITH_RC4_128_SHA = 0xC007 - TLS_ECDHE_ECDSA_WITH_3DES_EDE_CBC_SHA = 0xC008 - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA = 0xC009 - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA = 0xC00A - TLS_ECDH_RSA_WITH_NULL_SHA = 0xC00B - TLS_ECDH_RSA_WITH_RC4_128_SHA = 0xC00C - TLS_ECDH_RSA_WITH_3DES_EDE_CBC_SHA = 0xC00D - TLS_ECDH_RSA_WITH_AES_128_CBC_SHA = 0xC00E - TLS_ECDH_RSA_WITH_AES_256_CBC_SHA = 0xC00F - TLS_ECDHE_RSA_WITH_NULL_SHA = 0xC010 - TLS_ECDHE_RSA_WITH_RC4_128_SHA = 0xC011 - TLS_ECDHE_RSA_WITH_3DES_EDE_CBC_SHA = 0xC012 - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA = 0xC013 - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA = 0xC014 - TLS_ECDH_anon_WITH_NULL_SHA = 0xC015 - TLS_ECDH_anon_WITH_RC4_128_SHA = 0xC016 - TLS_ECDH_anon_WITH_3DES_EDE_CBC_SHA = 0xC017 - TLS_ECDH_anon_WITH_AES_128_CBC_SHA = 0xC018 - TLS_ECDH_anon_WITH_AES_256_CBC_SHA = 0xC019 - TLS_SRP_SHA_WITH_3DES_EDE_CBC_SHA = 0xC01A - TLS_SRP_SHA_RSA_WITH_3DES_EDE_CBC_SHA = 0xC01B - TLS_SRP_SHA_DSS_WITH_3DES_EDE_CBC_SHA = 0xC01C - TLS_SRP_SHA_WITH_AES_128_CBC_SHA = 0xC01D - TLS_SRP_SHA_RSA_WITH_AES_128_CBC_SHA = 0xC01E - TLS_SRP_SHA_DSS_WITH_AES_128_CBC_SHA = 0xC01F - TLS_SRP_SHA_WITH_AES_256_CBC_SHA = 0xC020 - TLS_SRP_SHA_RSA_WITH_AES_256_CBC_SHA = 0xC021 - TLS_SRP_SHA_DSS_WITH_AES_256_CBC_SHA = 0xC022 - TLS_ECDHE_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC023 - TLS_ECDHE_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC024 - TLS_ECDH_ECDSA_WITH_AES_128_CBC_SHA256 = 0xC025 - TLS_ECDH_ECDSA_WITH_AES_256_CBC_SHA384 = 0xC026 - TLS_ECDHE_RSA_WITH_AES_128_CBC_SHA256 = 0xC027 - TLS_ECDHE_RSA_WITH_AES_256_CBC_SHA384 = 0xC028 - TLS_ECDH_RSA_WITH_AES_128_CBC_SHA256 = 0xC029 - TLS_ECDH_RSA_WITH_AES_256_CBC_SHA384 = 0xC02A - TLS_ECDHE_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02B - TLS_ECDHE_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02C - TLS_ECDH_ECDSA_WITH_AES_128_GCM_SHA256 = 0xC02D - TLS_ECDH_ECDSA_WITH_AES_256_GCM_SHA384 = 0xC02E - TLS_ECDHE_RSA_WITH_AES_128_GCM_SHA256 = 0xC02F - TLS_ECDHE_RSA_WITH_AES_256_GCM_SHA384 = 0xC030 - TLS_ECDH_RSA_WITH_AES_128_GCM_SHA256 = 0xC031 - TLS_ECDH_RSA_WITH_AES_256_GCM_SHA384 = 0xC032 - TLS_ECDHE_PSK_WITH_RC4_128_SHA = 0xC033 - TLS_ECDHE_PSK_WITH_3DES_EDE_CBC_SHA = 0xC034 - TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA = 0xC035 - TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA = 0xC036 - TLS_ECDHE_PSK_WITH_AES_128_CBC_SHA256 = 0xC037 - TLS_ECDHE_PSK_WITH_AES_256_CBC_SHA384 = 0xC038 - TLS_ECDHE_PSK_WITH_NULL_SHA = 0xC039 - TLS_ECDHE_PSK_WITH_NULL_SHA256 = 0xC03A - TLS_ECDHE_PSK_WITH_NULL_SHA384 = 0xC03B - TLS_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC03C - TLS_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC03D - TLS_DH_DSS_WITH_ARIA_128_CBC_SHA256 = 0xC03E - TLS_DH_DSS_WITH_ARIA_256_CBC_SHA384 = 0xC03F - TLS_DH_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC040 - TLS_DH_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC041 - TLS_DHE_DSS_WITH_ARIA_128_CBC_SHA256 = 0xC042 - TLS_DHE_DSS_WITH_ARIA_256_CBC_SHA384 = 0xC043 - TLS_DHE_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC044 - TLS_DHE_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC045 - TLS_DH_anon_WITH_ARIA_128_CBC_SHA256 = 0xC046 - TLS_DH_anon_WITH_ARIA_256_CBC_SHA384 = 0xC047 - TLS_ECDHE_ECDSA_WITH_ARIA_128_CBC_SHA256 = 0xC048 - TLS_ECDHE_ECDSA_WITH_ARIA_256_CBC_SHA384 = 0xC049 - TLS_ECDH_ECDSA_WITH_ARIA_128_CBC_SHA256 = 0xC04A - TLS_ECDH_ECDSA_WITH_ARIA_256_CBC_SHA384 = 0xC04B - TLS_ECDHE_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC04C - TLS_ECDHE_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC04D - TLS_ECDH_RSA_WITH_ARIA_128_CBC_SHA256 = 0xC04E - TLS_ECDH_RSA_WITH_ARIA_256_CBC_SHA384 = 0xC04F - TLS_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC050 - TLS_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC051 - TLS_DHE_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC052 - TLS_DHE_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC053 - TLS_DH_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC054 - TLS_DH_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC055 - TLS_DHE_DSS_WITH_ARIA_128_GCM_SHA256 = 0xC056 - TLS_DHE_DSS_WITH_ARIA_256_GCM_SHA384 = 0xC057 - TLS_DH_DSS_WITH_ARIA_128_GCM_SHA256 = 0xC058 - TLS_DH_DSS_WITH_ARIA_256_GCM_SHA384 = 0xC059 - TLS_DH_anon_WITH_ARIA_128_GCM_SHA256 = 0xC05A - TLS_DH_anon_WITH_ARIA_256_GCM_SHA384 = 0xC05B - TLS_ECDHE_ECDSA_WITH_ARIA_128_GCM_SHA256 = 0xC05C - TLS_ECDHE_ECDSA_WITH_ARIA_256_GCM_SHA384 = 0xC05D - TLS_ECDH_ECDSA_WITH_ARIA_128_GCM_SHA256 = 0xC05E - TLS_ECDH_ECDSA_WITH_ARIA_256_GCM_SHA384 = 0xC05F - TLS_ECDHE_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC060 - TLS_ECDHE_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC061 - TLS_ECDH_RSA_WITH_ARIA_128_GCM_SHA256 = 0xC062 - TLS_ECDH_RSA_WITH_ARIA_256_GCM_SHA384 = 0xC063 - TLS_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC064 - TLS_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC065 - TLS_DHE_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC066 - TLS_DHE_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC067 - TLS_RSA_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC068 - TLS_RSA_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC069 - TLS_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06A - TLS_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06B - TLS_DHE_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06C - TLS_DHE_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06D - TLS_RSA_PSK_WITH_ARIA_128_GCM_SHA256 = 0xC06E - TLS_RSA_PSK_WITH_ARIA_256_GCM_SHA384 = 0xC06F - TLS_ECDHE_PSK_WITH_ARIA_128_CBC_SHA256 = 0xC070 - TLS_ECDHE_PSK_WITH_ARIA_256_CBC_SHA384 = 0xC071 - TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC072 - TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC073 - TLS_ECDH_ECDSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC074 - TLS_ECDH_ECDSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC075 - TLS_ECDHE_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC076 - TLS_ECDHE_RSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC077 - TLS_ECDH_RSA_WITH_CAMELLIA_128_CBC_SHA256 = 0xC078 - TLS_ECDH_RSA_WITH_CAMELLIA_256_CBC_SHA384 = 0xC079 - TLS_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07A - TLS_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07B - TLS_DHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07C - TLS_DHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07D - TLS_DH_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC07E - TLS_DH_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC07F - TLS_DHE_DSS_WITH_CAMELLIA_128_GCM_SHA256 = 0xC080 - TLS_DHE_DSS_WITH_CAMELLIA_256_GCM_SHA384 = 0xC081 - TLS_DH_DSS_WITH_CAMELLIA_128_GCM_SHA256 = 0xC082 - TLS_DH_DSS_WITH_CAMELLIA_256_GCM_SHA384 = 0xC083 - TLS_DH_anon_WITH_CAMELLIA_128_GCM_SHA256 = 0xC084 - TLS_DH_anon_WITH_CAMELLIA_256_GCM_SHA384 = 0xC085 - TLS_ECDHE_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC086 - TLS_ECDHE_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC087 - TLS_ECDH_ECDSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC088 - TLS_ECDH_ECDSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC089 - TLS_ECDHE_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08A - TLS_ECDHE_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08B - TLS_ECDH_RSA_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08C - TLS_ECDH_RSA_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08D - TLS_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC08E - TLS_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC08F - TLS_DHE_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC090 - TLS_DHE_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC091 - TLS_RSA_PSK_WITH_CAMELLIA_128_GCM_SHA256 = 0xC092 - TLS_RSA_PSK_WITH_CAMELLIA_256_GCM_SHA384 = 0xC093 - TLS_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC094 - TLS_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC095 - TLS_DHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC096 - TLS_DHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC097 - TLS_RSA_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC098 - TLS_RSA_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC099 - TLS_ECDHE_PSK_WITH_CAMELLIA_128_CBC_SHA256 = 0xC09A - TLS_ECDHE_PSK_WITH_CAMELLIA_256_CBC_SHA384 = 0xC09B - TLS_RSA_WITH_AES_128_CCM = 0xC09C - TLS_RSA_WITH_AES_256_CCM = 0xC09D - TLS_DHE_RSA_WITH_AES_128_CCM = 0xC09E - TLS_DHE_RSA_WITH_AES_256_CCM = 0xC09F - TLS_RSA_WITH_AES_128_CCM_8 = 0xC0A0 - TLS_RSA_WITH_AES_256_CCM_8 = 0xC0A1 - TLS_DHE_RSA_WITH_AES_128_CCM_8 = 0xC0A2 - TLS_DHE_RSA_WITH_AES_256_CCM_8 = 0xC0A3 - TLS_PSK_WITH_AES_128_CCM = 0xC0A4 - TLS_PSK_WITH_AES_256_CCM = 0xC0A5 - TLS_DHE_PSK_WITH_AES_128_CCM = 0xC0A6 - TLS_DHE_PSK_WITH_AES_256_CCM = 0xC0A7 - TLS_PSK_WITH_AES_128_CCM_8 = 0xC0A8 - TLS_PSK_WITH_AES_256_CCM_8 = 0xC0A9 - TLS_PSK_DHE_WITH_AES_128_CCM_8 = 0xC0AA - TLS_PSK_DHE_WITH_AES_256_CCM_8 = 0xC0AB - TLS_ECDHE_ECDSA_WITH_AES_128_CCM = 0xC0AC - TLS_ECDHE_ECDSA_WITH_AES_256_CCM = 0xC0AD - TLS_ECDHE_ECDSA_WITH_AES_128_CCM_8 = 0xC0AE - TLS_ECDHE_ECDSA_WITH_AES_256_CCM_8 = 0xC0AF - TLS_ECDHE_ECDSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCC14 - TLS_ECDHE_RSA_WITH_CHACHA20_POLY1305_SHA256 = 0xCC13 - - -def select_preferred_ciphersuite(client_supported, server_supported): - for i in server_supported: - assert isinstance(i, CipherSuites) - if i in client_supported: - return i - - raise UnsupportedCipherException( - "Client supported ciphersuites are not supported on the server." - ) diff --git a/libmproxy/contrib/tls/exceptions.py b/libmproxy/contrib/tls/exceptions.py deleted file mode 100644 index 75b34d11..00000000 --- a/libmproxy/contrib/tls/exceptions.py +++ /dev/null @@ -1,2 +0,0 @@ -class UnsupportedCipherException(Exception): - pass diff --git a/libmproxy/contrib/tls/hello_message.py b/libmproxy/contrib/tls/hello_message.py deleted file mode 100644 index 23cd872b..00000000 --- a/libmproxy/contrib/tls/hello_message.py +++ /dev/null @@ -1,178 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from __future__ import absolute_import, division, print_function - -from enum import Enum - -from characteristic import attributes - -from construct import Container - -from six import BytesIO - -from . import _constructs - - -@attributes(['major', 'minor']) -class ProtocolVersion(object): - """ - An object representing a ProtocolVersion struct. - """ - - -@attributes(['gmt_unix_time', 'random_bytes']) -class Random(object): - """ - An object representing a Random struct. - """ - - -@attributes(['type', 'data']) -class Extension(object): - """ - An object representing an Extension struct. - """ - def as_bytes(self): - return _constructs.Extension.build(Container( - type=self.type.value, length=len(self.data), data=self.data)) - - -@attributes(['client_version', 'random', 'session_id', 'cipher_suites', - 'compression_methods', 'extensions']) -class ClientHello(object): - """ - An object representing a ClientHello message. - """ - def as_bytes(self): - return _constructs.ClientHello.build( - Container( - version=Container(major=self.client_version.major, - minor=self.client_version.minor), - random=Container( - gmt_unix_time=self.random.gmt_unix_time, - random_bytes=self.random.random_bytes - ), - session_id=Container(length=len(self.session_id), - session_id=self.session_id), - cipher_suites=Container(length=len(self.cipher_suites) * 2, - cipher_suites=self.cipher_suites), - compression_methods=Container( - length=len(self.compression_methods), - compression_methods=self.compression_methods - ), - extensions_length=sum([2 + 2 + len(ext.data) - for ext in self.extensions]), - extensions_bytes=b''.join( - [ext.as_bytes() for ext in self.extensions] - ) - ) - ) - - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``ClientHello`` struct. - - :param bytes: the bytes representing the input. - :return: ClientHello object. - """ - construct = _constructs.ClientHello.parse(bytes) - # XXX Is there a better way in Construct to parse an array of - # variable-length structs? - extensions = [] - extensions_io = BytesIO(construct.extensions_bytes) - while extensions_io.tell() < construct.extensions_length: - extension_construct = _constructs.Extension.parse_stream( - extensions_io) - extensions.append( - Extension(type=ExtensionType(extension_construct.type), - data=extension_construct.data)) - return ClientHello( - client_version=ProtocolVersion( - major=construct.version.major, - minor=construct.version.minor, - ), - random=Random( - gmt_unix_time=construct.random.gmt_unix_time, - random_bytes=construct.random.random_bytes, - ), - session_id=construct.session_id.session_id, - # TODO: cipher suites should be enums - cipher_suites=construct.cipher_suites.cipher_suites, - compression_methods=( - construct.compression_methods.compression_methods - ), - extensions=extensions, - ) - - -class ExtensionType(Enum): - SIGNATURE_ALGORITHMS = 13 - # XXX: See http://tools.ietf.org/html/rfc5246#ref-TLSEXT - - -@attributes(['server_version', 'random', 'session_id', 'cipher_suite', - 'compression_method', 'extensions']) -class ServerHello(object): - """ - An object representing a ServerHello message. - """ - def as_bytes(self): - return _constructs.ServerHello.build( - Container( - version=Container(major=self.server_version.major, - minor=self.server_version.minor), - random=Container( - gmt_unix_time=self.random.gmt_unix_time, - random_bytes=self.random.random_bytes - ), - session_id=Container(length=len(self.session_id), - session_id=self.session_id), - cipher_suite=self.cipher_suite, - compression_method=self.compression_method.value, - extensions_length=sum([2 + 2 + len(ext.data) - for ext in self.extensions]), - extensions_bytes=b''.join( - [ext.as_bytes() for ext in self.extensions] - ) - ) - ) - - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``ServerHello`` struct. - - :param bytes: the bytes representing the input. - :return: ServerHello object. - """ - construct = _constructs.ServerHello.parse(bytes) - # XXX: Find a better way to parse extensions - extensions = [] - extensions_io = BytesIO(construct.extensions_bytes) - while extensions_io.tell() < construct.extensions_length: - extension_construct = _constructs.Extension.parse_stream( - extensions_io) - extensions.append( - Extension(type=ExtensionType(extension_construct.type), - data=extension_construct.data)) - return ServerHello( - server_version=ProtocolVersion( - major=construct.version.major, - minor=construct.version.minor, - ), - random=Random( - gmt_unix_time=construct.random.gmt_unix_time, - random_bytes=construct.random.random_bytes, - ), - session_id=construct.session_id.session_id, - cipher_suite=construct.cipher_suite, - compression_method=CompressionMethod(construct.compression_method), - extensions=extensions, - ) - - -class CompressionMethod(Enum): - NULL = 0 diff --git a/libmproxy/contrib/tls/message.py b/libmproxy/contrib/tls/message.py deleted file mode 100644 index b372859f..00000000 --- a/libmproxy/contrib/tls/message.py +++ /dev/null @@ -1,313 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from __future__ import absolute_import, division, print_function - -from enum import Enum - -from characteristic import attributes - -from construct import Container - -from six import BytesIO - -from . import _constructs - -from .hello_message import ( - ClientHello, ProtocolVersion, ServerHello -) - - -class ClientCertificateType(Enum): - RSA_SIGN = 1 - DSS_SIGN = 2 - RSA_FIXED_DH = 3 - DSS_FIXED_DH = 4 - RSA_EPHEMERAL_DH_RESERVED = 5 - DSS_EPHEMERAL_DH_RESERVED = 6 - FORTEZZA_DMS_RESERVED = 20 - - -class HashAlgorithm(Enum): - NONE = 0 - MD5 = 1 - SHA1 = 2 - SHA224 = 3 - SHA256 = 4 - SHA384 = 5 - SHA512 = 6 - - -class SignatureAlgorithm(Enum): - ANONYMOUS = 0 - RSA = 1 - DSA = 2 - ECDSA = 3 - - -class HandshakeType(Enum): - HELLO_REQUEST = 0 - CLIENT_HELLO = 1 - SERVER_HELLO = 2 - CERTIFICATE = 11 - SERVER_KEY_EXCHANGE = 12 - CERTIFICATE_REQUEST = 13 - SERVER_HELLO_DONE = 14 - CERTIFICATE_VERIFY = 15 - CLIENT_KEY_EXCHANGE = 16 - FINISHED = 20 - - -class HelloRequest(object): - """ - An object representing a HelloRequest struct. - """ - def as_bytes(self): - return b'' - - -class ServerHelloDone(object): - """ - An object representing a ServerHelloDone struct. - """ - def as_bytes(self): - return b'' - - -@attributes(['certificate_types', 'supported_signature_algorithms', - 'certificate_authorities']) -class CertificateRequest(object): - """ - An object representing a CertificateRequest struct. - """ - def as_bytes(self): - return _constructs.CertificateRequest.build(Container( - certificate_types=Container( - length=len(self.certificate_types), - certificate_types=[cert_type.value - for cert_type in self.certificate_types] - ), - supported_signature_algorithms=Container( - supported_signature_algorithms_length=2 * len( - self.supported_signature_algorithms - ), - algorithms=[Container( - hash=algorithm.hash.value, - signature=algorithm.signature.value, - ) - for algorithm in self.supported_signature_algorithms - ] - ), - certificate_authorities=Container( - length=len(self.certificate_authorities), - certificate_authorities=self.certificate_authorities - ) - )) - - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``CertificateRequest`` struct. - - :param bytes: the bytes representing the input. - :return: CertificateRequest object. - """ - construct = _constructs.CertificateRequest.parse(bytes) - return cls( - certificate_types=[ - ClientCertificateType(cert_type) - for cert_type in construct.certificate_types.certificate_types - ], - supported_signature_algorithms=[ - SignatureAndHashAlgorithm( - hash=HashAlgorithm(algorithm.hash), - signature=SignatureAlgorithm(algorithm.signature), - ) - for algorithm in ( - construct.supported_signature_algorithms.algorithms - ) - ], - certificate_authorities=( - construct.certificate_authorities.certificate_authorities - ) - ) - - -@attributes(['hash', 'signature']) -class SignatureAndHashAlgorithm(object): - """ - An object representing a SignatureAndHashAlgorithm struct. - """ - - -@attributes(['dh_p', 'dh_g', 'dh_Ys']) -class ServerDHParams(object): - """ - An object representing a ServerDHParams struct. - """ - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``ServerDHParams`` struct. - - :param bytes: the bytes representing the input. - :return: ServerDHParams object. - """ - construct = _constructs.ServerDHParams.parse(bytes) - return cls( - dh_p=construct.dh_p, - dh_g=construct.dh_g, - dh_Ys=construct.dh_Ys - ) - - -@attributes(['client_version', 'random']) -class PreMasterSecret(object): - """ - An object representing a PreMasterSecret struct. - """ - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``PreMasterSecret`` struct. - - :param bytes: the bytes representing the input. - :return: CertificateRequest object. - """ - construct = _constructs.PreMasterSecret.parse(bytes) - return cls( - client_version=ProtocolVersion( - major=construct.version.major, - minor=construct.version.minor, - ), - random=construct.random_bytes, - ) - - -@attributes(['asn1_cert']) -class ASN1Cert(object): - """ - An object representing ASN.1 Certificate - """ - def as_bytes(self): - return _constructs.ASN1Cert.build(Container( - length=len(self.asn1_cert), - asn1_cert=self.asn1_cert - )) - - -@attributes(['certificate_list']) -class Certificate(object): - """ - An object representing a Certificate struct. - """ - def as_bytes(self): - return _constructs.Certificate.build(Container( - certificates_length=sum([4 + len(asn1cert.asn1_cert) - for asn1cert in self.certificate_list]), - certificates_bytes=b''.join( - [asn1cert.as_bytes() for asn1cert in self.certificate_list] - ) - - )) - - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``Certificate`` struct. - - :param bytes: the bytes representing the input. - :return: Certificate object. - """ - construct = _constructs.Certificate.parse(bytes) - # XXX: Find a better way to parse an array of variable-length objects - certificates = [] - certificates_io = BytesIO(construct.certificates_bytes) - - while certificates_io.tell() < construct.certificates_length: - certificate_construct = _constructs.ASN1Cert.parse_stream( - certificates_io - ) - certificates.append( - ASN1Cert(asn1_cert=certificate_construct.asn1_cert) - ) - return cls( - certificate_list=certificates - ) - - -@attributes(['verify_data']) -class Finished(object): - def as_bytes(self): - return self.verify_data - - -@attributes(['msg_type', 'length', 'body']) -class Handshake(object): - """ - An object representing a Handshake struct. - """ - def as_bytes(self): - if self.msg_type in [ - HandshakeType.SERVER_HELLO, HandshakeType.CLIENT_HELLO, - HandshakeType.CERTIFICATE, HandshakeType.CERTIFICATE_REQUEST, - HandshakeType.HELLO_REQUEST, HandshakeType.SERVER_HELLO_DONE, - HandshakeType.FINISHED - ]: - _body_as_bytes = self.body.as_bytes() - else: - _body_as_bytes = b'' - return _constructs.Handshake.build( - Container( - msg_type=self.msg_type.value, - length=self.length, - body=_body_as_bytes - ) - ) - - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``Handshake`` struct. - - :param bytes: the bytes representing the input. - :return: Handshake object. - """ - construct = _constructs.Handshake.parse(bytes) - return cls( - msg_type=HandshakeType(construct.msg_type), - length=construct.length, - body=cls._get_handshake_message( - HandshakeType(construct.msg_type), construct.body - ), - ) - - @staticmethod - def _get_handshake_message(msg_type, body): - _handshake_message_parser = { - HandshakeType.CLIENT_HELLO: ClientHello.from_bytes, - HandshakeType.SERVER_HELLO: ServerHello.from_bytes, - HandshakeType.CERTIFICATE: Certificate.from_bytes, - # 12: parse_server_key_exchange, - HandshakeType.CERTIFICATE_REQUEST: CertificateRequest.from_bytes, - # 15: parse_certificate_verify, - # 16: parse_client_key_exchange, - } - - try: - if msg_type == HandshakeType.HELLO_REQUEST: - return HelloRequest() - elif msg_type == HandshakeType.SERVER_HELLO_DONE: - return ServerHelloDone() - elif msg_type == HandshakeType.FINISHED: - return Finished(verify_data=body) - elif msg_type in [HandshakeType.SERVER_KEY_EXCHANGE, - HandshakeType.CERTIFICATE_VERIFY, - HandshakeType.CLIENT_KEY_EXCHANGE, - ]: - raise NotImplementedError - else: - return _handshake_message_parser[msg_type](body) - except NotImplementedError: - return None # TODO diff --git a/libmproxy/contrib/tls/record.py b/libmproxy/contrib/tls/record.py deleted file mode 100644 index 481c93bc..00000000 --- a/libmproxy/contrib/tls/record.py +++ /dev/null @@ -1,110 +0,0 @@ -# This file is dual licensed under the terms of the Apache License, Version -# 2.0, and the BSD License. See the LICENSE file in the root of this repository -# for complete details. - -from __future__ import absolute_import, division, print_function - -from enum import Enum - -from characteristic import attributes - -from construct import Container - -from . import _constructs - - -@attributes(['major', 'minor']) -class ProtocolVersion(object): - """ - An object representing a ProtocolVersion struct. - """ - - -@attributes(['type', 'version', 'fragment']) -class TLSPlaintext(object): - """ - An object representing a TLSPlaintext struct. - """ - def as_bytes(self): - return _constructs.TLSPlaintext.build( - Container( - type=self.type.value, - version=Container(major=self.version.major, - minor=self.version.minor), - length=len(self.fragment), - fragment=self.fragment - ) - ) - - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``TLSPlaintext`` struct. - - :param bytes: the bytes representing the input. - :return: TLSPlaintext object. - """ - construct = _constructs.TLSPlaintext.parse(bytes) - return cls( - type=ContentType(construct.type), - version=ProtocolVersion( - major=construct.version.major, - minor=construct.version.minor - ), - fragment=construct.fragment - ) - - -@attributes(['type', 'version', 'fragment']) -class TLSCompressed(object): - """ - An object representing a TLSCompressed struct. - """ - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``TLSCompressed`` struct. - - :param bytes: the bytes representing the input. - :return: TLSCompressed object. - """ - construct = _constructs.TLSCompressed.parse(bytes) - return cls( - type=ContentType(construct.type), - version=ProtocolVersion( - major=construct.version.major, - minor=construct.version.minor - ), - fragment=construct.fragment - ) - - -@attributes(['type', 'version', 'fragment']) -class TLSCiphertext(object): - """ - An object representing a TLSCiphertext struct. - """ - @classmethod - def from_bytes(cls, bytes): - """ - Parse a ``TLSCiphertext`` struct. - - :param bytes: the bytes representing the input. - :return: TLSCiphertext object. - """ - construct = _constructs.TLSCiphertext.parse(bytes) - return cls( - type=ContentType(construct.type), - version=ProtocolVersion( - major=construct.version.major, - minor=construct.version.minor - ), - fragment=construct.fragment - ) - - -class ContentType(Enum): - CHANGE_CIPHER_SPEC = 20 - ALERT = 21 - HANDSHAKE = 22 - APPLICATION_DATA = 23 diff --git a/libmproxy/contrib/tls/utils.py b/libmproxy/contrib/tls/utils.py index a971af49..4c917303 100644 --- a/libmproxy/contrib/tls/utils.py +++ b/libmproxy/contrib/tls/utils.py @@ -24,29 +24,3 @@ class _UBInt24(construct.Adapter): def UBInt24(name): # noqa return _UBInt24(construct.Bytes(name, 3)) - - -def LengthPrefixedArray(subcon, length_field=construct.UBInt8("length")): - """ - An array prefixed by a byte length field. - - In contrast to construct.macros.PrefixedArray, - the length field signifies the number of bytes, not the number of elements. - """ - subcon_with_pos = construct.Struct( - subcon.name, - construct.Embed(subcon), - construct.Anchor("__current_pos") - ) - - return construct.Embed( - construct.Struct( - "", - length_field, - construct.Anchor("__start_pos"), - construct.RepeatUntil( - lambda obj, ctx: obj.__current_pos == ctx.__start_pos + getattr(ctx, length_field.name), - subcon_with_pos - ), - ) - ) \ No newline at end of file diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 7ef0ad8c..9c8aeb24 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -61,7 +61,12 @@ class TlsLayer(Layer): layer() def _get_client_hello(self): - # Read all records that contain the initial Client Hello message. + """ + Peek into the socket and read all records that contain the initial client hello message. + + Returns: + The raw handshake packet bytes, without TLS record header(s). + """ client_hello = "" client_hello_size = 1 offset = 0 @@ -75,10 +80,15 @@ class TlsLayer(Layer): return client_hello def _parse_client_hello(self): + """ + Peek into the connection, read the initial client hello and parse it to obtain ALPN values. + """ + raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. try: - client_hello = ClientHello.parse(self._get_client_hello()[4:]) + client_hello = ClientHello.parse(raw_client_hello) except ConstructError as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") + self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") return for extension in client_hello.extensions: -- cgit v1.2.3 From f6dadc2b0de712869d9b8aa928915dbb990bb6af Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 00:07:44 +0200 Subject: no more sni double-connects! --- doc-src/howmitmproxy.html | 9 --------- libmproxy/contrib/README | 2 +- 2 files changed, 1 insertion(+), 10 deletions(-) diff --git a/doc-src/howmitmproxy.html b/doc-src/howmitmproxy.html index fabd393a..16b5f722 100644 --- a/doc-src/howmitmproxy.html +++ b/doc-src/howmitmproxy.html @@ -145,15 +145,6 @@ passed to us. Now we can pause the conversation, and initiate an upstream connection using the correct SNI value, which then serves us the correct upstream certificate, from which we can extract the expected CN and SANs. -There's another wrinkle here. Due to a limitation of the SSL library mitmproxy -uses, we can't detect that a connection _hasn't_ sent an SNI request until it's -too late for upstream certificate sniffing. In practice, we therefore make a -vanilla SSL connection upstream to sniff non-SNI certificates, and then discard -the connection if the client sends an SNI notification. If you're watching your -traffic with a packet sniffer, you'll see two connections to the server when an -SNI request is made, the first of which is immediately closed after the SSL -handshake. Luckily, this is almost never an issue in practice. - ## Putting it all together Lets put all of this together into the complete explicitly proxied HTTPS flow. diff --git a/libmproxy/contrib/README b/libmproxy/contrib/README index e339310a..e5ce11da 100644 --- a/libmproxy/contrib/README +++ b/libmproxy/contrib/README @@ -10,5 +10,5 @@ wbxml - https://github.com/davidpshaw/PyWBXMLDecoder tls, BSD license - - https://github.com/mhils/tls/tree/extension-parsing + - https://github.com/mhils/tls/tree/mitmproxy - limited to required files. \ No newline at end of file -- cgit v1.2.3 From 0f97899fbd6cc68e974b1670e9c5188a28b52168 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 15:26:21 +0200 Subject: re-add --ignore and --tcp --- libmproxy/protocol2/root_context.py | 66 ++++++++++++++++++++++--------------- 1 file changed, 39 insertions(+), 27 deletions(-) diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 880bc160..78d48453 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -1,12 +1,12 @@ from __future__ import (absolute_import, print_function, division) -import string -from libmproxy.protocol2.layer import Kill +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol + from .rawtcp import RawTcpLayer from .tls import TlsLayer -from .http import Http1Layer, Http2Layer, HttpLayer +from .http import Http1Layer, Http2Layer -from netlib.http.http2 import HTTP2Protocol class RootContext(object): """ @@ -22,12 +22,19 @@ class RootContext(object): def next_layer(self, top_layer): """ This function determines the next layer in the protocol stack. - :param top_layer: the current top layer - :return: The next layer. + + Arguments: + top_layer: the current top layer. + + Returns: + The next layer """ - # TODO: Handle ignore and tcp passthrough + # 1. Check for --ignore. + if self.config.check_ignore(top_layer.server_conn.address): + return RawTcpLayer(top_layer) + # 2. Check for TLS # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello d = top_layer.client_conn.rfile.peek(3) @@ -37,30 +44,35 @@ class RootContext(object): d[1] == '\x03' and d[2] in ('\x00', '\x01', '\x02', '\x03') ) + if is_tls_client_hello: + return TlsLayer(top_layer, True, True) - d = top_layer.client_conn.rfile.peek(3) - is_ascii = ( - len(d) == 3 and - all(x in string.ascii_letters for x in d) # better be safe here and don't expect uppercase... - ) + # 3. Check for --tcp + if self.config.check_tcp(top_layer.server_conn.address): + return RawTcpLayer(top_layer) - d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) - is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) + # 4. Check for TLS ALPN (HTTP1/HTTP2) + if isinstance(top_layer, TlsLayer): + alpn = top_layer.client_conn.get_alpn_proto_negotiated() + if alpn == HTTP2Protocol.ALPN_PROTO_H2: + return Http2Layer(top_layer, 'transparent') + if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: + return Http1Layer(top_layer, 'transparent') - alpn_proto_negotiated = top_layer.client_conn.get_alpn_proto_negotiated() - is_alpn_h2_negotiated = ( - isinstance(top_layer, TlsLayer) and - alpn_proto_negotiated == HTTP2Protocol.ALPN_PROTO_H2 - ) + # 5. Assume HTTP1 by default + return Http1Layer(top_layer, 'transparent') - if is_tls_client_hello: - return TlsLayer(top_layer, True, True) - elif is_alpn_h2_negotiated or is_http2_magic: - return Http2Layer(top_layer, 'transparent') - elif is_ascii: - return Http1Layer(top_layer, 'transparent') - else: - return RawTcpLayer(top_layer) + # In a future version, we want to implement TCP passthrough as the last fallback, + # but we don't have the UI part ready for that. + # + # d = top_layer.client_conn.rfile.peek(3) + # is_ascii = ( + # len(d) == 3 and + # all(x in string.ascii_letters for x in d) # better be safe here and don't expect uppercase... + # ) + # # TODO: This could block if there are not enough bytes available? + # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) + # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) @property def layers(self): -- cgit v1.2.3 From ecfde4247fcfd8279948b4a22bc4f04c2fb2ba15 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 15:48:41 +0200 Subject: re-add http1 replay --- libmproxy/protocol2/http.py | 90 ++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 89 insertions(+), 1 deletion(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index e3878fa6..32c0116b 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -1,14 +1,18 @@ from __future__ import (absolute_import, print_function, division) from .. import version +import threading from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer from libmproxy import utils +from libmproxy.controller import Channel from libmproxy.protocol2.layer import Kill -from libmproxy.protocol import KILL +from libmproxy.protocol import KILL, Error from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest +from libmproxy.proxy import Log +from libmproxy.proxy.connection import ServerConnection from netlib import tcp from netlib.http import status_codes, http1, http2, HttpErrorConnClosed, HttpError from netlib.http.semantics import CONTENT_MISSING @@ -509,3 +513,87 @@ class HttpLayer(Layer): odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()]) )) raise InvalidCredentials("Proxy Authentication Required") + + +class RequestReplayThread(threading.Thread): + name = "RequestReplayThread" + + def __init__(self, config, flow, masterq, should_exit): + """ + masterqueue can be a queue or None, if no scripthooks should be + processed. + """ + self.config, self.flow = config, flow + if masterq: + self.channel = Channel(masterq, should_exit) + else: + self.channel = None + super(RequestReplayThread, self).__init__() + + def run(self): + r = self.flow.request + form_out_backup = r.form_out + try: + self.flow.response = None + + # If we have a channel, run script hooks. + if self.channel: + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == KILL: + raise Kill() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply + + if not self.flow.response: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + # FIXME + server_address = self.config.mode.get_upstream_server( + self.flow.client_conn + )[2:] + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + connect_request = make_connect_request((r.host, r.port)) + server.send(protocol.assemble(connect_request)) + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + + server.send(protocol.assemble(r)) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_protocol( + protocol, + r.method, + body_size_limit=self.config.body_size_limit, + ) + if self.channel: + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == KILL: + raise Kill() + except (HttpError, tcp.NetLibError) as v: + self.flow.error = Error(repr(v)) + if self.channel: + self.channel.ask("error", self.flow) + except Kill: + # KillSignal should only be raised if there's a channel in the + # first place. + self.channel.tell("log", Log("Connection killed", "info")) + finally: + r.form_out = form_out_backup -- cgit v1.2.3 From 515c0244483446350779db59a31b8fd7dc603a5b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 15:59:56 +0200 Subject: handle tls server errors more gracefully --- libmproxy/protocol2/tls.py | 20 +++++++++++++++++--- 1 file changed, 17 insertions(+), 3 deletions(-) diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 9c8aeb24..433dd65d 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -51,9 +51,7 @@ class TlsLayer(Layer): self._parse_client_hello() if client_tls_requires_server_cert: - self.ctx.connect() - self._establish_tls_with_server() - self._establish_tls_with_client() + self._establish_tls_with_client_and_server() elif self._client_tls: self._establish_tls_with_client() @@ -148,6 +146,22 @@ class TlsLayer(Layer): self.log("ALPN for client: %s" % choice, "debug") return choice + def _establish_tls_with_client_and_server(self): + self.ctx.connect() + + # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless + # to send an error message over TLS. + try: + self._establish_tls_with_server() + except Exception as e: + try: + self._establish_tls_with_client() + except: + pass + raise e + + self._establish_tls_with_client() + def _establish_tls_with_client(self): self.log("Establish TLS with client", "debug") cert, key, chain_file = self._find_cert() -- cgit v1.2.3 From 83decd6771c430cca9e99ec0050442249d0aa99a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 17:35:53 +0200 Subject: fix inline script redirects --- libmproxy/flow.py | 3 ++- libmproxy/protocol2/http.py | 19 +++++++++++------ libmproxy/protocol2/layer.py | 2 +- libmproxy/protocol2/tls.py | 4 ++-- test/test_server.py | 51 ++++++++++++++++++++++++-------------------- 5 files changed, 45 insertions(+), 34 deletions(-) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 3d9ef722..8605d7a1 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,6 +8,7 @@ import Cookie import cookielib import os import re +from libmproxy.protocol2.http import RequestReplayThread from netlib import odict, wsgi, tcp from netlib.http.semantics import CONTENT_MISSING @@ -934,7 +935,7 @@ class FlowMaster(controller.Master): f.response = None f.error = None self.process_new_request(f) - rt = http.RequestReplayThread( + rt = RequestReplayThread( self.server.config, f, self.masterq if run_scripthooks else False, diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 32c0116b..792cf266 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -212,10 +212,11 @@ class UpstreamConnectLayer(Layer): self.ctx.reconnect() self.send_to_server(self.connect_request) - def set_server(self, address, server_tls, sni, depth=1): + def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: if self.ctx.server_conn: self.ctx.reconnect() + address = Address.wrap(address) self.connect_request.host = address.host self.connect_request.port = address.port self.server_conn.address = address @@ -227,11 +228,16 @@ class HttpLayer(Layer): def __init__(self, ctx, mode): super(HttpLayer, self).__init__(ctx) self.mode = mode + self.__original_server_conn = None + "Contains the original destination in transparent mode, which needs to be restored" + "if an inline script modified the target server for a single http request" def __call__(self): + if self.mode == "transparent": + self.__original_server_conn = self.server_conn while True: try: - flow = HTTPFlow(self.client_conn, self.server_conn, live=True) + flow = HTTPFlow(self.client_conn, self.server_conn, live=self) try: request = self.read_from_client() @@ -288,7 +294,7 @@ class HttpLayer(Layer): flow.live = False def handle_regular_mode_connect(self, request): - self.set_server((request.host, request.port), False, None) + self.set_server((request.host, request.port)) self.send_to_client(make_connect_response(request.httpversion)) layer = self.ctx.next_layer(self) layer() @@ -433,11 +439,10 @@ class HttpLayer(Layer): if flow.request.form_in == "authority": flow.request.scheme = "http" # pseudo value else: - flow.request.host = self.ctx.server_conn.address.host - flow.request.port = self.ctx.server_conn.address.port - flow.request.scheme = "https" if self.server_conn.tls_established else "http" + flow.request.host = self.__original_server_conn.address.host + flow.request.port = self.__original_server_conn.address.port + flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" - # TODO: Expose .set_server functionality to inline scripts request_reply = self.channel.ask("request", flow) if request_reply is None or request_reply == KILL: raise Kill() diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 7cb76591..f72320ff 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -112,7 +112,7 @@ class ServerConnectionMixin(object): self.server_conn.address = address self.connect() - def set_server(self, address, server_tls, sni, depth=1): + def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: if self.server_conn: self._disconnect() diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 433dd65d..b1b80034 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -110,9 +110,9 @@ class TlsLayer(Layer): if self._server_tls and not self.server_conn.tls_established: self._establish_tls_with_server() - def set_server(self, address, server_tls, sni, depth=1): + def set_server(self, address, server_tls=None, sni=None, depth=1): self.ctx.set_server(address, server_tls, sni, depth) - if server_tls is not None: + if depth == 1 and server_tls is not None: self._sni_from_server_change = sni self._server_tls = server_tls diff --git a/test/test_server.py b/test/test_server.py index 529024f5..e9c40d1a 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -1,6 +1,7 @@ import socket import time from OpenSSL import SSL +from netlib.tcp import Address import netlib.tutils from netlib import tcp, http, socks @@ -655,63 +656,67 @@ class MasterRedirectRequest(tservers.TestMaster): redirect_port = None # Set by TestRedirectRequest def handle_request(self, f): - request = f.request - if request.path == "/p/201": - addr = f.live.c.server_conn.address - assert f.live.change_server( - ("127.0.0.1", self.redirect_port), ssl=False) - assert not f.live.change_server( - ("127.0.0.1", self.redirect_port), ssl=False) - tutils.raises( - "SSL handshake error", - f.live.change_server, - ("127.0.0.1", - self.redirect_port), - ssl=True) - assert f.live.change_server(addr, ssl=False) - request.url = "http://127.0.0.1:%s/p/201" % self.redirect_port - tservers.TestMaster.handle_request(self, f) + if f.request.path == "/p/201": + + # This part should have no impact, but it should not cause any exceptions. + addr = f.live.server_conn.address + addr2 = Address(("127.0.0.1", self.redirect_port)) + f.live.set_server(addr2) + f.live.connect() + f.live.set_server(addr) + f.live.connect() + + # This is the actual redirection. + f.request.port = self.redirect_port + super(MasterRedirectRequest, self).handle_request(f) def handle_response(self, f): f.response.content = str(f.client_conn.address.port) f.response.headers[ "server-conn-id"] = [str(f.server_conn.source_address.port)] - tservers.TestMaster.handle_response(self, f) + super(MasterRedirectRequest, self).handle_response(f) class TestRedirectRequest(tservers.HTTPProxTest): masterclass = MasterRedirectRequest + ssl = True def test_redirect(self): + """ + Imagine a single HTTPS connection with three requests: + + 1. First request should pass through unmodified + 2. Second request will be redirected to a different host by an inline script + 3. Third request should pass through unmodified + + This test verifies that the original destination is restored for the third request. + """ self.master.redirect_port = self.server2.port p = self.pathoc() self.server.clear_log() self.server2.clear_log() - r1 = p.request("get:'%s/p/200'" % self.server.urlbase) + r1 = p.request("get:'/p/200'") assert r1.status_code == 200 assert self.server.last_log() assert not self.server2.last_log() self.server.clear_log() self.server2.clear_log() - r2 = p.request("get:'%s/p/201'" % self.server.urlbase) + r2 = p.request("get:'/p/201'") assert r2.status_code == 201 assert not self.server.last_log() assert self.server2.last_log() self.server.clear_log() self.server2.clear_log() - r3 = p.request("get:'%s/p/202'" % self.server.urlbase) + r3 = p.request("get:'/p/202'") assert r3.status_code == 202 assert self.server.last_log() assert not self.server2.last_log() assert r1.content == r2.content == r3.content - assert r1.headers.get_first( - "server-conn-id") == r3.headers.get_first("server-conn-id") - # Make sure that we actually use the same connection in this test case class MasterStreamRequest(tservers.TestMaster): -- cgit v1.2.3 From 5b17496c7e5ea3c40a910c4973eeb7bfbcf065bd Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 18:31:15 +0200 Subject: start fixing proxy config --- libmproxy/proxy/config.py | 48 ++++++----------------------------------------- 1 file changed, 6 insertions(+), 42 deletions(-) diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 4ca15747..83030235 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -3,14 +3,11 @@ import os import re from OpenSSL import SSL -import netlib -from netlib import http, certutils, tcp +from netlib import certutils, tcp from netlib.http import authentication -from .. import utils, platform, version -from .primitives import RegularProxyMode, SpoofMode, SSLSpoofMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode +from .. import utils, platform -TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" @@ -40,15 +37,12 @@ class ProxyConfig: self, host='', port=8080, - server_version=version.NAMEVERSION, cadir=CA_DIR, clientcerts=None, no_upstream_cert=False, body_size_limit=None, mode=None, upstream_server=None, - http_form_in=None, - http_form_out=None, authenticator=None, ignore_hosts=[], tcp_hosts=[], @@ -57,39 +51,19 @@ class ProxyConfig: certs=[], ssl_version_client=tcp.SSL_DEFAULT_METHOD, ssl_version_server=tcp.SSL_DEFAULT_METHOD, - ssl_ports=TRANSPARENT_SSL_PORTS, - spoofed_ssl_port=None, ssl_verify_upstream_cert=False, ssl_upstream_trusted_cadir=None, ssl_upstream_trusted_ca=None ): self.host = host self.port = port - self.server_version = server_version self.ciphers_client = ciphers_client self.ciphers_server = ciphers_server self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit - - if mode == "transparent": - self.mode = TransparentProxyMode(platform.resolver(), ssl_ports) - elif mode == "socks5": - self.mode = Socks5ProxyMode(ssl_ports) - elif mode == "reverse": - self.mode = ReverseProxyMode(upstream_server) - elif mode == "upstream": - self.mode = UpstreamProxyMode(upstream_server) - elif mode == "spoof": - self.mode = SpoofMode() - elif mode == "sslspoof": - self.mode = SSLSpoofMode(spoofed_ssl_port) - else: - self.mode = RegularProxyMode() - - # Handle manual overrides of the http forms - self.mode.http_form_in = http_form_in or self.mode.http_form_in - self.mode.http_form_out = http_form_out or self.mode.http_form_out + self.mode = mode + self.upstream_server = upstream_server self.check_ignore = HostMatcher(ignore_hosts) self.check_tcp = HostMatcher(tcp_hosts) @@ -97,10 +71,10 @@ class ProxyConfig: self.cadir = os.path.expanduser(cadir) self.certstore = certutils.CertStore.from_store( self.cadir, - CONF_BASENAME) + CONF_BASENAME + ) for spec, cert in certs: self.certstore.add_cert_file(spec, cert) - self.ssl_ports = ssl_ports if isinstance(ssl_version_client, int): self.openssl_method_client = ssl_version_client @@ -279,16 +253,6 @@ def ssl_option_group(parser): dest="ssl_upstream_trusted_ca", help="Path to a PEM formatted trusted CA certificate." ) - group.add_argument( - "--ssl-port", - action="append", - type=int, - dest="ssl_ports", - default=list(TRANSPARENT_SSL_PORTS), - metavar="PORT", - help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " - "Defaults to %s." % - str(TRANSPARENT_SSL_PORTS)) group.add_argument( "--ssl-version-client", dest="ssl_version_client", type=str, default=tcp.SSL_DEFAULT_VERSION, choices=tcp.SSL_VERSIONS.keys(), -- cgit v1.2.3 From a86491eeed13c7889356e5102312f52bd86c3c66 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 27 Aug 2015 18:37:16 +0200 Subject: Revert "unify SSL version/method handling" This reverts commit 14e49f4fc7a38b63099ab0d42afd213b0d567c0f. --- libmproxy/proxy/config.py | 69 ++++++++++++++++++++++++++++++----------------- 1 file changed, 44 insertions(+), 25 deletions(-) diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 83030235..f438e9c2 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -49,11 +49,11 @@ class ProxyConfig: ciphers_client=None, ciphers_server=None, certs=[], - ssl_version_client=tcp.SSL_DEFAULT_METHOD, - ssl_version_server=tcp.SSL_DEFAULT_METHOD, + ssl_version_client="secure", + ssl_version_server="secure", ssl_verify_upstream_cert=False, ssl_upstream_trusted_cadir=None, - ssl_upstream_trusted_ca=None + ssl_upstream_trusted_ca=None, ): self.host = host self.port = port @@ -76,14 +76,10 @@ class ProxyConfig: for spec, cert in certs: self.certstore.add_cert_file(spec, cert) - if isinstance(ssl_version_client, int): - self.openssl_method_client = ssl_version_client - else: - self.openssl_method_client = tcp.SSL_VERSIONS[ssl_version_client] - if isinstance(ssl_version_server, int): - self.openssl_method_server = ssl_version_server - else: - self.openssl_method_server = tcp.SSL_VERSIONS[ssl_version_server] + self.openssl_method_client, self.openssl_options_client = version_to_openssl( + ssl_version_client) + self.openssl_method_server, self.openssl_options_server = version_to_openssl( + ssl_version_server) if ssl_verify_upstream_cert: self.openssl_verification_mode_server = SSL.VERIFY_PEER @@ -92,8 +88,33 @@ class ProxyConfig: self.openssl_trusted_cadir_server = ssl_upstream_trusted_cadir self.openssl_trusted_ca_server = ssl_upstream_trusted_ca - self.openssl_options_client = tcp.SSL_DEFAULT_OPTIONS - self.openssl_options_server = tcp.SSL_DEFAULT_OPTIONS + +sslversion_choices = ( + "all", + "secure", + "SSLv2", + "SSLv3", + "TLSv1", + "TLSv1_1", + "TLSv1_2") + + +def version_to_openssl(version): + """ + Convert a reasonable SSL version specification into the format OpenSSL expects. + Don't ask... + https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3 + """ + if version == "all": + return SSL.SSLv23_METHOD, None + elif version == "secure": + # SSLv23_METHOD + NO_SSLv2 + NO_SSLv3 == TLS 1.0+ + # TLSv1_METHOD would be TLS 1.0 only + return SSL.SSLv23_METHOD, (SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) + elif version in sslversion_choices: + return getattr(SSL, "%s_METHOD" % version), None + else: + raise ValueError("Invalid SSL version: %s" % version) def process_proxy_options(parser, options): @@ -254,18 +275,16 @@ def ssl_option_group(parser): help="Path to a PEM formatted trusted CA certificate." ) group.add_argument( - "--ssl-version-client", dest="ssl_version_client", type=str, default=tcp.SSL_DEFAULT_VERSION, - choices=tcp.SSL_VERSIONS.keys(), - help="""" - Use a specified protocol for client connections: - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. - Default to SSLv23.""" + "--ssl-version-client", dest="ssl_version_client", + default="secure", action="store", + choices=sslversion_choices, + help="Set supported SSL/TLS version for client connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure." ) group.add_argument( - "--ssl-version-server", dest="ssl_version_server", type=str, default=tcp.SSL_DEFAULT_VERSION, - choices=tcp.SSL_VERSIONS.keys(), - help="""" - Use a specified protocol for server connections: - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. - Default to SSLv23.""" + "--ssl-version-server", dest="ssl_version_server", + default="secure", action="store", + choices=sslversion_choices, + help="Set supported SSL/TLS version for server connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure." ) -- cgit v1.2.3 From 1cc48345e13917aadc1e0fd93d6011139e78e3d9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 28 Aug 2015 01:51:13 +0200 Subject: clean up config/cmdline, fix bugs, remove cruft --- libmproxy/cmdline.py | 247 +++++++++++++++++++++-------------- libmproxy/flow.py | 6 +- libmproxy/protocol/http.py | 2 +- libmproxy/protocol2/__init__.py | 7 +- libmproxy/protocol2/reverse_proxy.py | 5 +- libmproxy/protocol2/root_context.py | 10 +- libmproxy/protocol2/socks_proxy.py | 2 +- libmproxy/protocol2/tls.py | 38 +++++- libmproxy/proxy/config.py | 196 ++++++++------------------- libmproxy/proxy/server.py | 38 +++++- test/test_cmdline.py | 14 +- test/test_flow.py | 8 +- test/test_proxy.py | 8 +- test/test_server.py | 53 +------- test/tservers.py | 14 +- 15 files changed, 303 insertions(+), 345 deletions(-) diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index d033fb76..1d897717 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -2,8 +2,8 @@ from __future__ import absolute_import import os import re import configargparse +from netlib.tcp import Address -from netlib import http import netlib.utils from . import filt, utils, version @@ -102,32 +102,22 @@ def parse_setheader(s): return _parse_hook(s) -def parse_server_spec(url): +def parse_server_spec(url, allowed_schemes=("http", "https")): p = netlib.utils.parse_url(url) - if not p or not p[1] or p[0] not in ("http", "https"): + if not p or not p[1] or p[0] not in allowed_schemes: raise configargparse.ArgumentTypeError( "Invalid server specification: %s" % url ) - - if p[0].lower() == "https": - ssl = [True, True] - else: - ssl = [False, False] - - return ssl + list(p[1:3]) + address = Address(p[1:3]) + scheme = p[0].lower() + return config.ServerSpec(scheme, address) def parse_server_spec_special(url): """ Provides additional support for http2https and https2http schemes. """ - normalized_url = re.sub("^https?2", "", url) - ret = parse_server_spec(normalized_url) - if url.lower().startswith("https2http"): - ret[0] = True - elif url.lower().startswith("http2https"): - ret[0] = False - return ret + return parse_server_spec(url, allowed_schemes=("http", "https", "http2https", "https2http")) def get_common_options(options): @@ -192,24 +182,24 @@ def get_common_options(options): outfile=options.outfile, verbosity=options.verbose, nopop=options.nopop, - replay_ignore_content = options.replay_ignore_content, - replay_ignore_params = options.replay_ignore_params, - replay_ignore_payload_params = options.replay_ignore_payload_params, - replay_ignore_host = options.replay_ignore_host + replay_ignore_content=options.replay_ignore_content, + replay_ignore_params=options.replay_ignore_params, + replay_ignore_payload_params=options.replay_ignore_payload_params, + replay_ignore_host=options.replay_ignore_host ) -def common_options(parser): +def basic_options(parser): parser.add_argument( '--version', - action= 'version', - version= "%(prog)s" + " " + version.VERSION + action='version', + version="%(prog)s" + " " + version.VERSION ) parser.add_argument( '--shortversion', - action= 'version', - help = "show program's short version number and exit", - version = version.VERSION + action='version', + help="show program's short version number and exit", + version=version.VERSION ) parser.add_argument( "--anticache", @@ -301,11 +291,42 @@ def common_options(parser): """ ) + +def proxy_modes(parser): + group = parser.add_argument_group("Proxy Modes").add_mutually_exclusive_group() + group.add_argument( + "-R", "--reverse", + action="store", + type=parse_server_spec_special, + dest="reverse_proxy", + default=None, + help=""" + Forward all requests to upstream HTTP server: + http[s][2http[s]]://host[:port] + """ + ) + group.add_argument( + "--socks", + action="store_true", dest="socks_proxy", default=False, + help="Set SOCKS5 proxy mode." + ) + group.add_argument( + "-T", "--transparent", + action="store_true", dest="transparent_proxy", default=False, + help="Set transparent proxy mode." + ) + group.add_argument( + "-U", "--upstream", + action="store", + type=parse_server_spec, + dest="upstream_proxy", + default=None, + help="Forward all requests to upstream proxy server: http://host[:port]" + ) + + +def proxy_options(parser): group = parser.add_argument_group("Proxy Options") - # We could make a mutually exclusive group out of -R, -U, -T, but we don't - # do that because - --upstream-server should be in that group as well, but - # it's already in a different group. - our own error messages are more - # helpful group.add_argument( "-b", "--bind-address", action="store", type=str, dest="addr", default='', @@ -344,70 +365,78 @@ def common_options(parser): action="store", type=int, dest="port", default=8080, help="Proxy service port." ) + + +def proxy_ssl_options(parser): + # TODO: Agree to consistently either use "upstream" or "server". + group = parser.add_argument_group("SSL") group.add_argument( - "-R", "--reverse", - action="store", - type=parse_server_spec_special, - dest="reverse_proxy", - default=None, - help=""" - Forward all requests to upstream HTTP server: - http[s][2http[s]]://host[:port] - """ - ) + "--cert", + dest='certs', + default=[], + type=str, + metavar="SPEC", + action="append", + help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' + 'The domain may include a wildcard, and is equal to "*" if not specified. ' + 'The file at path is a certificate in PEM format. If a private key is included ' + 'in the PEM, it is used, else the default key in the conf dir is used. ' + 'The PEM file should contain the full certificate chain, with the leaf certificate ' + 'as the first entry. Can be passed multiple times.') group.add_argument( - "--socks", - action="store_true", dest="socks_proxy", default=False, - help="Set SOCKS5 proxy mode." + "--ciphers-client", action="store", + type=str, dest="ciphers_client", default=config.DEFAULT_CLIENT_CIPHERS, + help="Set supported ciphers for client connections. (OpenSSL Syntax)" ) group.add_argument( - "-T", "--transparent", - action="store_true", dest="transparent_proxy", default=False, - help="Set transparent proxy mode." + "--ciphers-server", action="store", + type=str, dest="ciphers_server", default=None, + help="Set supported ciphers for server connections. (OpenSSL Syntax)" ) group.add_argument( - "-U", "--upstream", - action="store", - type=parse_server_spec, - dest="upstream_proxy", - default=None, - help="Forward all requests to upstream proxy server: http://host[:port]" + "--client-certs", action="store", + type=str, dest="clientcerts", default=None, + help="Client certificate directory." ) group.add_argument( - "--spoof", - action="store_true", dest="spoof_mode", default=False, - help="Use Host header to connect to HTTP servers." + "--no-upstream-cert", default=False, + action="store_true", dest="no_upstream_cert", + help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--ssl-spoof", - action="store_true", dest="ssl_spoof_mode", default=False, - help="Use TLS SNI to connect to HTTPS servers." + "--verify-upstream-cert", default=False, + action="store_true", dest="ssl_verify_upstream_cert", + help="Verify upstream server SSL/TLS certificates and fail if invalid " + "or not present." ) group.add_argument( - "--spoofed-port", - action="store", dest="spoofed_ssl_port", type=int, default=443, - help="Port number of upstream HTTPS servers in SSL spoof mode." + "--upstream-trusted-cadir", default=None, action="store", + dest="ssl_verify_upstream_trusted_cadir", + help="Path to a directory of trusted CA certificates for upstream " + "server verification prepared using the c_rehash tool." ) - - group = parser.add_argument_group( - "Advanced Proxy Options", - """ - The following options allow a custom adjustment of the proxy - behavior. Normally, you don't want to use these options directly and - use the provided wrappers instead (-R, -U, -T). - """ + group.add_argument( + "--upstream-trusted-ca", default=None, action="store", + dest="ssl_verify_upstream_trusted_ca", + help="Path to a PEM formatted trusted CA certificate." ) group.add_argument( - "--http-form-in", dest="http_form_in", default=None, - action="store", choices=("relative", "absolute"), - help="Override the HTTP request form accepted by the proxy" + "--ssl-version-client", dest="ssl_version_client", + default="secure", action="store", + choices=config.sslversion_choices.keys(), + help="Set supported SSL/TLS version for client connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) group.add_argument( - "--http-form-out", dest="http_form_out", default=None, - action="store", choices=("relative", "absolute"), - help="Override the HTTP request form sent upstream by the proxy" + "--ssl-version-server", dest="ssl_version_server", + default="secure", action="store", + choices=config.sslversion_choices.keys(), + help="Set supported SSL/TLS version for server connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) + +def onboarding_app(parser): group = parser.add_argument_group("Onboarding App") group.add_argument( "--noapp", @@ -433,6 +462,8 @@ def common_options(parser): help="Port to serve the onboarding app from." ) + +def client_replay(parser): group = parser.add_argument_group("Client Replay") group.add_argument( "-c", "--client-replay", @@ -440,6 +471,8 @@ def common_options(parser): help="Replay client requests from a saved file." ) + +def server_replay(parser): group = parser.add_argument_group("Server Replay") group.add_argument( "-S", "--server-replay", @@ -504,6 +537,8 @@ def common_options(parser): default=False, help="Ignore request's destination host while searching for a saved flow to replay") + +def replacements(parser): group = parser.add_argument_group( "Replacements", """ @@ -520,14 +555,16 @@ def common_options(parser): ) group.add_argument( "--replace-from-file", - action = "append", type=str, dest="replace_file", default=[], - metavar = "PATH", - help = """ + action="append", type=str, dest="replace_file", default=[], + metavar="PATH", + help=""" Replacement pattern, where the replacement clause is a path to a file. """ ) + +def set_headers(parser): group = parser.add_argument_group( "Set Headers", """ @@ -543,21 +580,22 @@ def common_options(parser): help="Header set pattern." ) + +def proxy_authentication(parser): group = parser.add_argument_group( "Proxy Authentication", """ Specify which users are allowed to access the proxy and the method used for authenticating them. """ - ) - user_specification_group = group.add_mutually_exclusive_group() - user_specification_group.add_argument( + ).add_mutually_exclusive_group() + group.add_argument( "--nonanonymous", action="store_true", dest="auth_nonanonymous", help="Allow access to any user long as a credentials are specified." ) - user_specification_group.add_argument( + group.add_argument( "--singleuser", action="store", dest="auth_singleuser", type=str, metavar="USER", @@ -566,14 +604,25 @@ def common_options(parser): username:password. """ ) - user_specification_group.add_argument( + group.add_argument( "--htpasswd", action="store", dest="auth_htpasswd", type=str, metavar="PATH", help="Allow access to users specified in an Apache htpasswd file." ) - config.ssl_option_group(parser) + +def common_options(parser): + basic_options(parser) + proxy_modes(parser) + proxy_options(parser) + proxy_ssl_options(parser) + onboarding_app(parser) + client_replay(parser) + server_replay(parser) + replacements(parser) + set_headers(parser) + proxy_authentication(parser) def mitmproxy(): @@ -583,13 +632,13 @@ def mitmproxy(): parser = configargparse.ArgumentParser( usage="%(prog)s [options]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmproxy.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) common_options(parser) parser.add_argument( @@ -628,20 +677,20 @@ def mitmproxy(): def mitmdump(): parser = configargparse.ArgumentParser( usage="%(prog)s [options] [filter]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmdump.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) common_options(parser) parser.add_argument( "--keepserving", - action= "store_true", dest="keepserving", default=False, - help= """ + action="store_true", dest="keepserving", default=False, + help=""" Continue serving after client playback or file read. We exit by default. """ @@ -658,13 +707,13 @@ def mitmdump(): def mitmweb(): parser = configargparse.ArgumentParser( usage="%(prog)s [options]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmweb.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) group = parser.add_argument_group("Mitmweb") diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 8605d7a1..a2b807ba 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -860,9 +860,9 @@ class FlowMaster(controller.Master): """ if self.server and self.server.config.mode == "reverse": - f.request.host, f.request.port = self.server.config.mode.dst[2:] - f.request.scheme = "https" if self.server.config.mode.dst[ - 1] else "http" + f.request.host = self.server.config.upstream_server.address.host + f.request.port = self.server.config.upstream_server.address.port + f.request.scheme = re.sub("^https?2", "", self.server.config.upstream_server.scheme) f.reply = controller.DummyReply() if f.request: diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4472cb2a..56d7d57f 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -10,6 +10,7 @@ from email.utils import parsedate_tz, formatdate, mktime_tz import netlib from netlib import http, tcp, odict, utils, encoding from netlib.http import cookies, http1, http2 +from netlib.http.http1 import HTTP1Protocol from netlib.http.semantics import CONTENT_MISSING from .tcp import TCPHandler @@ -757,7 +758,6 @@ class RequestReplayThread(threading.Thread): server.send(self.flow.server_conn.protocol.assemble(r)) self.flow.server_conn = server - self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) self.flow.response = HTTPResponse.from_protocol( self.flow.server_conn.protocol, r.method, diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index d5dafaae..61b9a77e 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -3,8 +3,11 @@ from .root_context import RootContext from .socks_proxy import Socks5Proxy from .reverse_proxy import ReverseProxy from .http_proxy import HttpProxy, HttpUpstreamProxy -from .rawtcp import RawTcpLayer +from .transparent_proxy import TransparentProxy +from .http import make_error_response __all__ = [ - "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" + "RootContext", + "Socks5Proxy", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "TransparentProxy", + "make_error_response" ] diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index 9d5a4beb..76163c71 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -12,10 +12,7 @@ class ReverseProxy(Layer, ServerConnectionMixin): self._server_tls = server_tls def __call__(self): - if self._client_tls or self._server_tls: - layer = TlsLayer(self, self._client_tls, self._server_tls) - else: - layer = self.ctx.next_layer(self) + layer = TlsLayer(self, self._client_tls, self._server_tls) try: layer() diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 78d48453..af0e7a37 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -4,7 +4,7 @@ from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol from .rawtcp import RawTcpLayer -from .tls import TlsLayer +from .tls import TlsLayer, is_tls_record_magic from .http import Http1Layer, Http2Layer @@ -38,13 +38,7 @@ class RootContext(object): # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello d = top_layer.client_conn.rfile.peek(3) - is_tls_client_hello = ( - len(d) == 3 and - d[0] == '\x16' and - d[1] == '\x03' and - d[2] in ('\x00', '\x01', '\x02', '\x03') - ) - if is_tls_client_hello: + if is_tls_record_magic(d): return TlsLayer(top_layer, True, True) # 3. Check for --tcp diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py index 18b363d5..91935d24 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/protocol2/socks_proxy.py @@ -8,7 +8,7 @@ from .layer import Layer, ServerConnectionMixin class Socks5Proxy(Layer, ServerConnectionMixin): def __call__(self): try: - s5mode = Socks5ProxyMode(self.config.ssl_ports) + s5mode = Socks5ProxyMode([]) address = s5mode.get_upstream_server(self.client_conn)[2:] except ProxyError as e: # TODO: Unmonkeypatch diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index b1b80034..850bf5dc 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -11,6 +11,21 @@ from ..exceptions import ProtocolException from .layer import Layer +def is_tls_record_magic(d): + """ + Returns: + True, if the passed bytes start with the TLS record magic bytes. + False, otherwise. + """ + d = d[:3] + return ( + len(d) == 3 and + d[0] == '\x16' and + d[1] == '\x03' and + d[2] in ('\x00', '\x01', '\x02', '\x03') + ) + + class TlsLayer(Layer): def __init__(self, ctx, client_tls, server_tls): self.client_sni = None @@ -69,9 +84,13 @@ class TlsLayer(Layer): client_hello_size = 1 offset = 0 while len(client_hello) < client_hello_size: - record_header = self.client_conn.rfile.peek(offset+5)[offset:] + record_header = self.client_conn.rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) != 5: + raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) record_size = struct.unpack("!H", record_header[3:])[0] + 5 - record_body = self.client_conn.rfile.peek(offset+record_size)[offset+5:] + record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) client_hello += record_body offset += record_size client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 @@ -81,7 +100,12 @@ class TlsLayer(Layer): """ Peek into the connection, read the initial client hello and parse it to obtain ALPN values. """ - raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + try: + raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + except ProtocolException as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + return + try: client_hello = ClientHello.parse(raw_client_hello) except ConstructError as e: @@ -97,7 +121,10 @@ class TlsLayer(Layer): elif extension.type == 0x10: self.client_alpn_protocols = list(extension.alpn_protocols) - self.log("Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), "debug") + self.log( + "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), + "debug" + ) def connect(self): if not self.server_conn: @@ -226,7 +253,8 @@ class TlsLayer(Layer): host = self.server_conn.address.host sans = set() # Incorporate upstream certificate - if self.server_conn and self.server_conn.tls_established and (not self.config.no_upstream_cert): + if self.server_conn and self.server_conn.tls_established and ( + not self.config.no_upstream_cert): upstream_cert = self.server_conn.cert sans.update(upstream_cert.altnames) if upstream_cert.cn: diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index f438e9c2..8ab5a216 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,4 +1,5 @@ from __future__ import absolute_import +import collections import os import re from OpenSSL import SSL @@ -7,6 +8,7 @@ from netlib import certutils, tcp from netlib.http import authentication from .. import utils, platform +from netlib.tcp import Address CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" @@ -15,8 +17,9 @@ CA_DIR = "~/.mitmproxy" # https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" + class HostMatcher(object): - def __init__(self, patterns=[]): + def __init__(self, patterns=tuple()): self.patterns = list(patterns) self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] @@ -32,6 +35,9 @@ class HostMatcher(object): return bool(self.patterns) +ServerSpec = collections.namedtuple("ServerSpec", "scheme address") + + class ProxyConfig: def __init__( self, @@ -41,19 +47,19 @@ class ProxyConfig: clientcerts=None, no_upstream_cert=False, body_size_limit=None, - mode=None, + mode="regular", upstream_server=None, authenticator=None, - ignore_hosts=[], - tcp_hosts=[], + ignore_hosts=tuple(), + tcp_hosts=tuple(), ciphers_client=None, ciphers_server=None, - certs=[], + certs=tuple(), ssl_version_client="secure", ssl_version_server="secure", ssl_verify_upstream_cert=False, - ssl_upstream_trusted_cadir=None, - ssl_upstream_trusted_ca=None, + ssl_verify_upstream_trusted_cadir=None, + ssl_verify_upstream_trusted_ca=None, ): self.host = host self.port = port @@ -63,7 +69,10 @@ class ProxyConfig: self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit self.mode = mode - self.upstream_server = upstream_server + if upstream_server: + self.upstream_server = ServerSpec(upstream_server[0], Address.wrap(upstream_server[1])) + else: + self.upstream_server = None self.check_ignore = HostMatcher(ignore_hosts) self.check_tcp = HostMatcher(tcp_hosts) @@ -76,57 +85,46 @@ class ProxyConfig: for spec, cert in certs: self.certstore.add_cert_file(spec, cert) - self.openssl_method_client, self.openssl_options_client = version_to_openssl( - ssl_version_client) - self.openssl_method_server, self.openssl_options_server = version_to_openssl( - ssl_version_server) + self.openssl_method_client, self.openssl_options_client = \ + sslversion_choices[ssl_version_client] + self.openssl_method_server, self.openssl_options_server = \ + sslversion_choices[ssl_version_server] if ssl_verify_upstream_cert: self.openssl_verification_mode_server = SSL.VERIFY_PEER else: self.openssl_verification_mode_server = SSL.VERIFY_NONE - self.openssl_trusted_cadir_server = ssl_upstream_trusted_cadir - self.openssl_trusted_ca_server = ssl_upstream_trusted_ca - - -sslversion_choices = ( - "all", - "secure", - "SSLv2", - "SSLv3", - "TLSv1", - "TLSv1_1", - "TLSv1_2") - - -def version_to_openssl(version): - """ - Convert a reasonable SSL version specification into the format OpenSSL expects. - Don't ask... - https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3 - """ - if version == "all": - return SSL.SSLv23_METHOD, None - elif version == "secure": - # SSLv23_METHOD + NO_SSLv2 + NO_SSLv3 == TLS 1.0+ - # TLSv1_METHOD would be TLS 1.0 only - return SSL.SSLv23_METHOD, (SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3) - elif version in sslversion_choices: - return getattr(SSL, "%s_METHOD" % version), None - else: - raise ValueError("Invalid SSL version: %s" % version) + self.openssl_trusted_cadir_server = ssl_verify_upstream_trusted_cadir + self.openssl_trusted_ca_server = ssl_verify_upstream_trusted_ca + + +""" +Map a reasonable SSL version specification into the format OpenSSL expects. +Don't ask... +https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3 +""" +sslversion_choices = { + "all": (SSL.SSLv23_METHOD, 0), + # SSLv23_METHOD + NO_SSLv2 + NO_SSLv3 == TLS 1.0+ + # TLSv1_METHOD would be TLS 1.0 only + "secure": (SSL.SSLv23_METHOD, (SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)), + "SSLv2": (SSL.SSLv2_METHOD, 0), + "SSLv3": (SSL.SSLv3_METHOD, 0), + "TLSv1": (SSL.TLSv1_METHOD, 0), + "TLSv1_1": (SSL.TLSv1_1_METHOD, 0), + "TLSv1_2": (SSL.TLSv1_2_METHOD, 0), +} def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) c = 0 - mode, upstream_server, spoofed_ssl_port = None, None, None + mode, upstream_server = "regular", None if options.transparent_proxy: c += 1 if not platform.resolver: - return parser.error( - "Transparent mode not supported on this platform.") + return parser.error("Transparent mode not supported on this platform.") mode = "transparent" if options.socks_proxy: c += 1 @@ -139,32 +137,26 @@ def process_proxy_options(parser, options): c += 1 mode = "upstream" upstream_server = options.upstream_proxy - if options.spoof_mode: - c += 1 - mode = "spoof" - if options.ssl_spoof_mode: - c += 1 - mode = "sslspoof" - spoofed_ssl_port = options.spoofed_ssl_port if c > 1: return parser.error( "Transparent, SOCKS5, reverse and upstream proxy mode " - "are mutually exclusive.") + "are mutually exclusive. Read the docs on proxy modes to understand why." + ) if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) - if not os.path.exists( - options.clientcerts) or not os.path.isdir( - options.clientcerts): + if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): return parser.error( "Client certificate directory does not exist or is not a directory: %s" % - options.clientcerts) + options.clientcerts + ) - if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd): + if options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd: if options.auth_singleuser: if len(options.auth_singleuser.split(':')) != 2: return parser.error( - "Invalid single-user specification. Please use the format username:password") + "Invalid single-user specification. Please use the format username:password" + ) username, password = options.auth_singleuser.split(':') password_manager = authentication.PassManSingleUser(username, password) elif options.auth_nonanonymous: @@ -189,12 +181,6 @@ def process_proxy_options(parser, options): parser.error("Certificate file does not exist: %s" % parts[1]) certs.append(parts) - ssl_ports = options.ssl_ports - if options.ssl_ports != TRANSPARENT_SSL_PORTS: - # arparse appends to default value by default, strip that off. - # see http://bugs.python.org/issue16399 - ssl_ports = ssl_ports[len(TRANSPARENT_SSL_PORTS):] - return ProxyConfig( host=options.addr, port=options.port, @@ -204,87 +190,15 @@ def process_proxy_options(parser, options): body_size_limit=body_size_limit, mode=mode, upstream_server=upstream_server, - http_form_in=options.http_form_in, - http_form_out=options.http_form_out, ignore_hosts=options.ignore_hosts, tcp_hosts=options.tcp_hosts, authenticator=authenticator, ciphers_client=options.ciphers_client, ciphers_server=options.ciphers_server, - certs=certs, + certs=tuple(certs), ssl_version_client=options.ssl_version_client, ssl_version_server=options.ssl_version_server, - ssl_ports=ssl_ports, - spoofed_ssl_port=spoofed_ssl_port, ssl_verify_upstream_cert=options.ssl_verify_upstream_cert, - ssl_upstream_trusted_cadir=options.ssl_upstream_trusted_cadir, - ssl_upstream_trusted_ca=options.ssl_upstream_trusted_ca - ) - - -def ssl_option_group(parser): - group = parser.add_argument_group("SSL") - group.add_argument( - "--cert", - dest='certs', - default=[], - type=str, - metavar="SPEC", - action="append", - help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' - 'The domain may include a wildcard, and is equal to "*" if not specified. ' - 'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' - 'it is used, else the default key in the conf dir is used. ' - 'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. ' - 'Can be passed multiple times.') - group.add_argument( - "--ciphers-client", action="store", - type=str, dest="ciphers_client", default=DEFAULT_CLIENT_CIPHERS, - help="Set supported ciphers for client connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--ciphers-server", action="store", - type=str, dest="ciphers_server", default=None, - help="Set supported ciphers for server connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--client-certs", action="store", - type=str, dest="clientcerts", default=None, - help="Client certificate directory." - ) - group.add_argument( - "--no-upstream-cert", default=False, - action="store_true", dest="no_upstream_cert", - help="Don't connect to upstream server to look up certificate details." - ) - group.add_argument( - "--verify-upstream-cert", default=False, - action="store_true", dest="ssl_verify_upstream_cert", - help="Verify upstream server SSL/TLS certificates and fail if invalid " - "or not present." - ) - group.add_argument( - "--upstream-trusted-cadir", default=None, action="store", - dest="ssl_upstream_trusted_cadir", - help="Path to a directory of trusted CA certificates for upstream " - "server verification prepared using the c_rehash tool." - ) - group.add_argument( - "--upstream-trusted-ca", default=None, action="store", - dest="ssl_upstream_trusted_ca", - help="Path to a PEM formatted trusted CA certificate." - ) - group.add_argument( - "--ssl-version-client", dest="ssl_version_client", - default="secure", action="store", - choices=sslversion_choices, - help="Set supported SSL/TLS version for client connections. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure." - ) - group.add_argument( - "--ssl-version-server", dest="ssl_version_server", - default="secure", action="store", - choices=sslversion_choices, - help="Set supported SSL/TLS version for server connections. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure." - ) + ssl_verify_upstream_trusted_cadir=options.ssl_verify_upstream_trusted_cadir, + ssl_verify_upstream_trusted_ca=options.ssl_verify_upstream_trusted_ca + ) \ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 19ddb930..1fc4cbda 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -5,6 +5,8 @@ import sys import socket from libmproxy.protocol2.layer import Kill from netlib import tcp +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError from ..protocol.handle import protocol_handler from .. import protocol2 @@ -82,11 +84,31 @@ class ConnectionHandler2: self.channel ) - # FIXME: properly parse config - if self.config.mode == "upstream": - root_layer = protocol2.HttpUpstreamProxy(root_context, ("localhost", 8081)) - else: + mode = self.config.mode + if mode == "upstream": + root_layer = protocol2.HttpUpstreamProxy( + root_context, + self.config.upstream_server.address + ) + elif mode == "transparent": + root_layer = protocol2.TransparentProxy(root_context) + elif mode == "reverse": + client_tls = self.config.upstream_server.scheme.startswith("https") + server_tls = self.config.upstream_server.scheme.endswith("https") + root_layer = protocol2.ReverseProxy( + root_context, + self.config.upstream_server.address, + client_tls, + server_tls + ) + elif mode == "socks5": + root_layer = protocol2.Socks5Proxy(root_context) + elif mode == "regular": root_layer = protocol2.HttpProxy(root_context) + elif callable(mode): # pragma: nocover + root_layer = mode(root_context) + else: # pragma: nocover + raise ValueError("Unknown proxy mode: %s" % mode) try: root_layer() @@ -94,6 +116,14 @@ class ConnectionHandler2: self.log("Connection killed", "info") except ProtocolException as e: self.log(e, "info") + # If an error propagates to the topmost level, + # we send an HTTP error response, which is both + # understandable by HTTP clients and humans. + try: + error_response = protocol2.make_error_response(502, repr(e)) + self.client_conn.send(HTTP1Protocol().assemble(error_response)) + except NetLibError: + pass except Exception: self.log(traceback.format_exc(), "error") print(traceback.format_exc(), file=sys.stderr) diff --git a/test/test_cmdline.py b/test/test_cmdline.py index eafcbde4..ee2f7044 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -38,15 +38,15 @@ def test_parse_replace_hook(): def test_parse_server_spec(): tutils.raises("Invalid server specification", cmdline.parse_server_spec, "") assert cmdline.parse_server_spec( - "http://foo.com:88") == [False, False, "foo.com", 88] + "http://foo.com:88") == ("http", ("foo.com", 88)) assert cmdline.parse_server_spec( - "http://foo.com") == [False, False, "foo.com", 80] + "http://foo.com") == ("http", ("foo.com", 80)) assert cmdline.parse_server_spec( - "https://foo.com") == [True, True, "foo.com", 443] + "https://foo.com") == ("https", ("foo.com", 443)) assert cmdline.parse_server_spec_special( - "https2http://foo.com") == [True, False, "foo.com", 80] + "https2http://foo.com") == ("https2http", ("foo.com", 80)) assert cmdline.parse_server_spec_special( - "http2https://foo.com") == [False, True, "foo.com", 443] + "http2https://foo.com") == ("http2https", ("foo.com", 443)) tutils.raises( "Invalid server specification", cmdline.parse_server_spec, @@ -55,6 +55,10 @@ def test_parse_server_spec(): "Invalid server specification", cmdline.parse_server_spec, "http://") + tutils.raises( + "Invalid server specification", + cmdline.parse_server_spec, + "https2http://foo.com") def test_parse_setheaders(): diff --git a/test/test_flow.py b/test/test_flow.py index 711688da..5c49deed 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -4,6 +4,7 @@ import os.path from cStringIO import StringIO import email.utils import mock +from libmproxy.cmdline import parse_server_spec import netlib.utils from netlib import odict @@ -672,11 +673,8 @@ class TestSerialize: s = flow.State() conf = ProxyConfig( mode="reverse", - upstream_server=[ - True, - True, - "use-this-domain", - 80]) + upstream_server=("https", ("use-this-domain", 80)) + ) fm = flow.FlowMaster(DummyServer(conf), s) fm.load_flows(r) assert s.flows[0].request.host == "use-this-domain" diff --git a/test/test_proxy.py b/test/test_proxy.py index 6ab19e02..9c01ab63 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -97,13 +97,7 @@ class TestProcessProxyOptions: self.assert_err("expected one argument", "-U") self.assert_err("Invalid server specification", "-U", "upstream") - self.assert_noerr("--spoof") - self.assert_noerr("--ssl-spoof") - - self.assert_noerr("--spoofed-port", "443") - self.assert_err("expected one argument", "--spoofed-port") - - self.assert_err("mutually exclusive", "-R", "http://localhost", "-T") + self.assert_err("not allowed with", "-R", "http://localhost", "-T") def test_client_certs(self): with tutils.tmpdir() as cadir: diff --git a/test/test_server.py b/test/test_server.py index e9c40d1a..1216a349 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -464,60 +464,11 @@ class TestSocks5(tservers.SocksModeTest): assert "SOCKS5 mode failure" in f.content -class TestSpoof(tservers.SpoofModeTest): - def test_http(self): - alist = ( - ("localhost", self.server.port), - ("127.0.0.1", self.server.port) - ) - for a in alist: - self.server.clear_log() - p = self.pathoc() - f = p.request("get:/p/304:h'Host'='%s:%s'" % a) - assert self.server.last_log() - assert f.status_code == 304 - l = self.master.state.view[-1] - assert l.server_conn.address - assert l.server_conn.address.host == a[0] - assert l.server_conn.address.port == a[1] - - def test_http_without_host(self): - p = self.pathoc() - f = p.request("get:/p/304:r") - assert f.status_code == 400 - - -class TestSSLSpoof(tservers.SSLSpoofModeTest): - def test_https(self): - alist = ( - ("localhost", self.server.port), - ("127.0.0.1", self.server.port) - ) - for a in alist: - self.server.clear_log() - self.config.mode.sslport = a[1] - p = self.pathoc(sni=a[0]) - f = p.request("get:/p/304") - assert self.server.last_log() - assert f.status_code == 304 - l = self.master.state.view[-1] - assert l.server_conn.address - assert l.server_conn.address.host == a[0] - assert l.server_conn.address.port == a[1] - - def test_https_without_sni(self): - a = ("localhost", self.server.port) - self.config.mode.sslport = a[1] - p = self.pathoc(sni=None) - f = p.request("get:/p/304") - assert f.status_code == 400 - - class TestHttps2Http(tservers.ReverseProxTest): @classmethod def get_proxy_config(cls): d = super(TestHttps2Http, cls).get_proxy_config() - d["upstream_server"][0] = True + d["upstream_server"] = ("https2http", d["upstream_server"][1]) return d def pathoc(self, ssl, sni=None): @@ -541,7 +492,7 @@ class TestHttps2Http(tservers.ReverseProxTest): def test_http(self): p = self.pathoc(ssl=False) - assert p.request("get:'/p/200'").status_code == 400 + assert p.request("get:'/p/200'").status_code == 502 class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin): diff --git a/test/tservers.py b/test/tservers.py index 3c73b262..43ebf2bb 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -1,6 +1,5 @@ import os.path import threading -import Queue import shutil import tempfile import flask @@ -130,7 +129,6 @@ class ProxTestBase(object): no_upstream_cert = cls.no_upstream_cert, cadir = cls.cadir, authenticator = cls.authenticator, - ssl_ports=([cls.server.port, cls.server2.port] if cls.ssl else []), clientcerts = tutils.test_data.path("data/clientcert") if cls.clientcerts else None ) @@ -235,12 +233,10 @@ class ReverseProxTest(ProxTestBase): @classmethod def get_proxy_config(cls): d = ProxTestBase.get_proxy_config() - d["upstream_server"] = [ - True if cls.ssl else False, - True if cls.ssl else False, - "127.0.0.1", - cls.server.port - ] + d["upstream_server"] = ( + "https" if cls.ssl else "http", + ("127.0.0.1", cls.server.port) + ) d["mode"] = "reverse" return d @@ -360,7 +356,7 @@ class ChainProxTest(ProxTestBase): if cls.chain: # First proxy is in normal mode. d.update( mode="upstream", - upstream_server=(False, False, "127.0.0.1", cls.chain[0].port) + upstream_server=("http", ("127.0.0.1", cls.chain[0].port)) ) return d -- cgit v1.2.3 From 2dfba2105b4b5ad094ee364124c0552d2e4a4947 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 29 Aug 2015 12:34:01 +0200 Subject: move sslversion mapping to netlib --- libmproxy/cmdline.py | 10 +++++----- libmproxy/proxy/config.py | 20 +------------------- 2 files changed, 6 insertions(+), 24 deletions(-) diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 1d897717..591e87ed 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -2,7 +2,7 @@ from __future__ import absolute_import import os import re import configargparse -from netlib.tcp import Address +from netlib.tcp import Address, sslversion_choices import netlib.utils @@ -423,15 +423,15 @@ def proxy_ssl_options(parser): group.add_argument( "--ssl-version-client", dest="ssl_version_client", default="secure", action="store", - choices=config.sslversion_choices.keys(), - help="Set supported SSL/TLS version for client connections. " + choices=sslversion_choices.keys(), + help="Set supported SSL/TLS versions for client connections. " "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) group.add_argument( "--ssl-version-server", dest="ssl_version_server", default="secure", action="store", - choices=config.sslversion_choices.keys(), - help="Set supported SSL/TLS version for server connections. " + choices=sslversion_choices.keys(), + help="Set supported SSL/TLS versions for server connections. " "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 8ab5a216..415ee215 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -8,7 +8,7 @@ from netlib import certutils, tcp from netlib.http import authentication from .. import utils, platform -from netlib.tcp import Address +from netlib.tcp import Address, sslversion_choices CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" @@ -98,24 +98,6 @@ class ProxyConfig: self.openssl_trusted_ca_server = ssl_verify_upstream_trusted_ca -""" -Map a reasonable SSL version specification into the format OpenSSL expects. -Don't ask... -https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3 -""" -sslversion_choices = { - "all": (SSL.SSLv23_METHOD, 0), - # SSLv23_METHOD + NO_SSLv2 + NO_SSLv3 == TLS 1.0+ - # TLSv1_METHOD would be TLS 1.0 only - "secure": (SSL.SSLv23_METHOD, (SSL.OP_NO_SSLv2 | SSL.OP_NO_SSLv3)), - "SSLv2": (SSL.SSLv2_METHOD, 0), - "SSLv3": (SSL.SSLv3_METHOD, 0), - "TLSv1": (SSL.TLSv1_METHOD, 0), - "TLSv1_1": (SSL.TLSv1_1_METHOD, 0), - "TLSv1_2": (SSL.TLSv1_2_METHOD, 0), -} - - def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) -- cgit v1.2.3 From 63844df34367bf7147c2d43a9e4061515f6430c9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 29 Aug 2015 14:28:11 +0200 Subject: fix streaming --- libmproxy/protocol2/http.py | 192 +++++++++++++++++++++++++++--------------- test/scripts/stream_modify.py | 4 +- 2 files changed, 124 insertions(+), 72 deletions(-) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 792cf266..0fde9fb1 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -25,32 +25,101 @@ from netlib.http.http2 import HTTP2Protocol # TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. -class Http1Layer(Layer): +class _HttpLayer(Layer): + supports_streaming = False + + def read_request(self): + raise NotImplementedError() + + def send_request(self, request): + raise NotImplementedError() + + def read_response(self, request_method): + raise NotImplementedError() + + def send_response(self, response): + raise NotImplementedError() + +class _StreamingHttpLayer(_HttpLayer): + supports_streaming = True + + def read_response_headers(self): + raise NotImplementedError + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + raise NotImplementedError() + yield "this is a generator" + + def send_response_headers(self, response): + raise NotImplementedError + + def send_response_body(self, response, chunks): + raise NotImplementedError() + + +class Http1Layer(_StreamingHttpLayer): + def __init__(self, ctx, mode): super(Http1Layer, self).__init__(ctx) self.mode = mode self.client_protocol = HTTP1Protocol(self.client_conn) self.server_protocol = HTTP1Protocol(self.server_conn) - def read_from_client(self): + def read_request(self): return HTTPRequest.from_protocol( self.client_protocol, body_size_limit=self.config.body_size_limit ) - def read_from_server(self, request_method): + def send_request(self, request): + self.server_conn.send(self.server_protocol.assemble(request)) + + def read_response(self, request_method): return HTTPResponse.from_protocol( self.server_protocol, - request_method, + request_method=request_method, body_size_limit=self.config.body_size_limit, - include_body=False, + include_body=True ) - def send_to_client(self, message): - self.client_conn.send(self.client_protocol.assemble(message)) + def send_response(self, response): + self.client_conn.send(self.client_protocol.assemble(response)) - def send_to_server(self, message): - self.server_conn.send(self.server_protocol.assemble(message)) + def read_response_headers(self): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=None, # does not matter if we don't read the body. + body_size_limit=self.config.body_size_limit, + include_body=False + ) + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + return self.server_protocol.read_http_body_chunked( + headers, + self.config.body_size_limit, + request_method, + response_code, + False, + max_chunk_size + ) + + def send_response_headers(self, response): + h = self.client_protocol._assemble_response_first_line(response) + self.client_conn.wfile.write(h+"\r\n") + h = self.client_protocol._assemble_response_headers( + response, + preserve_transfer_encoding=True + ) + self.client_conn.send(h+"\r\n") + + def send_response_body(self, response, chunks): + if self.client_protocol.has_chunked_encoding(response.headers): + chunks = ( + "%d\r\n%s\r\n" % (len(chunk), chunk) + for chunk in chunks + ) + for chunk in chunks: + self.client_conn.send(chunk) def connect(self): self.ctx.connect() @@ -69,14 +138,14 @@ class Http1Layer(Layer): layer() -class Http2Layer(Layer): +class Http2Layer(_HttpLayer): def __init__(self, ctx, mode): super(Http2Layer, self).__init__(ctx) self.mode = mode self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) - def read_from_client(self): + def read_request(self): request = HTTPRequest.from_protocol( self.client_protocol, body_size_limit=self.config.body_size_limit @@ -84,23 +153,23 @@ class Http2Layer(Layer): self._stream_id = request.stream_id return request - def read_from_server(self, request_method): + def send_request(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.server_conn.send(self.server_protocol.assemble(message)) + + def read_response(self, request_method): return HTTPResponse.from_protocol( self.server_protocol, - request_method, + request_method=request_method, body_size_limit=self.config.body_size_limit, include_body=True, stream_id=self._stream_id ) - def send_to_client(self, message): + def send_response(self, message): # TODO: implement flow control and WINDOW_UPDATE frames self.client_conn.send(self.client_protocol.assemble(message)) - def send_to_server(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames - self.server_conn.send(self.server_protocol.assemble(message)) - def connect(self): self.ctx.connect() self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) @@ -122,7 +191,7 @@ class Http2Layer(Layer): layer() def handle_unexpected_frame(self, frm): - print(frm.human_readable()) + self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") def make_error_response(status_code, message, headers=None): @@ -204,13 +273,13 @@ class UpstreamConnectLayer(Layer): def connect(self): if not self.server_conn: self.ctx.connect() - self.send_to_server(self.connect_request) + self.send_request(self.connect_request) else: pass # swallow the message def reconnect(self): self.ctx.reconnect() - self.send_to_server(self.connect_request) + self.send_request(self.connect_request) def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: @@ -240,7 +309,7 @@ class HttpLayer(Layer): flow = HTTPFlow(self.client_conn, self.server_conn, live=self) try: - request = self.read_from_client() + request = self.read_request() except tcp.NetLibError: # don't throw an error for disconnects that happen # before/between requests. @@ -280,7 +349,7 @@ class HttpLayer(Layer): except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: try: - self.send_to_client(make_error_response( + self.send_response(make_error_response( getattr(e, "code", 502), repr(e) )) @@ -295,7 +364,7 @@ class HttpLayer(Layer): def handle_regular_mode_connect(self, request): self.set_server((request.host, request.port)) - self.send_to_client(make_connect_response(request.httpversion)) + self.send_response(make_connect_response(request.httpversion)) layer = self.ctx.next_layer(self) layer() @@ -334,44 +403,33 @@ class HttpLayer(Layer): return close_connection def send_response_to_client(self, flow): - if not flow.response.stream: + if not (self.supports_streaming and flow.response.stream): # no streaming: # we already received the full response from the server and can # send it to the client straight away. - self.send_to_client(flow.response) + self.send_response(flow.response) else: # streaming: - # First send the headers and then transfer the response - # incrementally: - h = self.client_protocol._assemble_response_first_line(flow.response) - self.send_to_client(h + "\r\n") - h = self.client_protocol._assemble_response_headers(flow.response, preserve_transfer_encoding=True) - self.send_to_client(h + "\r\n") - - chunks = self.client_protocol.read_http_body_chunked( - flow.response.headers, - self.config.body_size_limit, - flow.request.method, - flow.response.code, - False, - 4096 + # First send the headers and then transfer the response incrementally + self.send_response_headers(flow.response) + chunks = self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code, + max_chunk_size=4096 ) - if callable(flow.response.stream): chunks = flow.response.stream(chunks) - - for chunk in chunks: - for part in chunk: - # TODO: That's going to fail. - self.send_to_client(part) - self.client_conn.wfile.flush() - + self.send_response_body(flow.response, chunks) flow.response.timestamp_end = utils.timestamp() def get_response_from_server(self, flow): def get_response(): - self.send_to_server(flow.request) - flow.response = self.read_from_server(flow.request.method) + self.send_request(flow.request) + if self.supports_streaming: + flow.response = self.read_response_headers() + else: + flow.response = self.read_response() try: get_response() @@ -400,18 +458,15 @@ class HttpLayer(Layer): if flow is None or flow == KILL: raise Kill() - if isinstance(self.ctx, Http2Layer): - pass # streaming is not implemented for http2 yet. - elif flow.response.stream: - flow.response.content = CONTENT_MISSING - else: - flow.response.content = self.server_protocol.read_http_body( - flow.response.headers, - self.config.body_size_limit, - flow.request.method, - flow.response.code, - False - ) + if self.supports_streaming: + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = "".join(self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code + )) flow.response.timestamp_end = utils.timestamp() # no further manipulation of self.server_conn beyond this point @@ -480,14 +535,14 @@ class HttpLayer(Layer): if self.server_conn.tls_established: self.reconnect() - self.send_to_server(make_connect_request(address)) + self.send_request(make_connect_request(address)) tls_layer = TlsLayer(self, False, True) tls_layer._establish_tls_with_server() """ def validate_request(self, request): if request.form_in == "absolute" and request.scheme != "http": - self.send_to_client(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + self.send_response(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { @@ -501,7 +556,7 @@ class HttpLayer(Layer): err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( " or ".join(allowed_request_forms), request.form_in ) - self.send_to_client(make_error_response(400, err_message)) + self.send_response(make_error_response(400, err_message)) raise HttpException(err_message) if self.mode == "regular": @@ -512,7 +567,7 @@ class HttpLayer(Layer): if self.config.authenticator.authenticate(request.headers): self.config.authenticator.clean(request.headers) else: - self.send_to_client(make_error_response( + self.send_response(make_error_response( 407, "Proxy Authentication Required", odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()]) @@ -552,10 +607,7 @@ class RequestReplayThread(threading.Thread): if not self.flow.response: # In all modes, we directly connect to the server displayed if self.config.mode == "upstream": - # FIXME - server_address = self.config.mode.get_upstream_server( - self.flow.client_conn - )[2:] + server_address = self.config.upstream_server.address server = ServerConnection(server_address) server.connect() protocol = HTTP1Protocol(server) diff --git a/test/scripts/stream_modify.py b/test/scripts/stream_modify.py index e5c323be..e26d83f1 100644 --- a/test/scripts/stream_modify.py +++ b/test/scripts/stream_modify.py @@ -1,6 +1,6 @@ def modify(chunks): - for prefix, content, suffix in chunks: - yield prefix, content.replace("foo", "bar"), suffix + for chunk in chunks: + yield chunk.replace("foo", "bar") def responseheaders(context, flow): -- cgit v1.2.3 From a7058e2a3c59cc2b13aaea3d7c767a3ca4a4bc40 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 29 Aug 2015 20:53:25 +0200 Subject: fix bugs, fix tests --- libmproxy/console/statusbar.py | 11 ++++---- libmproxy/protocol2/http.py | 54 ++++++++++++++++++++++++----------- test/test_proxy.py | 9 +++--- test/test_server.py | 16 ++++++----- test/tservers.py | 64 ++++++++---------------------------------- 5 files changed, 69 insertions(+), 85 deletions(-) diff --git a/libmproxy/console/statusbar.py b/libmproxy/console/statusbar.py index 7eb2131b..ea2dbfa8 100644 --- a/libmproxy/console/statusbar.py +++ b/libmproxy/console/statusbar.py @@ -199,11 +199,12 @@ class StatusBar(urwid.WidgetWrap): r.append("[%s]" % (":".join(opts))) if self.master.server.config.mode in ["reverse", "upstream"]: - dst = self.master.server.config.mode.dst - scheme = "https" if dst[0] else "http" - if dst[1] != dst[0]: - scheme += "2https" if dst[1] else "http" - r.append("[dest:%s]" % utils.unparse_url(scheme, *dst[2:])) + dst = self.master.server.config.upstream_server + r.append("[dest:%s]" % netlib.utils.unparse_url( + dst.scheme, + dst.address.host, + dst.address.port + )) if self.master.scripts: r.append("[") r.append(("heading_key", "s")) diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 0fde9fb1..a3f32926 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -40,6 +40,7 @@ class _HttpLayer(Layer): def send_response(self, response): raise NotImplementedError() + class _StreamingHttpLayer(_HttpLayer): supports_streaming = True @@ -58,7 +59,6 @@ class _StreamingHttpLayer(_HttpLayer): class Http1Layer(_StreamingHttpLayer): - def __init__(self, ctx, mode): super(Http1Layer, self).__init__(ctx) self.mode = mode @@ -105,12 +105,12 @@ class Http1Layer(_StreamingHttpLayer): def send_response_headers(self, response): h = self.client_protocol._assemble_response_first_line(response) - self.client_conn.wfile.write(h+"\r\n") + self.client_conn.wfile.write(h + "\r\n") h = self.client_protocol._assemble_response_headers( response, preserve_transfer_encoding=True ) - self.client_conn.send(h+"\r\n") + self.client_conn.send(h + "\r\n") def send_response_body(self, response, chunks): if self.client_protocol.has_chunked_encoding(response.headers): @@ -142,8 +142,10 @@ class Http2Layer(_HttpLayer): def __init__(self, ctx, mode): super(Http2Layer, self).__init__(ctx) self.mode = mode - self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) def read_request(self): request = HTTPRequest.from_protocol( @@ -172,17 +174,20 @@ class Http2Layer(_HttpLayer): def connect(self): self.ctx.connect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol.perform_connection_preface() def reconnect(self): self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol.perform_connection_preface() def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) self.server_protocol.perform_connection_preface() def __call__(self): @@ -264,7 +269,10 @@ class UpstreamConnectLayer(Layer): def __init__(self, ctx, connect_request): super(UpstreamConnectLayer, self).__init__(ctx) self.connect_request = connect_request - self.server_conn = ConnectServerConnection((connect_request.host, connect_request.port), self.ctx) + self.server_conn = ConnectServerConnection( + (connect_request.host, connect_request.port), + self.ctx + ) def __call__(self): layer = self.ctx.next_layer(self) @@ -280,6 +288,9 @@ class UpstreamConnectLayer(Layer): def reconnect(self): self.ctx.reconnect() self.send_request(self.connect_request) + resp = self.read_response("CONNECT") + if resp.code != 200: + raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: @@ -290,7 +301,7 @@ class UpstreamConnectLayer(Layer): self.connect_request.port = address.port self.server_conn.address = address else: - self.ctx.set_server(address, server_tls, sni, depth-1) + self.ctx.set_server(address, server_tls, sni, depth - 1) class HttpLayer(Layer): @@ -413,10 +424,10 @@ class HttpLayer(Layer): # First send the headers and then transfer the response incrementally self.send_response_headers(flow.response) chunks = self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code, - max_chunk_size=4096 + flow.response.headers, + flow.request.method, + flow.response.code, + max_chunk_size=4096 ) if callable(flow.response.stream): chunks = flow.response.stream(chunks) @@ -521,7 +532,8 @@ class HttpLayer(Layer): # If there's not TlsLayer below which could catch the exception, # TLS will not be established. if tls and not self.server_conn.tls_established: - raise ProtocolException("Cannot upgrade to SSL, no TLS layer on the protocol stack.") + raise ProtocolException( + "Cannot upgrade to SSL, no TLS layer on the protocol stack.") else: if not self.server_conn: self.connect() @@ -542,7 +554,8 @@ class HttpLayer(Layer): def validate_request(self, request): if request.form_in == "absolute" and request.scheme != "http": - self.send_response(make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + self.send_response( + make_error_response(400, "Invalid request scheme: %s" % request.scheme)) raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { @@ -570,7 +583,11 @@ class HttpLayer(Layer): self.send_response(make_error_response( 407, "Proxy Authentication Required", - odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()]) + odict.ODictCaseless( + [ + [k, v] for k, v in + self.config.authenticator.auth_challenge_headers().items() + ]) )) raise InvalidCredentials("Proxy Authentication Required") @@ -614,6 +631,9 @@ class RequestReplayThread(threading.Thread): if r.scheme == "https": connect_request = make_connect_request((r.host, r.port)) server.send(protocol.assemble(connect_request)) + resp = protocol.read_response("CONNECT") + if resp.code != 200: + raise HttpError(502, "Upstream server refuses CONNECT request") server.establish_ssl( self.config.clientcerts, sni=self.flow.server_conn.sni diff --git a/test/test_proxy.py b/test/test_proxy.py index 9c01ab63..fac4a4f4 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -1,9 +1,8 @@ -import argparse from libmproxy import cmdline from libmproxy.proxy import ProxyConfig, process_proxy_options from libmproxy.proxy.connection import ServerConnection from libmproxy.proxy.primitives import ProxyError -from libmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler +from libmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler2 import tutils from libpathod import test from netlib import http, tcp @@ -175,8 +174,10 @@ class TestDummyServer: class TestConnectionHandler: def test_fatal_error(self): config = mock.Mock() - config.mode.get_upstream_server.side_effect = RuntimeError - c = ConnectionHandler( + root_layer = mock.Mock() + root_layer.side_effect = RuntimeError + config.mode.return_value = root_layer + c = ConnectionHandler2( config, mock.MagicMock(), ("127.0.0.1", diff --git a/test/test_server.py b/test/test_server.py index 1216a349..7b66c582 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -68,7 +68,7 @@ class CommonMixin: # SSL with the upstream proxy. rt = self.master.replay_request(l, block=True) assert not rt - if isinstance(self, tservers.HTTPUpstreamProxTest) and not self.ssl: + if isinstance(self, tservers.HTTPUpstreamProxTest): assert l.response.code == 502 else: assert l.error @@ -506,7 +506,7 @@ class TestTransparentSSL(tservers.TransparentProxTest, CommonMixin, TcpMixin): p = pathoc.Pathoc(("localhost", self.proxy.port), fp=None) p.connect() r = p.request("get:/") - assert r.status_code == 400 + assert r.status_code == 502 class TestProxy(tservers.HTTPProxTest): @@ -724,9 +724,9 @@ class TestStreamRequest(tservers.HTTPProxTest): assert resp.headers["Transfer-Encoding"][0] == 'chunked' assert resp.status_code == 200 - chunks = list( - content for _, content, _ in protocol.read_http_body_chunked( - resp.headers, None, "GET", 200, False)) + chunks = list(protocol.read_http_body_chunked( + resp.headers, None, "GET", 200, False + )) assert chunks == ["this", "isatest", ""] connection.close() @@ -959,6 +959,9 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxTest): p = self.pathoc() req = p.request("get:'/p/418:b\"content\"'") + assert req.content == "content" + assert req.status_code == 418 + assert self.proxy.tmaster.state.flow_count() == 2 # CONNECT and request # CONNECT, failing request, assert self.chain[0].tmaster.state.flow_count() == 4 @@ -967,8 +970,7 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxTest): assert self.chain[1].tmaster.state.flow_count() == 2 # (doesn't store (repeated) CONNECTs from chain[0] # as it is a regular proxy) - assert req.content == "content" - assert req.status_code == 418 + assert not self.chain[1].tmaster.state.flows[0].response # killed assert self.chain[1].tmaster.state.flows[1].response diff --git a/test/tservers.py b/test/tservers.py index 43ebf2bb..dfd3f627 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -181,22 +181,24 @@ class TResolver: def original_addr(self, sock): return ("127.0.0.1", self.port) - class TransparentProxTest(ProxTestBase): ssl = None resolver = TResolver @classmethod - @mock.patch("libmproxy.platform.resolver") - def setupAll(cls, _): + def setupAll(cls): super(TransparentProxTest, cls).setupAll() - if cls.ssl: - ports = [cls.server.port, cls.server2.port] - else: - ports = [] - cls.config.mode = TransparentProxyMode( - cls.resolver(cls.server.port), - ports) + + cls._resolver = mock.patch( + "libmproxy.platform.resolver", + new=lambda: cls.resolver(cls.server.port) + ) + cls._resolver.start() + + @classmethod + def teardownAll(cls): + cls._resolver.stop() + super(TransparentProxTest, cls).teardownAll() @classmethod def get_proxy_config(cls): @@ -270,48 +272,6 @@ class SocksModeTest(HTTPProxTest): d["mode"] = "socks5" return d -class SpoofModeTest(ProxTestBase): - ssl = None - - @classmethod - def get_proxy_config(cls): - d = ProxTestBase.get_proxy_config() - d["upstream_server"] = None - d["mode"] = "spoof" - return d - - def pathoc(self, sni=None): - """ - Returns a connected Pathoc instance. - """ - p = libpathod.pathoc.Pathoc( - ("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None - ) - p.connect() - return p - - -class SSLSpoofModeTest(ProxTestBase): - ssl = True - - @classmethod - def get_proxy_config(cls): - d = ProxTestBase.get_proxy_config() - d["upstream_server"] = None - d["mode"] = "sslspoof" - d["spoofed_ssl_port"] = 443 - return d - - def pathoc(self, sni=None): - """ - Returns a connected Pathoc instance. - """ - p = libpathod.pathoc.Pathoc( - ("localhost", self.proxy.port), ssl=self.ssl, sni=sni, fp=None - ) - p.connect() - return p - class ChainProxTest(ProxTestBase): """ -- cgit v1.2.3 From 100ea27c30d89b895a02a1b128edc5472ab84b3e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 29 Aug 2015 23:08:16 +0200 Subject: simplify raw tcp protocol --- libmproxy/protocol2/rawtcp.py | 62 +++++++++++++++++++++++++++++++++++++------ 1 file changed, 54 insertions(+), 8 deletions(-) diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index 6819ad6e..e8e3cf65 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -1,21 +1,67 @@ from __future__ import (absolute_import, print_function, division) +import socket +import select + +from OpenSSL import SSL -import OpenSSL from ..exceptions import ProtocolException +from netlib.tcp import NetLibError +from netlib.utils import cleanBin from ..protocol.tcp import TCPHandler from .layer import Layer class RawTcpLayer(Layer): + chunk_size = 4096 + + def __init__(self, ctx, logging=True): + self.logging = logging + super(RawTcpLayer, self).__init__(ctx) + def __call__(self): self.connect() - tcp_handler = TCPHandler(self) + + buf = memoryview(bytearray(self.chunk_size)) + + client = self.client_conn.connection + server = self.server_conn.connection + conns = [client, server] + try: - tcp_handler.handle_messages() - except OpenSSL.SSL.Error as e: - raise ProtocolException("SSL error: %s" % repr(e), e) + while True: + r, _, _ = select.select(conns, [], [], 10) + for conn in r: + + size = conn.recv_into(buf, self.chunk_size) + if not size: + conns.remove(conn) + # Shutdown connection to the other peer + if isinstance(conn, SSL.Connection): + # We can't half-close a connection, so we just close everything here. + # Sockets will be cleaned up on a higher level. + return + else: + conn.shutdown(socket.SHUT_WR) + + if len(conns) == 0: + return + continue + + dst = server if conn == client else client + dst.sendall(buf[:size]) + if self.logging: + # log messages are prepended with the client address, + # hence the "weird" direction string. + if dst == server: + direction = "-> tcp -> {!r}".format(self.server_conn.address) + else: + direction = "<- tcp <- {!r}".format(self.server_conn.address) + data = cleanBin(buf[:size].tobytes()) + self.log( + "{}\r\n{}".format(direction, data), + "info" + ) - def establish_server_connection(self): - pass - # FIXME: Remove method, currently just here to mock TCPHandler's call to it. + except (socket.error, NetLibError, SSL.Error) as e: + raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) \ No newline at end of file -- cgit v1.2.3 From dd7f50d64bef38fa67b4cace91913d03691dde26 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 01:21:58 +0200 Subject: restructure code, remove cruft --- libmproxy/exceptions.py | 4 + libmproxy/flow.py | 2 +- libmproxy/protocol/http.py | 83 +------ libmproxy/protocol2/http.py | 112 +-------- libmproxy/protocol2/http_proxy.py | 3 +- libmproxy/protocol2/http_replay.py | 95 ++++++++ libmproxy/protocol2/layer.py | 19 +- libmproxy/protocol2/messages.py | 46 ---- libmproxy/protocol2/rawtcp.py | 13 +- libmproxy/protocol2/reverse_proxy.py | 5 +- libmproxy/protocol2/root_context.py | 5 +- libmproxy/protocol2/socks_proxy.py | 54 ++++- libmproxy/protocol2/tls.py | 22 +- libmproxy/protocol2/transparent_proxy.py | 3 +- libmproxy/proxy/__init__.py | 13 +- libmproxy/proxy/connection.py | 6 +- libmproxy/proxy/primitives.py | 179 +------------- libmproxy/proxy/server.py | 401 +++---------------------------- libmproxy/utils.py | 4 - test/test_dump.py | 4 +- test/test_proxy.py | 22 +- test/tservers.py | 1 - 22 files changed, 250 insertions(+), 846 deletions(-) create mode 100644 libmproxy/protocol2/http_replay.py delete mode 100644 libmproxy/protocol2/messages.py diff --git a/libmproxy/exceptions.py b/libmproxy/exceptions.py index 3825c409..f34d9707 100644 --- a/libmproxy/exceptions.py +++ b/libmproxy/exceptions.py @@ -18,6 +18,10 @@ class ProtocolException(ProxyException): pass +class Socks5Exception(ProtocolException): + pass + + class HttpException(ProtocolException): pass diff --git a/libmproxy/flow.py b/libmproxy/flow.py index a2b807ba..dac607a0 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,7 +8,7 @@ import Cookie import cookielib import os import re -from libmproxy.protocol2.http import RequestReplayThread +from libmproxy.protocol2.http_replay import RequestReplayThread from netlib import odict, wsgi, tcp from netlib.http.semantics import CONTENT_MISSING diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 56d7d57f..a30437d1 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -695,85 +695,4 @@ class HTTPHandler(ProtocolHandler): else: raise http.HttpAuthenticationError( self.c.config.authenticator.auth_challenge_headers()) - return request.headers - - -class RequestReplayThread(threading.Thread): - name = "RequestReplayThread" - - def __init__(self, config, flow, masterq, should_exit): - """ - masterqueue can be a queue or None, if no scripthooks should be - processed. - """ - self.config, self.flow = config, flow - if masterq: - self.channel = controller.Channel(masterq, should_exit) - else: - self.channel = None - super(RequestReplayThread, self).__init__() - - def run(self): - r = self.flow.request - form_out_backup = r.form_out - try: - self.flow.response = None - - # If we have a channel, run script hooks. - if self.channel: - request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == KILL: - raise KillSignal() - elif isinstance(request_reply, HTTPResponse): - self.flow.response = request_reply - - if not self.flow.response: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - # FIXME - server_address = self.config.mode.get_upstream_server( - self.flow.client_conn - )[2:] - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - send_connect_request(server, r.host, r.port) - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - else: - r.form_out = "absolute" - else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - - server.send(self.flow.server_conn.protocol.assemble(r)) - self.flow.server_conn = server - self.flow.response = HTTPResponse.from_protocol( - self.flow.server_conn.protocol, - r.method, - body_size_limit=self.config.body_size_limit, - ) - if self.channel: - response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == KILL: - raise KillSignal() - except (proxy.ProxyError, http.HttpError, tcp.NetLibError) as v: - self.flow.error = Error(repr(v)) - if self.channel: - self.channel.ask("error", self.flow) - except KillSignal: - # KillSignal should only be raised if there's a channel in the - # first place. - self.channel.tell("log", proxy.Log("Connection killed", "info")) - finally: - r.form_out = form_out_backup + return request.headers \ No newline at end of file diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index a3f32926..a508ae8b 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -1,28 +1,20 @@ from __future__ import (absolute_import, print_function, division) -from .. import version -import threading -from ..exceptions import InvalidCredentials, HttpException, ProtocolException -from .layer import Layer -from libmproxy import utils -from libmproxy.controller import Channel -from libmproxy.protocol2.layer import Kill -from libmproxy.protocol import KILL, Error - -from libmproxy.protocol.http import HTTPFlow -from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest -from libmproxy.proxy import Log -from libmproxy.proxy.connection import ServerConnection from netlib import tcp -from netlib.http import status_codes, http1, http2, HttpErrorConnClosed, HttpError +from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError from netlib.http.semantics import CONTENT_MISSING from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol - -# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. +from .. import version, utils +from ..exceptions import InvalidCredentials, HttpException, ProtocolException +from .layer import Layer +from ..proxy import Kill +from libmproxy.protocol import KILL, Error +from libmproxy.protocol.http import HTTPFlow +from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest class _HttpLayer(Layer): @@ -138,6 +130,7 @@ class Http1Layer(_StreamingHttpLayer): layer() +# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. class Http2Layer(_HttpLayer): def __init__(self, ctx, mode): super(Http2Layer, self).__init__(ctx) @@ -359,6 +352,9 @@ class HttpLayer(Layer): return except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: + if flow.request and not flow.response: + flow.error = Error(repr(e)) + self.channel.ask("error", flow) try: self.send_response(make_error_response( getattr(e, "code", 502), @@ -590,87 +586,3 @@ class HttpLayer(Layer): ]) )) raise InvalidCredentials("Proxy Authentication Required") - - -class RequestReplayThread(threading.Thread): - name = "RequestReplayThread" - - def __init__(self, config, flow, masterq, should_exit): - """ - masterqueue can be a queue or None, if no scripthooks should be - processed. - """ - self.config, self.flow = config, flow - if masterq: - self.channel = Channel(masterq, should_exit) - else: - self.channel = None - super(RequestReplayThread, self).__init__() - - def run(self): - r = self.flow.request - form_out_backup = r.form_out - try: - self.flow.response = None - - # If we have a channel, run script hooks. - if self.channel: - request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == KILL: - raise Kill() - elif isinstance(request_reply, HTTPResponse): - self.flow.response = request_reply - - if not self.flow.response: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - server_address = self.config.upstream_server.address - server = ServerConnection(server_address) - server.connect() - protocol = HTTP1Protocol(server) - if r.scheme == "https": - connect_request = make_connect_request((r.host, r.port)) - server.send(protocol.assemble(connect_request)) - resp = protocol.read_response("CONNECT") - if resp.code != 200: - raise HttpError(502, "Upstream server refuses CONNECT request") - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - else: - r.form_out = "absolute" - else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - protocol = HTTP1Protocol(server) - if r.scheme == "https": - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - - server.send(protocol.assemble(r)) - self.flow.server_conn = server - self.flow.response = HTTPResponse.from_protocol( - protocol, - r.method, - body_size_limit=self.config.body_size_limit, - ) - if self.channel: - response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == KILL: - raise Kill() - except (HttpError, tcp.NetLibError) as v: - self.flow.error = Error(repr(v)) - if self.channel: - self.channel.ask("error", self.flow) - except Kill: - # KillSignal should only be raised if there's a channel in the - # first place. - self.channel.tell("log", Log("Connection killed", "info")) - finally: - r.form_out = form_out_backup diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index c24af6cf..b3389eb7 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -13,6 +13,7 @@ class HttpProxy(Layer, ServerConnectionMixin): if self.server_conn: self._disconnect() + class HttpUpstreamProxy(Layer, ServerConnectionMixin): def __init__(self, ctx, server_address): super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) @@ -23,4 +24,4 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): layer() finally: if self.server_conn: - self._disconnect() \ No newline at end of file + self._disconnect() diff --git a/libmproxy/protocol2/http_replay.py b/libmproxy/protocol2/http_replay.py new file mode 100644 index 00000000..872ef9cd --- /dev/null +++ b/libmproxy/protocol2/http_replay.py @@ -0,0 +1,95 @@ +import threading +from netlib.http import HttpError +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError + +from ..controller import Channel +from ..protocol import KILL, Error +from ..protocol.http_wrappers import HTTPResponse +from ..proxy import Log, Kill +from ..proxy.connection import ServerConnection +from .http import make_connect_request + + +class RequestReplayThread(threading.Thread): + name = "RequestReplayThread" + + def __init__(self, config, flow, masterq, should_exit): + """ + masterqueue can be a queue or None, if no scripthooks should be + processed. + """ + self.config, self.flow = config, flow + if masterq: + self.channel = Channel(masterq, should_exit) + else: + self.channel = None + super(RequestReplayThread, self).__init__() + + def run(self): + r = self.flow.request + form_out_backup = r.form_out + try: + self.flow.response = None + + # If we have a channel, run script hooks. + if self.channel: + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == KILL: + raise Kill() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply + + if not self.flow.response: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + server_address = self.config.upstream_server.address + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + connect_request = make_connect_request((r.host, r.port)) + server.send(protocol.assemble(connect_request)) + resp = protocol.read_response("CONNECT") + if resp.code != 200: + raise HttpError(502, "Upstream server refuses CONNECT request") + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + + server.send(protocol.assemble(r)) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_protocol( + protocol, + r.method, + body_size_limit=self.config.body_size_limit, + ) + if self.channel: + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == KILL: + raise Kill() + except (HttpError, NetLibError) as v: + self.flow.error = Error(repr(v)) + if self.channel: + self.channel.ask("error", self.flow) + except Kill: + # KillSignal should only be raised if there's a channel in the + # first place. + self.channel.tell("log", Log("Connection killed", "info")) + finally: + r.form_out = form_out_backup diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index f72320ff..2b47cc26 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -30,8 +30,6 @@ Further goals: inline scripts shall have a chance to handle everything locally. """ from __future__ import (absolute_import, print_function, division) -import Queue -import threading from netlib import tcp from ..proxy import Log from ..proxy.connection import ServerConnection @@ -80,10 +78,8 @@ class Layer(_LayerCodeCompletion): def log(self, msg, level, subs=()): full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] + "{}: {}".format(repr(self.client_conn.address), msg) + ] for i in subs: full_msg.append(" -> " + i) full_msg = "\n".join(full_msg) @@ -119,7 +115,7 @@ class ServerConnectionMixin(object): self.log("Set new server address: " + repr(address), "debug") self.server_conn.address = address else: - self.ctx.set_server(address, server_tls, sni, depth-1) + self.ctx.set_server(address, server_tls, sni, depth - 1) def _disconnect(self): """ @@ -138,10 +134,5 @@ class ServerConnectionMixin(object): try: self.server_conn.connect() except tcp.NetLibError as e: - raise ProtocolException("Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) - - -class Kill(Exception): - """ - Kill a connection. - """ \ No newline at end of file + raise ProtocolException( + "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py deleted file mode 100644 index de049486..00000000 --- a/libmproxy/protocol2/messages.py +++ /dev/null @@ -1,46 +0,0 @@ -""" -This module contains all valid messages layers can send to the underlying layers. -""" -from __future__ import (absolute_import, print_function, division) -from netlib.tcp import Address - - -class _Message(object): - def __eq__(self, other): - # Allow message == Connect checks. - if isinstance(self, other): - return True - return self is other - - def __ne__(self, other): - return not self.__eq__(other) - - -class Connect(_Message): - """ - Connect to the server - """ - - -class Reconnect(_Message): - """ - Re-establish the server connection - """ - - -class SetServer(_Message): - """ - Change the upstream server. - """ - - def __init__(self, address, server_tls, sni, depth=1): - self.address = Address.wrap(address) - self.server_tls = server_tls - self.sni = sni - - # upstream proxy scenario: you may want to change either the final target or the upstream proxy. - # We can express this neatly as the "nth-server-providing-layer" - # ServerConnection could get a `via` attribute. - self.depth = depth - - diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py index e8e3cf65..b10217f1 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol2/rawtcp.py @@ -4,10 +4,9 @@ import select from OpenSSL import SSL -from ..exceptions import ProtocolException from netlib.tcp import NetLibError from netlib.utils import cleanBin -from ..protocol.tcp import TCPHandler +from ..exceptions import ProtocolException from .layer import Layer @@ -31,6 +30,7 @@ class RawTcpLayer(Layer): while True: r, _, _ = select.select(conns, [], [], 10) for conn in r: + dst = server if conn == client else client size = conn.recv_into(buf, self.chunk_size) if not size: @@ -41,22 +41,21 @@ class RawTcpLayer(Layer): # Sockets will be cleaned up on a higher level. return else: - conn.shutdown(socket.SHUT_WR) + dst.shutdown(socket.SHUT_WR) if len(conns) == 0: return continue - dst = server if conn == client else client dst.sendall(buf[:size]) if self.logging: # log messages are prepended with the client address, # hence the "weird" direction string. if dst == server: - direction = "-> tcp -> {!r}".format(self.server_conn.address) + direction = "-> tcp -> {}".format(repr(self.server_conn.address)) else: - direction = "<- tcp <- {!r}".format(self.server_conn.address) + direction = "<- tcp <- {}".format(repr(self.server_conn.address)) data = cleanBin(buf[:size].tobytes()) self.log( "{}\r\n{}".format(direction, data), @@ -64,4 +63,4 @@ class RawTcpLayer(Layer): ) except (socket.error, NetLibError, SSL.Error) as e: - raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) \ No newline at end of file + raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index 76163c71..e959db86 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -5,17 +5,18 @@ from .tls import TlsLayer class ReverseProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx, server_address, client_tls, server_tls): super(ReverseProxy, self).__init__(ctx, server_address=server_address) self._client_tls = client_tls self._server_tls = server_tls def __call__(self): + # Always use a TLS layer here; if someone changes the scheme, there needs to be a + # TLS layer underneath. layer = TlsLayer(self, self._client_tls, self._server_tls) try: layer() finally: if self.server_conn: - self._disconnect() \ No newline at end of file + self._disconnect() diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index af0e7a37..4d69204f 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -32,7 +32,7 @@ class RootContext(object): # 1. Check for --ignore. if self.config.check_ignore(top_layer.server_conn.address): - return RawTcpLayer(top_layer) + return RawTcpLayer(top_layer, logging=False) # 2. Check for TLS # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 @@ -62,7 +62,8 @@ class RootContext(object): # d = top_layer.client_conn.rfile.peek(3) # is_ascii = ( # len(d) == 3 and - # all(x in string.ascii_letters for x in d) # better be safe here and don't expect uppercase... + # # better be safe here and don't expect uppercase... + # all(x in string.ascii_letters for x in d) # ) # # TODO: This could block if there are not enough bytes available? # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py index 91935d24..525520e8 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/protocol2/socks_proxy.py @@ -1,27 +1,59 @@ from __future__ import (absolute_import, print_function, division) -from ..exceptions import ProtocolException -from ..proxy import ProxyError, Socks5ProxyMode +from netlib import socks +from netlib.tcp import NetLibError +from ..exceptions import Socks5Exception from .layer import Layer, ServerConnectionMixin class Socks5Proxy(Layer, ServerConnectionMixin): def __call__(self): try: - s5mode = Socks5ProxyMode([]) - address = s5mode.get_upstream_server(self.client_conn)[2:] - except ProxyError as e: - # TODO: Unmonkeypatch - raise ProtocolException(str(e), e) + # Parse Client Greeting + client_greet = socks.ClientGreeting.from_file(self.client_conn.rfile, fail_early=True) + client_greet.assert_socks5() + if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: + raise socks.SocksError( + socks.METHOD.NO_ACCEPTABLE_METHODS, + "mitmproxy only supports SOCKS without authentication" + ) - self.server_conn.address = address + # Send Server Greeting + server_greet = socks.ServerGreeting( + socks.VERSION.SOCKS5, + socks.METHOD.NO_AUTHENTICATION_REQUIRED + ) + server_greet.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() - # TODO: Kill event + # Parse Connect Request + connect_request = socks.Message.from_file(self.client_conn.rfile) + connect_request.assert_socks5() + if connect_request.msg != socks.CMD.CONNECT: + raise socks.SocksError( + socks.REP.COMMAND_NOT_SUPPORTED, + "mitmproxy only supports SOCKS5 CONNECT." + ) - layer = self.ctx.next_layer(self) + # We always connect lazily, but we need to pretend to the client that we connected. + connect_reply = socks.Message( + socks.VERSION.SOCKS5, + socks.REP.SUCCEEDED, + connect_request.atyp, + # dummy value, we don't have an upstream connection yet. + connect_request.addr + ) + connect_reply.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + except (socks.SocksError, NetLibError) as e: + raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e) + self.server_conn.address = connect_request.addr + + layer = self.ctx.next_layer(self) try: layer() finally: if self.server_conn: - self._disconnect() \ No newline at end of file + self._disconnect() diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 850bf5dc..0c02b0ea 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -1,11 +1,11 @@ from __future__ import (absolute_import, print_function, division) import struct -from construct import ConstructError -from netlib import tcp -import netlib.http.http2 +from construct import ConstructError +from netlib.tcp import NetLibError, NetLibInvalidCertificateError +from netlib.http.http1 import HTTP1Protocol from ..contrib.tls._constructs import ClientHello from ..exceptions import ProtocolException from .layer import Layer @@ -161,7 +161,7 @@ class TlsLayer(Layer): """ # This gets triggered if we haven't established an upstream connection yet. - default_alpn = netlib.http.http1.HTTP1Protocol.ALPN_PROTO_HTTP1 + default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 if self.alpn_for_client_connection in options: @@ -203,7 +203,7 @@ class TlsLayer(Layer): chain_file=chain_file, alpn_select_callback=self.__alpn_select_callback, ) - except tcp.NetLibError as e: + except NetLibError as e: raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) def _establish_tls_with_server(self): @@ -236,7 +236,7 @@ class TlsLayer(Layer): (tls_cert_err['depth'], tls_cert_err['errno']), "error") self.log("Ignoring server verification error, continuing with connection", "error") - except tcp.NetLibInvalidCertificateError as e: + except NetLibInvalidCertificateError as e: tls_cert_err = self.server_conn.ssl_verification_error self.log( "TLS verification failed for upstream server at depth %s with error: %s" % @@ -244,7 +244,7 @@ class TlsLayer(Layer): "error") self.log("Aborting connection attempt", "error") raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) - except tcp.NetLibError as e: + except NetLibError as e: raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") @@ -253,8 +253,12 @@ class TlsLayer(Layer): host = self.server_conn.address.host sans = set() # Incorporate upstream certificate - if self.server_conn and self.server_conn.tls_established and ( - not self.config.no_upstream_cert): + use_upstream_cert = ( + self.server_conn and + self.server_conn.tls_established and + (not self.config.no_upstream_cert) + ) + if use_upstream_cert: upstream_cert = self.server_conn.cert sans.update(upstream_cert.altnames) if upstream_cert.cn: diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py index 9263dbde..e6ebf115 100644 --- a/libmproxy/protocol2/transparent_proxy.py +++ b/libmproxy/protocol2/transparent_proxy.py @@ -6,7 +6,6 @@ from .layer import Layer, ServerConnectionMixin class TransparentProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx): super(TransparentProxy, self).__init__(ctx) self.resolver = platform.resolver() @@ -22,4 +21,4 @@ class TransparentProxy(Layer, ServerConnectionMixin): layer() finally: if self.server_conn: - self._disconnect() \ No newline at end of file + self._disconnect() diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index 7d664707..709654cb 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1,2 +1,11 @@ -from .primitives import * -from .config import ProxyConfig, process_proxy_options \ No newline at end of file +from __future__ import (absolute_import, print_function, division) + +from .primitives import Log, Kill +from .config import ProxyConfig +from .connection import ClientConnection, ServerConnection + +__all__ = [ + "Log", "Kill", + "ProxyConfig", + "ClientConnection", "ServerConnection" +] \ No newline at end of file diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index c329ed64..94f318f6 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -12,7 +12,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): # Eventually, this object is restored from state. We don't have a # connection then. if client_connection: - tcp.BaseHandler.__init__(self, client_connection, address, server) + super(ClientConnection, self).__init__(client_connection, address, server) else: self.connection = None self.server = None @@ -80,11 +80,11 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): return f def convert_to_ssl(self, *args, **kwargs): - tcp.BaseHandler.convert_to_ssl(self, *args, **kwargs) + super(ClientConnection, self).convert_to_ssl(*args, **kwargs) self.timestamp_ssl_setup = utils.timestamp() def finish(self): - tcp.BaseHandler.finish(self) + super(ClientConnection, self).finish() self.timestamp_end = utils.timestamp() diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index a9f31181..2e440fe8 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -1,178 +1,15 @@ from __future__ import absolute_import +import collections from netlib import socks, tcp -class ProxyError(Exception): - def __init__(self, code, message, headers=None): - super(ProxyError, self).__init__(message) - self.code, self.headers = code, headers - - -class ProxyServerError(Exception): - pass - - -class ProxyMode(object): - http_form_in = None - http_form_out = None - - def get_upstream_server(self, client_conn): - """ - Returns the address of the server to connect to. - Returns None if the address needs to be determined on the protocol level (regular proxy mode) - """ - raise NotImplementedError() # pragma: nocover - - @property - def name(self): - return self.__class__.__name__.replace("ProxyMode", "").lower() - - def __str__(self): - return self.name - - def __eq__(self, other): - """ - Allow comparisions with "regular" etc. - """ - if isinstance(other, ProxyMode): - return self is other - else: - return self.name == other - - def __ne__(self, other): - return not self.__eq__(other) - - -class RegularProxyMode(ProxyMode): - http_form_in = "absolute" - http_form_out = "relative" - - def get_upstream_server(self, client_conn): - return None - - -class SpoofMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def get_upstream_server(self, client_conn): - return None - - @property - def name(self): - return "spoof" - - -class SSLSpoofMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, sslport): - self.sslport = sslport - - def get_upstream_server(self, client_conn): - return None - - @property - def name(self): - return "sslspoof" - - -class TransparentProxyMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, resolver, sslports): - self.resolver = resolver - self.sslports = sslports - - def get_upstream_server(self, client_conn): - try: - dst = self.resolver.original_addr(client_conn.connection) - except Exception as e: - raise ProxyError(502, "Transparent mode failure: %s" % str(e)) - - if dst[1] in self.sslports: - ssl = True - else: - ssl = False - return [ssl, ssl] + list(dst) - - -class Socks5ProxyMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, sslports): - self.sslports = sslports - - def get_upstream_server(self, client_conn): - try: - # Parse Client Greeting - client_greet = socks.ClientGreeting.from_file(client_conn.rfile, fail_early=True) - client_greet.assert_socks5() - if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: - raise socks.SocksError( - socks.METHOD.NO_ACCEPTABLE_METHODS, - "mitmproxy only supports SOCKS without authentication" - ) - - # Send Server Greeting - server_greet = socks.ServerGreeting( - socks.VERSION.SOCKS5, - socks.METHOD.NO_AUTHENTICATION_REQUIRED - ) - server_greet.to_file(client_conn.wfile) - client_conn.wfile.flush() - - # Parse Connect Request - connect_request = socks.Message.from_file(client_conn.rfile) - connect_request.assert_socks5() - if connect_request.msg != socks.CMD.CONNECT: - raise socks.SocksError( - socks.REP.COMMAND_NOT_SUPPORTED, - "mitmproxy only supports SOCKS5 CONNECT." - ) - - # We do not connect here yet, as the clientconnect event has not - # been handled yet. - - connect_reply = socks.Message( - socks.VERSION.SOCKS5, - socks.REP.SUCCEEDED, - connect_request.atyp, - # dummy value, we don't have an upstream connection yet. - connect_request.addr - ) - connect_reply.to_file(client_conn.wfile) - client_conn.wfile.flush() - - ssl = bool(connect_request.addr.port in self.sslports) - return ssl, ssl, connect_request.addr.host, connect_request.addr.port - - except (socks.SocksError, tcp.NetLibError) as e: - raise ProxyError(502, "SOCKS5 mode failure: %s" % repr(e)) - - -class _ConstDestinationProxyMode(ProxyMode): - def __init__(self, dst): - self.dst = dst - - def get_upstream_server(self, client_conn): - return self.dst - - -class ReverseProxyMode(_ConstDestinationProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - -class UpstreamProxyMode(_ConstDestinationProxyMode): - http_form_in = "absolute" - http_form_out = "absolute" - - -class Log: +class Log(object): def __init__(self, msg, level="info"): self.msg = msg self.level = level + + +class Kill(Exception): + """ + Kill a connection. + """ \ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 1fc4cbda..69784014 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -3,16 +3,14 @@ from __future__ import absolute_import, print_function import traceback import sys import socket -from libmproxy.protocol2.layer import Kill from netlib import tcp from netlib.http.http1 import HTTP1Protocol from netlib.tcp import NetLibError -from ..protocol.handle import protocol_handler from .. import protocol2 -from ..exceptions import ProtocolException -from .primitives import ProxyServerError, Log, ProxyError -from .connection import ClientConnection, ServerConnection +from ..exceptions import ProtocolException, ServerException +from .primitives import Log, Kill +from .connection import ClientConnection class DummyServer: @@ -38,9 +36,9 @@ class ProxyServer(tcp.TCPServer): """ self.config = config try: - tcp.TCPServer.__init__(self, (config.host, config.port)) - except socket.error as v: - raise ProxyServerError('Error starting proxy server: ' + repr(v)) + super(ProxyServer, self).__init__((config.host, config.port)) + except socket.error as e: + raise ServerException('Error starting proxy server: ' + repr(e), e) self.channel = None def start_slave(self, klass, channel): @@ -51,33 +49,28 @@ class ProxyServer(tcp.TCPServer): self.channel = channel def handle_client_connection(self, conn, client_address): - h = ConnectionHandler2( - self.config, + h = ConnectionHandler( conn, client_address, - self, - self.channel) + self.config, + self.channel + ) h.handle() - h.finish() -class ConnectionHandler2: - # FIXME: parameter ordering - # FIXME: remove server attribute - def __init__(self, config, client_conn, client_address, server, channel): +class ConnectionHandler(object): + def __init__(self, client_conn, client_address, config, channel): self.config = config """@type: libmproxy.proxy.config.ProxyConfig""" self.client_conn = ClientConnection( client_conn, client_address, - server) + None) """@type: libmproxy.proxy.connection.ClientConnection""" self.channel = channel """@type: libmproxy.controller.Channel""" - def handle(self): - self.log("clientconnect", "info") - + def _create_root_layer(self): root_context = protocol2.RootContext( self.client_conn, self.config, @@ -86,33 +79,38 @@ class ConnectionHandler2: mode = self.config.mode if mode == "upstream": - root_layer = protocol2.HttpUpstreamProxy( + return protocol2.HttpUpstreamProxy( root_context, self.config.upstream_server.address ) elif mode == "transparent": - root_layer = protocol2.TransparentProxy(root_context) + return protocol2.TransparentProxy(root_context) elif mode == "reverse": client_tls = self.config.upstream_server.scheme.startswith("https") server_tls = self.config.upstream_server.scheme.endswith("https") - root_layer = protocol2.ReverseProxy( + return protocol2.ReverseProxy( root_context, self.config.upstream_server.address, client_tls, server_tls ) elif mode == "socks5": - root_layer = protocol2.Socks5Proxy(root_context) + return protocol2.Socks5Proxy(root_context) elif mode == "regular": - root_layer = protocol2.HttpProxy(root_context) + return protocol2.HttpProxy(root_context) elif callable(mode): # pragma: nocover - root_layer = mode(root_context) + return mode(root_context) else: # pragma: nocover raise ValueError("Unknown proxy mode: %s" % mode) + def handle(self): + self.log("clientconnect", "info") + + root_layer = self._create_root_layer() + try: root_layer() - except Kill as e: + except Kill: self.log("Connection killed", "info") except ProtocolException as e: self.log(e, "info") @@ -131,349 +129,8 @@ class ConnectionHandler2: print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) self.log("clientdisconnect", "info") - - def finish(self): - self.client_conn.finish() - - def log(self, msg, level, subs=()): - # FIXME: Duplicate code - full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - -class ConnectionHandler: - def __init__( - self, - config, - client_connection, - client_address, - server, - channel): - self.config = config - """@type: libmproxy.proxy.config.ProxyConfig""" - self.client_conn = ClientConnection( - client_connection, - client_address, - server) - """@type: libmproxy.proxy.connection.ClientConnection""" - self.server_conn = None - """@type: libmproxy.proxy.connection.ServerConnection""" - self.channel = channel - """@type: libmproxy.controller.Channel""" - - self.conntype = "http" - - def handle(self): - try: - self.log("clientconnect", "info") - - # Can we already identify the target server and connect to it? - client_ssl, server_ssl = False, False - conn_kwargs = dict() - upstream_info = self.config.mode.get_upstream_server( - self.client_conn) - if upstream_info: - self.set_server_address(upstream_info[2:]) - client_ssl, server_ssl = upstream_info[:2] - if self.config.check_ignore(self.server_conn.address): - self.log( - "Ignore host: %s:%s" % - self.server_conn.address(), - "info") - self.conntype = "tcp" - conn_kwargs["log"] = False - client_ssl, server_ssl = False, False - else: - # No upstream info from the metadata: upstream info in the - # protocol (e.g. HTTP absolute-form) - pass - - self.channel.ask("clientconnect", self) - - # Check for existing connection: If an inline script already established a - # connection, do not apply client_ssl or server_ssl. - if self.server_conn and not self.server_conn.connection: - self.establish_server_connection() - if client_ssl or server_ssl: - self.establish_ssl(client=client_ssl, server=server_ssl) - - if self.config.check_tcp(self.server_conn.address): - self.log( - "Generic TCP mode for host: %s:%s" % - self.server_conn.address(), - "info") - self.conntype = "tcp" - - elif not self.server_conn and self.config.mode == "sslspoof": - port = self.config.mode.sslport - self.set_server_address(("-", port)) - self.establish_ssl(client=True) - host = self.client_conn.connection.get_servername() - if host: - self.set_server_address((host, port)) - self.establish_server_connection() - self.establish_ssl(server=True, sni=host) - - # Delegate handling to the protocol handler - protocol_handler( - self.conntype)( - self, - **conn_kwargs).handle_messages() - - self.log("clientdisconnect", "info") - self.channel.tell("clientdisconnect", self) - - except ProxyError as e: - protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e) - except Exception: - import traceback - import sys - - self.log(traceback.format_exc(), "error") - print(traceback.format_exc(), file=sys.stderr) - print("mitmproxy has crashed!", file=sys.stderr) - print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) - finally: - # Make sure that we close the server connection in any case. - # The client connection is closed by the ProxyServer and does not - # have be handled here. - self.del_server_connection() - - def del_server_connection(self): - """ - Deletes (and closes) an existing server connection. - """ - if self.server_conn and self.server_conn.connection: - self.server_conn.finish() - self.server_conn.close() - self.log( - "serverdisconnect", "debug", [ - "%s:%s" % - (self.server_conn.address.host, self.server_conn.address.port)]) - self.channel.tell("serverdisconnect", self) - self.server_conn = None - - def set_server_address(self, addr): - """ - Sets a new server address with the given priority. - Does not re-establish either connection or SSL handshake. - """ - address = tcp.Address.wrap(addr) - - # Don't reconnect to the same destination. - if self.server_conn and self.server_conn.address == address: - return - - if self.server_conn: - self.del_server_connection() - - self.log( - "Set new server address: %s:%s" % - (address.host, address.port), "debug") - self.server_conn = ServerConnection(address) - - def establish_server_connection(self, ask=True): - """ - Establishes a new server connection. - If there is already an existing server connection, the function returns immediately. - - By default, this function ".ask"s the proxy master. This is deadly if this function is already called from the - master (e.g. via change_server), because this navigates us in a simple deadlock (the master is single-threaded). - In these scenarios, ask=False can be passed to suppress the call to the master. - """ - if self.server_conn.connection: - return - self.log( - "serverconnect", "debug", [ - "%s:%s" % - self.server_conn.address()[ - :2]]) - if ask: - self.channel.ask("serverconnect", self) - try: - self.server_conn.connect() - except tcp.NetLibError as v: - raise ProxyError(502, v) - - def establish_ssl(self, client=False, server=False, sni=None): - """ - Establishes SSL on the existing connection(s) to the server or the client, - as specified by the parameters. - """ - - # Logging - if client or server: - subs = [] - if client: - subs.append("with client") - if server: - subs.append("with server (sni: %s)" % sni) - self.log("Establish SSL", "debug", subs) - - if server: - if not self.server_conn or not self.server_conn.connection: - raise ProxyError(502, "No server connection.") - if self.server_conn.ssl_established: - raise ProxyError(502, "SSL to Server already established.") - try: - self.server_conn.establish_ssl( - self.config.clientcerts, - sni, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify_options=self.config.openssl_verification_mode_server, - ca_path=self.config.openssl_trusted_cadir_server, - ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, - ) - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") - except tcp.NetLibError as v: - e = ProxyError(502, repr(v)) - # Workaround for https://github.com/mitmproxy/mitmproxy/issues/427 - # The upstream server may reject connections without SNI, which means we need to - # establish SSL with the client first, hope for a SNI (which triggers a reconnect which replaces the - # ServerConnection object) and see whether that worked. - if client and "handshake failure" in e.message: - self.server_conn.may_require_sni = e - else: - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") - raise e - if client: - if self.client_conn.ssl_established: - raise ProxyError(502, "SSL to Client already established.") - cert, key, chain_file = self.find_cert() - try: - self.client_conn.convert_to_ssl( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - handle_sni=self.handle_sni, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - except tcp.NetLibError as v: - raise ProxyError(400, repr(v)) - - # Workaround for #427 part 2 - if server and hasattr(self.server_conn, "may_require_sni"): - raise self.server_conn.may_require_sni - - def server_reconnect(self, new_sni=False): - address = self.server_conn.address - had_ssl = self.server_conn.ssl_established - state = self.server_conn.state - sni = new_sni or self.server_conn.sni - self.log("(server reconnect follows)", "debug") - self.del_server_connection() - self.set_server_address(address) - self.establish_server_connection() - - for s in state: - protocol_handler(s[0])(self).handle_server_reconnect(s[1]) - self.server_conn.state = state - - # Receiving new_sni where had_ssl is False is a weird case that happens when the workaround for - # https://github.com/mitmproxy/mitmproxy/issues/427 is active. In this - # case, we want to establish SSL as well. - if had_ssl or new_sni: - self.establish_ssl(server=True, sni=sni) - - def finish(self): self.client_conn.finish() - def log(self, msg, level, subs=()): - full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - def find_cert(self): - host = self.server_conn.address.host - sans = [] - if self.server_conn.ssl_established and ( - not self.config.no_upstream_cert): - upstream_cert = self.server_conn.cert - sans.extend(upstream_cert.altnames) - if upstream_cert.cn: - sans.append(host) - host = upstream_cert.cn.decode("utf8").encode("idna") - if self.server_conn.sni: - sans.append(self.server_conn.sni) - # for ssl spoof mode - if hasattr(self.client_conn, "sni"): - sans.append(self.client_conn.sni) - - ret = self.config.certstore.get_cert(host, sans) - if not ret: - raise ProxyError(502, "Unable to generate dummy cert.") - return ret - - def handle_sni(self, connection): - """ - This callback gets called during the SSL handshake with the client. - The client has just sent the Sever Name Indication (SNI). We now connect upstream to - figure out which certificate needs to be served. - """ - try: - sn = connection.get_servername() - if not sn: - return - sni = sn.decode("utf8").encode("idna") - # for ssl spoof mode - self.client_conn.sni = sni - - if sni != self.server_conn.sni: - self.log("SNI received: %s" % sni, "debug") - # We should only re-establish upstream SSL if one of the following conditions is true: - # - We established SSL with the server previously - # - We initially wanted to establish SSL with the server, - # but the server refused to negotiate without SNI. - if self.server_conn.ssl_established or hasattr( - self.server_conn, - "may_require_sni"): - # reconnect to upstream server with SNI - self.server_reconnect(sni) - # Now, change client context to reflect changed certificate: - cert, key, chain_file = self.find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - connection.set_context(new_context) - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except: # pragma: no cover - import traceback - self.log( - "Error in handle_sni:\r\n" + - traceback.format_exc(), - "error") + def log(self, msg, level): + msg = "{}: {}".format(repr(self.client_conn.address), msg) + self.channel.tell("log", Log(msg, level)) \ No newline at end of file diff --git a/libmproxy/utils.py b/libmproxy/utils.py index 3ac3cc01..a6ca55f7 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -1,14 +1,10 @@ from __future__ import absolute_import import os import datetime -import urllib import re import time -import functools -import cgi import json -import netlib.utils def timestamp(): """ diff --git a/test/test_dump.py b/test/test_dump.py index b3d724a5..b05f6a0f 100644 --- a/test/test_dump.py +++ b/test/test_dump.py @@ -5,8 +5,8 @@ import netlib.tutils from netlib.http.semantics import CONTENT_MISSING from libmproxy import dump, flow -from libmproxy.protocol import http, http_wrappers -from libmproxy.proxy.primitives import Log +from libmproxy.protocol import http_wrappers +from libmproxy.proxy import Log import tutils import mock diff --git a/test/test_proxy.py b/test/test_proxy.py index fac4a4f4..301ce2ca 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -1,8 +1,8 @@ from libmproxy import cmdline -from libmproxy.proxy import ProxyConfig, process_proxy_options +from libmproxy.proxy import ProxyConfig +from libmproxy.proxy.config import process_proxy_options from libmproxy.proxy.connection import ServerConnection -from libmproxy.proxy.primitives import ProxyError -from libmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler2 +from libmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler import tutils from libpathod import test from netlib import http, tcp @@ -11,11 +11,6 @@ import mock from OpenSSL import SSL -def test_proxy_error(): - p = ProxyError(111, "msg") - assert str(p) - - class TestServerConnection: def setUp(self): self.d = test.Daemon() @@ -177,12 +172,11 @@ class TestConnectionHandler: root_layer = mock.Mock() root_layer.side_effect = RuntimeError config.mode.return_value = root_layer - c = ConnectionHandler2( - config, + c = ConnectionHandler( mock.MagicMock(), - ("127.0.0.1", - 8080), - None, - mock.MagicMock()) + ("127.0.0.1", 8080), + config, + mock.MagicMock() + ) with tutils.capture_stderr(c.handle) as output: assert "mitmproxy has crashed" in output diff --git a/test/tservers.py b/test/tservers.py index dfd3f627..c5256e53 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -7,7 +7,6 @@ import mock from libmproxy.proxy.config import ProxyConfig from libmproxy.proxy.server import ProxyServer -from libmproxy.proxy.primitives import TransparentProxyMode import libpathod.test import libpathod.pathoc from libmproxy import flow, controller -- cgit v1.2.3 From 1dd09a5509219e7390abbb8c0b6818c7e792daa1 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 02:27:38 +0200 Subject: always insert tls layer for inline script upgrades --- libmproxy/cmdline.py | 13 +++---------- libmproxy/protocol2/http_proxy.py | 5 ++--- libmproxy/protocol2/reverse_proxy.py | 10 +++------- libmproxy/protocol2/root_context.py | 33 +++++++++++++++++++++++++-------- libmproxy/protocol2/tls.py | 13 +++++++++++++ libmproxy/proxy/config.py | 2 ++ libmproxy/proxy/server.py | 4 +--- test/test_cmdline.py | 4 ---- test/test_server.py | 4 ---- 9 files changed, 49 insertions(+), 39 deletions(-) diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 591e87ed..55377af2 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -102,9 +102,9 @@ def parse_setheader(s): return _parse_hook(s) -def parse_server_spec(url, allowed_schemes=("http", "https")): +def parse_server_spec(url): p = netlib.utils.parse_url(url) - if not p or not p[1] or p[0] not in allowed_schemes: + if not p or not p[1] or p[0] not in ("http", "https"): raise configargparse.ArgumentTypeError( "Invalid server specification: %s" % url ) @@ -113,13 +113,6 @@ def parse_server_spec(url, allowed_schemes=("http", "https")): return config.ServerSpec(scheme, address) -def parse_server_spec_special(url): - """ - Provides additional support for http2https and https2http schemes. - """ - return parse_server_spec(url, allowed_schemes=("http", "https", "http2https", "https2http")) - - def get_common_options(options): stickycookie, stickyauth = None, None if options.stickycookie_filt: @@ -297,7 +290,7 @@ def proxy_modes(parser): group.add_argument( "-R", "--reverse", action="store", - type=parse_server_spec_special, + type=parse_server_spec, dest="reverse_proxy", default=None, help=""" diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index b3389eb7..2876c022 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -1,12 +1,11 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin -from .http import Http1Layer class HttpProxy(Layer, ServerConnectionMixin): def __call__(self): - layer = Http1Layer(self, "regular") + layer = self.ctx.next_layer(self) try: layer() finally: @@ -19,7 +18,7 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) def __call__(self): - layer = Http1Layer(self, "upstream") + layer = self.ctx.next_layer(self) try: layer() finally: diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index e959db86..c4cabccc 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -5,16 +5,12 @@ from .tls import TlsLayer class ReverseProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx, server_address, client_tls, server_tls): + def __init__(self, ctx, server_address, server_tls): super(ReverseProxy, self).__init__(ctx, server_address=server_address) - self._client_tls = client_tls - self._server_tls = server_tls + self.server_tls = server_tls def __call__(self): - # Always use a TLS layer here; if someone changes the scheme, there needs to be a - # TLS layer underneath. - layer = TlsLayer(self, self._client_tls, self._server_tls) - + layer = self.ctx.next_layer(self) try: layer() finally: diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 4d69204f..210ba6ab 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -6,7 +6,9 @@ from netlib.http.http2 import HTTP2Protocol from .rawtcp import RawTcpLayer from .tls import TlsLayer, is_tls_record_magic from .http import Http1Layer, Http2Layer - +from .layer import ServerConnectionMixin +from .http_proxy import HttpProxy, HttpUpstreamProxy +from .reverse_proxy import ReverseProxy class RootContext(object): """ @@ -34,18 +36,33 @@ class RootContext(object): if self.config.check_ignore(top_layer.server_conn.address): return RawTcpLayer(top_layer, logging=False) - # 2. Check for TLS - # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 - # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello d = top_layer.client_conn.rfile.peek(3) - if is_tls_record_magic(d): + client_tls = is_tls_record_magic(d) + + # 2. Always insert a TLS layer, even if there's neither client nor server tls. + # An inline script may upgrade from http to https, + # in which case we need some form of TLS layer. + if isinstance(top_layer, ReverseProxy): + return TlsLayer(top_layer, client_tls, top_layer.server_tls) + if isinstance(top_layer, ServerConnectionMixin): + return TlsLayer(top_layer, client_tls, client_tls) + + # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. + if isinstance(top_layer, TlsLayer): + if isinstance(top_layer.ctx, HttpProxy): + return Http1Layer(top_layer, "regular") + if isinstance(top_layer.ctx, HttpUpstreamProxy): + return Http1Layer(top_layer, "upstream") + + # 4. Check for other TLS cases (e.g. after CONNECT). + if client_tls: return TlsLayer(top_layer, True, True) - # 3. Check for --tcp + # 4. Check for --tcp if self.config.check_tcp(top_layer.server_conn.address): return RawTcpLayer(top_layer) - # 4. Check for TLS ALPN (HTTP1/HTTP2) + # 5. Check for TLS ALPN (HTTP1/HTTP2) if isinstance(top_layer, TlsLayer): alpn = top_layer.client_conn.get_alpn_proto_negotiated() if alpn == HTTP2Protocol.ALPN_PROTO_H2: @@ -53,7 +70,7 @@ class RootContext(object): if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: return Http1Layer(top_layer, 'transparent') - # 5. Assume HTTP1 by default + # 6. Assume HTTP1 by default return Http1Layer(top_layer, 'transparent') # In a future version, we want to implement TCP passthrough as the last fallback, diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 0c02b0ea..041adaaa 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -18,6 +18,9 @@ def is_tls_record_magic(d): False, otherwise. """ d = d[:3] + + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello return ( len(d) == 3 and d[0] == '\x16' and @@ -73,6 +76,16 @@ class TlsLayer(Layer): layer = self.ctx.next_layer(self) layer() + def __repr__(self): + if self._client_tls and self._server_tls: + return "TlsLayer(client and server)" + elif self._client_tls: + return "TlsLayer(client)" + elif self._server_tls: + return "TlsLayer(server)" + else: + return "TlsLayer(inactive)" + def _get_client_hello(self): """ Peek into the socket and read all records that contain the initial client hello message. diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 415ee215..b360abbd 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -24,6 +24,8 @@ class HostMatcher(object): self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] def __call__(self, address): + if not address: + return False address = tcp.Address.wrap(address) host = "%s:%s" % (address.host, address.port) if any(rex.search(host) for rex in self.regexes): diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 69784014..5abd0877 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -86,12 +86,10 @@ class ConnectionHandler(object): elif mode == "transparent": return protocol2.TransparentProxy(root_context) elif mode == "reverse": - client_tls = self.config.upstream_server.scheme.startswith("https") - server_tls = self.config.upstream_server.scheme.endswith("https") + server_tls = self.config.upstream_server.scheme == "https" return protocol2.ReverseProxy( root_context, self.config.upstream_server.address, - client_tls, server_tls ) elif mode == "socks5": diff --git a/test/test_cmdline.py b/test/test_cmdline.py index ee2f7044..1443ee1c 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -43,10 +43,6 @@ def test_parse_server_spec(): "http://foo.com") == ("http", ("foo.com", 80)) assert cmdline.parse_server_spec( "https://foo.com") == ("https", ("foo.com", 443)) - assert cmdline.parse_server_spec_special( - "https2http://foo.com") == ("https2http", ("foo.com", 80)) - assert cmdline.parse_server_spec_special( - "http2https://foo.com") == ("http2https", ("foo.com", 443)) tutils.raises( "Invalid server specification", cmdline.parse_server_spec, diff --git a/test/test_server.py b/test/test_server.py index 7b66c582..b691804b 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -490,10 +490,6 @@ class TestHttps2Http(tservers.ReverseProxTest): assert p.request("get:'/p/200'").status_code == 200 assert all("Error in handle_sni" not in msg for msg in self.proxy.log) - def test_http(self): - p = self.pathoc(ssl=False) - assert p.request("get:'/p/200'").status_code == 502 - class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin): ssl = False -- cgit v1.2.3 From 21e7f420d2870d89ebc05181c1fca674d80e4e7c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 03:23:57 +0200 Subject: minor fixes --- libmproxy/main.py | 6 +++--- libmproxy/protocol2/tls.py | 3 ++- 2 files changed, 5 insertions(+), 4 deletions(-) diff --git a/libmproxy/main.py b/libmproxy/main.py index 4dd6fdb1..faef8c82 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -2,11 +2,11 @@ from __future__ import print_function, absolute_import import os import signal import sys -import netlib.version from netlib.version_check import check_pyopenssl_version, check_mitmproxy_version from . import version, cmdline -from .proxy import process_proxy_options, ProxyServerError +from .exceptions import ServerException from .proxy.server import DummyServer, ProxyServer +from .proxy.config import process_proxy_options def assert_utf8_env(): @@ -31,7 +31,7 @@ def get_server(dummy_server, options): else: try: return ProxyServer(options) - except ProxyServerError as v: + except ServerException as v: print(str(v), file=sys.stderr) sys.exit(1) diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py index 041adaaa..73bb12f3 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol2/tls.py @@ -66,7 +66,8 @@ class TlsLayer(Layer): self._client_tls and self._server_tls and not self.config.no_upstream_cert ) - self._parse_client_hello() + if self._client_tls: + self._parse_client_hello() if client_tls_requires_server_cert: self._establish_tls_with_client_and_server() -- cgit v1.2.3 From 3873e08339fd701738a1522af32e37363fcec14b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 03:42:11 +0200 Subject: remove old code --- examples/ignore_websocket.py | 37 --- libmproxy/flow.py | 7 +- libmproxy/protocol/handle.py | 20 -- libmproxy/protocol/http.py | 608 +----------------------------------- libmproxy/protocol/http_wrappers.py | 40 +-- libmproxy/protocol/primitives.py | 130 +------- libmproxy/protocol/tcp.py | 97 ------ 7 files changed, 22 insertions(+), 917 deletions(-) delete mode 100644 examples/ignore_websocket.py delete mode 100644 libmproxy/protocol/handle.py delete mode 100644 libmproxy/protocol/tcp.py diff --git a/examples/ignore_websocket.py b/examples/ignore_websocket.py deleted file mode 100644 index 57e11d5b..00000000 --- a/examples/ignore_websocket.py +++ /dev/null @@ -1,37 +0,0 @@ -# This script makes mitmproxy switch to passthrough mode for all HTTP -# responses with "Connection: Upgrade" header. This is useful to make -# WebSockets work in untrusted environments. -# -# Note: Chrome (and possibly other browsers), when explicitly configured -# to use a proxy (i.e. mitmproxy's regular mode), send a CONNECT request -# to the proxy before they initiate the websocket connection. -# To make WebSockets work in these cases, supply -# `--ignore :80$` as an additional parameter. -# (see http://mitmproxy.org/doc/features/passthrough.html) - -import netlib.http.semantics - -from libmproxy.protocol.tcp import TCPHandler -from libmproxy.protocol import KILL -from libmproxy.script import concurrent - - -def start(context, argv): - netlib.http.semantics.Request._headers_to_strip_off.remove("Connection") - netlib.http.semantics.Request._headers_to_strip_off.remove("Upgrade") - - -def done(context): - netlib.http.semantics.Request._headers_to_strip_off.append("Connection") - netlib.http.semantics.Request._headers_to_strip_off.append("Upgrade") - - -@concurrent -def response(context, flow): - value = flow.response.headers.get_first("Connection", None) - if value and value.upper() == "UPGRADE": - # We need to send the response manually now... - flow.client_conn.send(flow.client_conn.protocol.assemble(flow.response)) - # ...and then delegate to tcp passthrough. - TCPHandler(flow.live.c, log=False).handle_messages() - flow.reply(KILL) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index dac607a0..a2f57512 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,15 +8,16 @@ import Cookie import cookielib import os import re +from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol2.http_replay import RequestReplayThread -from netlib import odict, wsgi, tcp +from netlib import odict, wsgi from netlib.http.semantics import CONTENT_MISSING import netlib.http from . import controller, protocol, tnetstring, filt, script, version from .onboarding import app -from .protocol import http, handle +from .protocol import http from .proxy.config import HostMatcher from .proxy.connection import ClientConnection, ServerConnection import urlparse @@ -1090,7 +1091,7 @@ class FlowReader: "Incompatible serialized data version: %s" % v ) off = self.fo.tell() - yield handle.protocols[data["type"]]["flow"].from_state(data) + yield HTTPFlow.from_state(data) except ValueError as v: # Error is due to EOF if self.fo.tell() == off and self.fo.read() == '': diff --git a/libmproxy/protocol/handle.py b/libmproxy/protocol/handle.py deleted file mode 100644 index 49cb3c1b..00000000 --- a/libmproxy/protocol/handle.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import absolute_import -from . import http, tcp - -protocols = { - 'http': dict(handler=http.HTTPHandler, flow=http.HTTPFlow), - 'tcp': dict(handler=tcp.TCPHandler) -} - - -def protocol_handler(protocol): - """ - @type protocol: str - @returns: libmproxy.protocol.primitives.ProtocolHandler - """ - if protocol in protocols: - return protocols[protocol]["handler"] - - raise NotImplementedError( - "Unknown Protocol: %s" % - protocol) # pragma: nocover diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index a30437d1..bde7b088 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,62 +1,9 @@ from __future__ import absolute_import -import Cookie -import copy -import threading -import time -import urllib -import urlparse -from email.utils import parsedate_tz, formatdate, mktime_tz -import netlib -from netlib import http, tcp, odict, utils, encoding -from netlib.http import cookies, http1, http2 -from netlib.http.http1 import HTTP1Protocol -from netlib.http.semantics import CONTENT_MISSING - -from .tcp import TCPHandler -from .primitives import KILL, ProtocolHandler, Flow, Error -from ..proxy.connection import ServerConnection -from .. import utils, controller, stateobject, proxy +from .primitives import Flow from .http_wrappers import decoded, HTTPRequest, HTTPResponse - -class KillSignal(Exception): - pass - - -def send_connect_request(conn, host, port, update_state=True): - upstream_request = HTTPRequest( - "authority", - "CONNECT", - None, - host, - port, - None, - (1, 1), - odict.ODictCaseless(), - "" - ) - - # we currently only support HTTP/1 CONNECT requests - protocol = http1.HTTP1Protocol(conn) - - conn.send(protocol.assemble(upstream_request)) - resp = HTTPResponse.from_protocol(protocol, upstream_request.method) - if resp.status_code != 200: - raise proxy.ProxyError(resp.status_code, - "Cannot establish SSL " + - "connection with upstream proxy: \r\n" + - repr(resp)) - if update_state: - conn.state.append(("http", { - "state": "connect", - "host": host, - "port": port} - )) - return resp - - class HTTPFlow(Flow): """ A HTTPFlow is a collection of objects representing a single HTTP @@ -143,556 +90,3 @@ class HTTPFlow(Flow): if self.response: c += self.response.replace(pattern, repl, *args, **kwargs) return c - - -class HTTPHandler(ProtocolHandler): - """ - HTTPHandler implements mitmproxys understanding of the HTTP protocol. - - """ - - def __init__(self, c): - super(HTTPHandler, self).__init__(c) - self.expected_form_in = c.config.mode.http_form_in - self.expected_form_out = c.config.mode.http_form_out - self.skip_authentication = False - - def handle_messages(self): - while self.handle_flow(): - pass - - def get_response_from_server(self, flow): - self.c.establish_server_connection() - - for attempt in (0, 1): - try: - if not self.c.server_conn.protocol: - # instantiate new protocol if connection does not have one yet - # TODO: select correct protocol based on ALPN (?) - self.c.server_conn.protocol = http1.HTTP1Protocol(self.c.server_conn) - # self.c.server_conn.protocol = http2.HTTP2Protocol(self.c.server_conn) - # self.c.server_conn.protocol.perform_connection_preface() - - self.c.server_conn.send(self.c.server_conn.protocol.assemble(flow.request)) - - # Only get the headers at first... - flow.response = HTTPResponse.from_protocol( - self.c.server_conn.protocol, - flow.request.method, - body_size_limit=self.c.config.body_size_limit, - include_body=False, - ) - break - except (tcp.NetLibError, http.HttpErrorConnClosed) as v: - self.c.log( - "error in server communication: %s" % repr(v), - level="debug" - ) - if attempt == 0: - # In any case, we try to reconnect at least once. This is - # necessary because it might be possible that we already - # initiated an upstream connection after clientconnect that - # has already been expired, e.g consider the following event - # log: - # > clientconnect (transparent mode destination known) - # > serverconnect - # > read n% of large request - # > server detects timeout, disconnects - # > read (100-n)% of large request - # > send large request upstream - self.c.server_reconnect() - else: - raise - - # call the appropriate script hook - this is an opportunity for an - # inline script to set flow.stream = True - flow = self.c.channel.ask("responseheaders", flow) - if flow is None or flow == KILL: - raise KillSignal() - else: - # now get the rest of the request body, if body still needs to be - # read but not streaming this response - if flow.response.stream: - flow.response.content = CONTENT_MISSING - else: - if isinstance(self.c.server_conn.protocol, http1.HTTP1Protocol): - # streaming is only supported with HTTP/1 at the moment - flow.response.content = self.c.server_conn.protocol.read_http_body( - flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, - flow.response.code, - False - ) - flow.response.timestamp_end = utils.timestamp() - - def handle_flow(self): - flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live) - - try: - try: - if not flow.client_conn.protocol: - # instantiate new protocol if connection does not have one yet - # the first request might be a CONNECT - which is currently only supported with HTTP/1 - flow.client_conn.protocol = http1.HTTP1Protocol(self.c.client_conn) - - req = HTTPRequest.from_protocol( - flow.client_conn.protocol, - body_size_limit=self.c.config.body_size_limit - ) - except tcp.NetLibError: - # don't throw an error for disconnects that happen - # before/between requests. - return False - - self.c.log( - "request", - "debug", - [repr(req)] - ) - ret = self.process_request(flow, req) - if ret: - # instantiate new protocol if connection does not have one yet - # TODO: select correct protocol based on ALPN (?) - flow.client_conn.protocol = http1.HTTP1Protocol(self.c.client_conn) - # flow.client_conn.protocol = http2.HTTP2Protocol(self.c.client_conn, is_server=True) - if ret is not None: - return ret - - # Be careful NOT to assign the request to the flow before - # process_request completes. This is because the call can raise an - # exception. If the request object is already attached, this results - # in an Error object that has an attached request that has not been - # sent through to the Master. - flow.request = req - request_reply = self.c.channel.ask("request", flow) - if request_reply is None or request_reply == KILL: - raise KillSignal() - - # The inline script may have changed request.host - self.process_server_address(flow) - - if isinstance(request_reply, HTTPResponse): - flow.response = request_reply - else: - self.get_response_from_server(flow) - - # no further manipulation of self.c.server_conn beyond this point - # we can safely set it as the final attribute value here. - flow.server_conn = self.c.server_conn - - self.c.log( - "response", - "debug", - [repr(flow.response)] - ) - response_reply = self.c.channel.ask("response", flow) - if response_reply is None or response_reply == KILL: - raise KillSignal() - - self.send_response_to_client(flow) - - if self.check_close_connection(flow): - return False - - # We sent a CONNECT request to an upstream proxy. - if flow.request.form_in == "authority" and flow.response.code == 200: - # TODO: Possibly add headers (memory consumption/usefulness - # tradeoff) Make sure to add state info before the actual - # processing of the CONNECT request happens. During an SSL - # upgrade, we may receive an SNI indication from the client, - # which resets the upstream connection. If this is the case, we - # must already re-issue the CONNECT request at this point. - self.c.server_conn.state.append( - ( - "http", { - "state": "connect", - "host": flow.request.host, - "port": flow.request.port - } - ) - ) - if not self.process_connect_request( - (flow.request.host, flow.request.port)): - return False - - # If the user has changed the target server on this connection, - # restore the original target server - flow.live.restore_server() - - return True # Next flow please. - except ( - http.HttpAuthenticationError, - http.HttpError, - proxy.ProxyError, - tcp.NetLibError, - ) as e: - self.handle_error(e, flow) - except KillSignal: - self.c.log("Connection killed", "info") - finally: - flow.live = None # Connection is not live anymore. - return False - - def handle_server_reconnect(self, state): - if state["state"] == "connect": - send_connect_request( - self.c.server_conn, - state["host"], - state["port"], - update_state=False - ) - else: # pragma: nocover - raise RuntimeError("Unknown State: %s" % state["state"]) - - def handle_error(self, error, flow=None): - message = repr(error) - message_debug = None - - if isinstance(error, tcp.NetLibError): - message = None - message_debug = "TCP connection closed unexpectedly." - elif "tlsv1 alert unknown ca" in message: - message = "TLSv1 Alert Unknown CA: The client does not trust the proxy's certificate." - elif "handshake error" in message: - message_debug = message - message = "SSL handshake error: The client may not trust the proxy's certificate." - - if message: - self.c.log(message, level="info") - if message_debug: - self.c.log(message_debug, level="debug") - - if flow: - # TODO: no flows without request or with both request and response - # at the moment. - if flow.request and not flow.response: - flow.error = Error(message or message_debug) - self.c.channel.ask("error", flow) - try: - status_code = getattr(error, "code", 502) - headers = getattr(error, "headers", None) - - html_message = message or "" - if message_debug: - html_message += "
%s
" % message_debug - self.send_error(status_code, html_message, headers) - except: - pass - - def send_error(self, status_code, message, headers): - response = http.status_codes.RESPONSES.get(status_code, "Unknown") - body = """ - - - %d %s - - %s - - """ % (status_code, response, message) - - if not headers: - headers = odict.ODictCaseless() - assert isinstance(headers, odict.ODictCaseless) - - headers["Server"] = [self.c.config.server_version] - headers["Connection"] = ["close"] - headers["Content-Length"] = [len(body)] - headers["Content-Type"] = ["text/html"] - - resp = HTTPResponse( - (1, 1), # if HTTP/2 is used, this value is ignored anyway - status_code, - response, - headers, - body, - ) - - # if no protocol is assigned yet - just assume HTTP/1 - # TODO: maybe check ALPN and use HTTP/2 if required? - protocol = self.c.client_conn.protocol or http1.HTTP1Protocol(self.c.client_conn) - self.c.client_conn.send(protocol.assemble(resp)) - - def process_request(self, flow, request): - """ - @returns: - True, if the request should not be sent upstream - False, if the connection should be aborted - None, if the request should be sent upstream - (a status code != None should be returned directly by handle_flow) - """ - - if not self.skip_authentication: - self.authenticate(request) - - # Determine .scheme, .host and .port attributes - # For absolute-form requests, they are directly given in the request. - # For authority-form requests, we only need to determine the request scheme. - # For relative-form requests, we need to determine host and port as - # well. - if not request.scheme: - request.scheme = "https" if flow.server_conn and flow.server_conn.ssl_established else "http" - if not request.host: - # Host/Port Complication: In upstream mode, use the server we CONNECTed to, - # not the upstream proxy. - if flow.server_conn: - for s in flow.server_conn.state: - if s[0] == "http" and s[1]["state"] == "connect": - request.host, request.port = s[1]["host"], s[1]["port"] - if not request.host and flow.server_conn: - request.host, request.port = flow.server_conn.address.host, flow.server_conn.address.port - - - # Now we can process the request. - if request.form_in == "authority": - if self.c.client_conn.ssl_established: - raise http.HttpError( - 400, - "Must not CONNECT on already encrypted connection" - ) - - if self.c.config.mode == "regular": - self.c.set_server_address((request.host, request.port)) - # Update server_conn attribute on the flow - flow.server_conn = self.c.server_conn - - # since we currently only support HTTP/1 CONNECT requests - # the response must be HTTP/1 as well - self.c.client_conn.send( - ('HTTP/%s.%s 200 ' % (request.httpversion[0], request.httpversion[1])) + - 'Connection established\r\n' + - 'Content-Length: 0\r\n' + - ('Proxy-agent: %s\r\n' % self.c.config.server_version) + - '\r\n' - ) - return self.process_connect_request(self.c.server_conn.address) - elif self.c.config.mode == "upstream": - return None - else: - # CONNECT should never occur if we don't expect absolute-form - # requests - pass - - elif request.form_in == self.expected_form_in: - request.form_out = self.expected_form_out - if request.form_in == "absolute": - if request.scheme != "http": - raise http.HttpError( - 400, - "Invalid request scheme: %s" % request.scheme - ) - if self.c.config.mode == "regular": - # Update info so that an inline script sees the correct - # value at flow.server_conn - self.c.set_server_address((request.host, request.port)) - flow.server_conn = self.c.server_conn - - elif request.form_in == "relative": - if self.c.config.mode == "spoof": - # Host header - h = request.pretty_host(hostheader=True) - if h is None: - raise http.HttpError( - 400, - "Invalid request: No host information" - ) - p = netlib.utils.parse_url("http://" + h) - request.scheme = p[0] - request.host = p[1] - request.port = p[2] - self.c.set_server_address((request.host, request.port)) - flow.server_conn = self.c.server_conn - - if self.c.config.mode == "sslspoof": - # SNI is processed in server.py - if not (flow.server_conn and flow.server_conn.ssl_established): - raise http.HttpError( - 400, - "Invalid request: No host information" - ) - - return None - - raise http.HttpError( - 400, "Invalid HTTP request form (expected: %s, got: %s)" % ( - self.expected_form_in, request.form_in - ) - ) - - def process_server_address(self, flow): - # Depending on the proxy mode, server handling is entirely different - # We provide a mostly unified API to the user, which needs to be - # unfiddled here - # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 ) - address = tcp.Address((flow.request.host, flow.request.port)) - - ssl = (flow.request.scheme == "https") - - if self.c.config.mode == "upstream": - # The connection to the upstream proxy may have a state we may need - # to take into account. - connected_to = None - for s in flow.server_conn.state: - if s[0] == "http" and s[1]["state"] == "connect": - connected_to = tcp.Address((s[1]["host"], s[1]["port"])) - - # We need to reconnect if the current flow either requires a - # (possibly impossible) change to the connection state, e.g. the - # host has changed but we already CONNECTed somewhere else. - needs_server_change = ( - ssl != self.c.server_conn.ssl_established - or - # HTTP proxying is "stateless", CONNECT isn't. - (connected_to and address != connected_to) - ) - - if needs_server_change: - # force create new connection to the proxy server to reset - # state - self.live.change_server(self.c.server_conn.address, force=True) - if ssl: - send_connect_request( - self.c.server_conn, - address.host, - address.port - ) - self.c.establish_ssl(server=True) - else: - # If we're not in upstream mode, we just want to update the host - # and possibly establish TLS. This is a no op if the addresses - # match. - self.live.change_server(address, ssl=ssl) - - flow.server_conn = self.c.server_conn - - def send_response_to_client(self, flow): - if not flow.response.stream: - # no streaming: - # we already received the full response from the server and can - # send it to the client straight away. - self.c.client_conn.send(self.c.client_conn.protocol.assemble(flow.response)) - else: - if isinstance(self.c.client_conn.protocol, http2.HTTP2Protocol): - raise NotImplementedError("HTTP streaming with HTTP/2 is currently not supported.") - - - # streaming: - # First send the headers and then transfer the response - # incrementally: - h = self.c.client_conn.protocol._assemble_response_first_line(flow.response) - self.c.client_conn.send(h + "\r\n") - h = self.c.client_conn.protocol._assemble_response_headers(flow.response, preserve_transfer_encoding=True) - self.c.client_conn.send(h + "\r\n") - - chunks = self.c.server_conn.protocol.read_http_body_chunked( - flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, - flow.response.code, - False, - 4096 - ) - - if callable(flow.response.stream): - chunks = flow.response.stream(chunks) - - for chunk in chunks: - for part in chunk: - self.c.client_conn.wfile.write(part) - self.c.client_conn.wfile.flush() - - flow.response.timestamp_end = utils.timestamp() - - def check_close_connection(self, flow): - """ - Checks if the connection should be closed depending on the HTTP - semantics. Returns True, if so. - """ - - # TODO: add logic for HTTP/2 - - close_connection = ( - http1.HTTP1Protocol.connection_close( - flow.request.httpversion, - flow.request.headers - ) or http1.HTTP1Protocol.connection_close( - flow.response.httpversion, - flow.response.headers - ) or http1.HTTP1Protocol.expected_http_body_size( - flow.response.headers, - False, - flow.request.method, - flow.response.code) == -1 - ) - if close_connection: - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 - # and no Content-Length header - pass - else: - return True - return False - - def process_connect_request(self, address): - """ - Process a CONNECT request. - Returns True if the CONNECT request has been processed successfully. - Returns False, if the connection should be closed immediately. - """ - address = tcp.Address.wrap(address) - if self.c.config.check_ignore(address): - self.c.log("Ignore host: %s:%s" % address(), "info") - TCPHandler(self.c, log=False).handle_messages() - return False - else: - self.expected_form_in = "relative" - self.expected_form_out = "relative" - self.skip_authentication = True - - # In practice, nobody issues a CONNECT request to send unencrypted - # HTTP requests afterwards. If we don't delegate to TCP mode, we - # should always negotiate a SSL connection. - # - # FIXME: Turns out the previous statement isn't entirely true. - # Chrome on Windows CONNECTs to :80 if an explicit proxy is - # configured and a websocket connection should be established. We - # don't support websocket at the moment, so it fails anyway, but we - # should come up with a better solution to this if we start to - # support WebSockets. - should_establish_ssl = ( - address.port in self.c.config.ssl_ports - or - not self.c.config.check_tcp(address) - ) - - if should_establish_ssl: - self.c.log( - "Received CONNECT request to SSL port. " - "Upgrading to SSL...", "debug" - ) - server_ssl = not self.c.config.no_upstream_cert - if server_ssl: - self.c.establish_server_connection() - self.c.establish_ssl(server=server_ssl, client=True) - self.c.log("Upgrade to SSL completed.", "debug") - - if self.c.config.check_tcp(address): - self.c.log( - "Generic TCP mode for host: %s:%s" % address(), - "info" - ) - TCPHandler(self.c).handle_messages() - return False - - return True - - def authenticate(self, request): - if self.c.config.authenticator: - if self.c.config.authenticator.authenticate(request.headers): - self.c.config.authenticator.clean(request.headers) - else: - raise http.HttpAuthenticationError( - self.c.config.authenticator.auth_challenge_headers()) - return request.headers \ No newline at end of file diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index b1000a79..a26ddbb4 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -1,20 +1,12 @@ from __future__ import absolute_import import Cookie import copy -import threading import time -import urllib -import urlparse from email.utils import parsedate_tz, formatdate, mktime_tz -import netlib -from netlib import http, tcp, odict, utils, encoding -from netlib.http import cookies, semantics, http1 - -from .tcp import TCPHandler -from .primitives import KILL, ProtocolHandler, Flow, Error -from ..proxy.connection import ServerConnection -from .. import utils, controller, stateobject, proxy +from netlib import odict, encoding +from netlib.http import semantics, CONTENT_MISSING +from .. import utils, stateobject class decoded(object): @@ -170,19 +162,19 @@ class HTTPRequest(MessageMixin, semantics.Request): """ def __init__( - self, - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - body, - timestamp_start=None, - timestamp_end=None, - form_out=None, + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + body, + timestamp_start=None, + timestamp_end=None, + form_out=None, ): semantics.Request.__init__( self, diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py index 92fc95e5..c663f0c5 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/protocol/primitives.py @@ -1,11 +1,10 @@ from __future__ import absolute_import import copy import uuid -import netlib.tcp + from .. import stateobject, utils, version from ..proxy.connection import ClientConnection, ServerConnection - KILL = 0 # const for killed requests @@ -165,130 +164,3 @@ class Flow(stateobject.StateObject): self.intercepted = False self.reply() master.handle_accept_intercept(self) - - - -class ProtocolHandler(object): - """ - A ProtocolHandler implements an application-layer protocol, e.g. HTTP. - See: libmproxy.protocol.http.HTTPHandler - """ - - def __init__(self, c): - self.c = c - """@type: libmproxy.proxy.server.ConnectionHandler""" - self.live = LiveConnection(c) - """@type: LiveConnection""" - - def handle_messages(self): - """ - This method gets called if a client connection has been made. Depending - on the proxy settings, a server connection might already exist as well. - """ - raise NotImplementedError # pragma: nocover - - def handle_server_reconnect(self, state): - """ - This method gets called if a server connection needs to reconnect and - there's a state associated with the server connection (e.g. a - previously-sent CONNECT request or a SOCKS proxy request). This method - gets called after the connection has been restablished but before SSL is - established. - """ - raise NotImplementedError # pragma: nocover - - def handle_error(self, error): - """ - This method gets called should there be an uncaught exception during the - connection. This might happen outside of handle_messages, e.g. if the - initial SSL handshake fails in transparent mode. - """ - raise error # pragma: nocover - - -class LiveConnection(object): - """ - This facade allows interested parties (FlowMaster, inline scripts) to - interface with a live connection, without exposing the internals - of the ConnectionHandler. - """ - - def __init__(self, c): - self.c = c - """@type: libmproxy.proxy.server.ConnectionHandler""" - self._backup_server_conn = None - """@type: libmproxy.proxy.connection.ServerConnection""" - - def change_server( - self, - address, - ssl=None, - sni=None, - force=False, - persistent_change=False): - """ - Change the server connection to the specified address. - @returns: - True, if a new connection has been established, - False, if an existing connection has been used - """ - address = netlib.tcp.Address.wrap(address) - - ssl_mismatch = ( - ssl is not None and - ( - (self.c.server_conn.connection and ssl != self.c.server_conn.ssl_established) - or - (sni is not None and sni != self.c.server_conn.sni) - ) - ) - address_mismatch = (address != self.c.server_conn.address) - - if persistent_change: - self._backup_server_conn = None - - if ssl_mismatch or address_mismatch or force: - - self.c.log( - "Change server connection: %s:%s -> %s:%s [persistent: %s]" % ( - self.c.server_conn.address.host, - self.c.server_conn.address.port, - address.host, - address.port, - persistent_change - ), - "debug" - ) - - if not self._backup_server_conn and not persistent_change: - self._backup_server_conn = self.c.server_conn - self.c.server_conn = None - else: - # This is at least the second temporary change. We can kill the - # current connection. - self.c.del_server_connection() - - self.c.set_server_address(address) - self.c.establish_server_connection(ask=False) - if ssl: - self.c.establish_ssl(server=True, sni=sni) - return True - return False - - def restore_server(self): - # TODO: Similar to _backup_server_conn, introduce _cache_server_conn, - # which keeps the changed connection open This may be beneficial if a - # user is rewriting all requests from http to https or similar. - if not self._backup_server_conn: - return - - self.c.log("Restore original server connection: %s:%s -> %s:%s" % ( - self.c.server_conn.address.host, - self.c.server_conn.address.port, - self._backup_server_conn.address.host, - self._backup_server_conn.address.port - ), "debug") - - self.c.del_server_connection() - self.c.server_conn = self._backup_server_conn - self._backup_server_conn = None diff --git a/libmproxy/protocol/tcp.py b/libmproxy/protocol/tcp.py deleted file mode 100644 index 0feb77c6..00000000 --- a/libmproxy/protocol/tcp.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import absolute_import -import select -import socket -from .primitives import ProtocolHandler -from netlib.utils import cleanBin -from netlib.tcp import NetLibError - - -class TCPHandler(ProtocolHandler): - """ - TCPHandler acts as a generic TCP forwarder. - Data will be .log()ed, but not stored any further. - """ - - chunk_size = 4096 - - def __init__(self, c, log=True): - super(TCPHandler, self).__init__(c) - self.log = log - - def handle_messages(self): - self.c.establish_server_connection() - - server = "%s:%s" % self.c.server_conn.address()[:2] - buf = memoryview(bytearray(self.chunk_size)) - conns = [self.c.client_conn.rfile, self.c.server_conn.rfile] - - try: - while True: - r, _, _ = select.select(conns, [], [], 10) - for rfile in r: - if self.c.client_conn.rfile == rfile: - src, dst = self.c.client_conn, self.c.server_conn - direction = "-> tcp ->" - src_str, dst_str = "client", server - else: - dst, src = self.c.client_conn, self.c.server_conn - direction = "<- tcp <-" - dst_str, src_str = "client", server - - closed = False - if src.ssl_established: - # Unfortunately, pyOpenSSL lacks a recv_into function. - # We need to read a single byte before .pending() - # becomes usable - contents = src.rfile.read(1) - contents += src.rfile.read(src.connection.pending()) - if not contents: - closed = True - else: - size = src.connection.recv_into(buf) - if not size: - closed = True - - if closed: - conns.remove(src.rfile) - # Shutdown connection to the other peer - if dst.ssl_established: - # We can't half-close a connection, so we just close everything here. - # Sockets will be cleaned up on a higher level. - return - else: - dst.connection.shutdown(socket.SHUT_WR) - - if len(conns) == 0: - return - continue - - if src.ssl_established or dst.ssl_established: - # if one of the peers is over SSL, we need to send - # bytes/strings - if not src.ssl_established: - # we revc'd into buf but need bytes/string now. - contents = buf[:size].tobytes() - if self.log: - self.c.log( - "%s %s\r\n%s" % ( - direction, dst_str, cleanBin(contents) - ), - "info" - ) - # Do not use dst.connection.send here, which may raise - # OpenSSL-specific errors. - dst.send(contents) - else: - # socket.socket.send supports raw bytearrays/memoryviews - if self.log: - self.c.log( - "%s %s\r\n%s" % ( - direction, dst_str, cleanBin(buf.tobytes()) - ), - "info" - ) - dst.connection.send(buf[:size]) - except (socket.error, NetLibError) as e: - self.c.log("TCP connection closed unexpectedly.", "debug") - return -- cgit v1.2.3 From 421b241ff010ae979cff8df504b6744e4c291aeb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 13:40:23 +0200 Subject: remove http2http references --- doc-src/features/reverseproxy.html | 17 ++++++----------- doc-src/modes.html | 8 ++++---- libmproxy/protocol2/reverse_proxy.py | 1 - libmproxy/protocol2/root_context.py | 1 + test/test_cmdline.py | 4 ---- test/test_server.py | 8 ++++++-- 6 files changed, 17 insertions(+), 22 deletions(-) diff --git a/doc-src/features/reverseproxy.html b/doc-src/features/reverseproxy.html index 5ef4efc5..af5a5c53 100644 --- a/doc-src/features/reverseproxy.html +++ b/doc-src/features/reverseproxy.html @@ -7,22 +7,17 @@ mitmproxy forwards HTTP proxy requests to an upstream proxy server. - +
command-line -R schema://hostname[:port]command-line -R scheme://hostname[:port]
-Here, **schema** is one of http, https, http2https or https2http. The latter -two extended schema specifications control the use of HTTP and HTTPS on -mitmproxy and the upstream server. You can indicate that mitmproxy should use -HTTP, and the upstream server uses HTTPS like this: +Here, **scheme** signifies if the proxy should use TLS to connect to the server. +mitmproxy accepts both encrypted and unencrypted requests and transforms them to what the server +expects. - http2https://hostname:port - -And you can indicate that mitmproxy should use HTTPS while the upstream -service uses HTTP like this: - - https2http://hostname:port + mitmdump -R https://httpbin.org -p 80 + mitmdump -R https://httpbin.org -p 443 ### Host Header diff --git a/doc-src/modes.html b/doc-src/modes.html index b5a38696..a878fd82 100644 --- a/doc-src/modes.html +++ b/doc-src/modes.html @@ -149,7 +149,7 @@ this:

Reverse Proxy

-Mitmproxy is usually used with a client that uses the proxy to access the +mitmproxy is usually used with a client that uses the proxy to access the Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal HTTP server: @@ -174,14 +174,14 @@ requests recorded in mitmproxy. - Say you have some toy project that should get SSL support. Simply set up mitmproxy with SSL termination and you're done (mitmdump -p 443 -R -https2http://localhost:80/). There are better tools for this specific +http://localhost:80/). There are better tools for this specific task, but mitmproxy is very quick and simple way to set up an SSL-speaking server. - Want to add a non-SSL-capable compression proxy in front of your server? You -could even spawn a mitmproxy instance that terminates SSL (https2http://...), +could even spawn a mitmproxy instance that terminates SSL (-R http://...), point it to the compression proxy and let the compression proxy point to a -SSL-initiating mitmproxy (http2https://...), which then points to the real +SSL-initiating mitmproxy (-R https://...), which then points to the real server. As you see, it's a fairly flexible thing. Note that mitmproxy supports either an HTTP or an HTTPS upstream server, not diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index c4cabccc..3ca998d5 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -1,7 +1,6 @@ from __future__ import (absolute_import, print_function, division) from .layer import Layer, ServerConnectionMixin -from .tls import TlsLayer class ReverseProxy(Layer, ServerConnectionMixin): diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 210ba6ab..daea54bd 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -10,6 +10,7 @@ from .layer import ServerConnectionMixin from .http_proxy import HttpProxy, HttpUpstreamProxy from .reverse_proxy import ReverseProxy + class RootContext(object): """ The outmost context provided to the root layer. diff --git a/test/test_cmdline.py b/test/test_cmdline.py index 1443ee1c..bb54d011 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -51,10 +51,6 @@ def test_parse_server_spec(): "Invalid server specification", cmdline.parse_server_spec, "http://") - tutils.raises( - "Invalid server specification", - cmdline.parse_server_spec, - "https2http://foo.com") def test_parse_setheaders(): diff --git a/test/test_server.py b/test/test_server.py index b691804b..66c3a0ae 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -468,7 +468,7 @@ class TestHttps2Http(tservers.ReverseProxTest): @classmethod def get_proxy_config(cls): d = super(TestHttps2Http, cls).get_proxy_config() - d["upstream_server"] = ("https2http", d["upstream_server"][1]) + d["upstream_server"] = ("http", d["upstream_server"][1]) return d def pathoc(self, ssl, sni=None): @@ -476,7 +476,7 @@ class TestHttps2Http(tservers.ReverseProxTest): Returns a connected Pathoc instance. """ p = pathoc.Pathoc( - ("localhost", self.proxy.port), ssl=ssl, sni=sni, fp=None + ("localhost", self.proxy.port), ssl=True, sni=sni, fp=None ) p.connect() return p @@ -490,6 +490,10 @@ class TestHttps2Http(tservers.ReverseProxTest): assert p.request("get:'/p/200'").status_code == 200 assert all("Error in handle_sni" not in msg for msg in self.proxy.log) + def test_http(self): + p = self.pathoc(ssl=False) + assert p.request("get:'/p/200'").status_code == 200 + class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin): ssl = False -- cgit v1.2.3 From a86ec56012136664688fa4a8efcd866b5e3e17a8 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 15:27:29 +0200 Subject: move files around --- libmproxy/console/common.py | 2 +- libmproxy/console/flowview.py | 2 +- libmproxy/filt.py | 2 +- libmproxy/flow.py | 23 +- libmproxy/models/__init__.py | 16 + libmproxy/models/connections.py | 194 ++++++++++ libmproxy/models/flow.py | 166 ++++++++ libmproxy/models/http.py | 554 ++++++++++++++++++++++++++ libmproxy/protocol/__init__.py | 13 +- libmproxy/protocol/base.py | 152 ++++++++ libmproxy/protocol/http.py | 602 +++++++++++++++++++++++++---- libmproxy/protocol/http_replay.py | 95 +++++ libmproxy/protocol/http_wrappers.py | 413 -------------------- libmproxy/protocol/primitives.py | 166 -------- libmproxy/protocol/rawtcp.py | 66 ++++ libmproxy/protocol/tls.py | 288 ++++++++++++++ libmproxy/protocol2/__init__.py | 13 - libmproxy/protocol2/http.py | 588 ---------------------------- libmproxy/protocol2/http_proxy.py | 26 -- libmproxy/protocol2/http_replay.py | 95 ----- libmproxy/protocol2/layer.py | 138 ------- libmproxy/protocol2/rawtcp.py | 66 ---- libmproxy/protocol2/reverse_proxy.py | 17 - libmproxy/protocol2/root_context.py | 95 ----- libmproxy/protocol2/socks_proxy.py | 59 --- libmproxy/protocol2/tls.py | 288 -------------- libmproxy/protocol2/transparent_proxy.py | 24 -- libmproxy/proxy/__init__.py | 8 +- libmproxy/proxy/config.py | 2 +- libmproxy/proxy/connection.py | 193 --------- libmproxy/proxy/modes/__init__.py | 12 + libmproxy/proxy/modes/http_proxy.py | 26 ++ libmproxy/proxy/modes/reverse_proxy.py | 17 + libmproxy/proxy/modes/socks_proxy.py | 60 +++ libmproxy/proxy/modes/transparent_proxy.py | 24 ++ libmproxy/proxy/primitives.py | 15 - libmproxy/proxy/root_context.py | 93 +++++ libmproxy/proxy/server.py | 23 +- test/test_dump.py | 10 +- test/test_filt.py | 2 +- test/test_flow.py | 83 ++-- test/test_proxy.py | 8 +- test/test_server.py | 14 +- test/tutils.py | 23 +- 44 files changed, 2397 insertions(+), 2379 deletions(-) create mode 100644 libmproxy/models/__init__.py create mode 100644 libmproxy/models/connections.py create mode 100644 libmproxy/models/flow.py create mode 100644 libmproxy/models/http.py create mode 100644 libmproxy/protocol/base.py create mode 100644 libmproxy/protocol/http_replay.py delete mode 100644 libmproxy/protocol/http_wrappers.py delete mode 100644 libmproxy/protocol/primitives.py create mode 100644 libmproxy/protocol/rawtcp.py create mode 100644 libmproxy/protocol/tls.py delete mode 100644 libmproxy/protocol2/__init__.py delete mode 100644 libmproxy/protocol2/http.py delete mode 100644 libmproxy/protocol2/http_proxy.py delete mode 100644 libmproxy/protocol2/http_replay.py delete mode 100644 libmproxy/protocol2/layer.py delete mode 100644 libmproxy/protocol2/rawtcp.py delete mode 100644 libmproxy/protocol2/reverse_proxy.py delete mode 100644 libmproxy/protocol2/root_context.py delete mode 100644 libmproxy/protocol2/socks_proxy.py delete mode 100644 libmproxy/protocol2/tls.py delete mode 100644 libmproxy/protocol2/transparent_proxy.py delete mode 100644 libmproxy/proxy/connection.py create mode 100644 libmproxy/proxy/modes/__init__.py create mode 100644 libmproxy/proxy/modes/http_proxy.py create mode 100644 libmproxy/proxy/modes/reverse_proxy.py create mode 100644 libmproxy/proxy/modes/socks_proxy.py create mode 100644 libmproxy/proxy/modes/transparent_proxy.py delete mode 100644 libmproxy/proxy/primitives.py create mode 100644 libmproxy/proxy/root_context.py diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 1940e390..c25f7267 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -8,7 +8,7 @@ from netlib.http.semantics import CONTENT_MISSING import netlib.utils from .. import utils -from ..protocol.http import decoded +from ..models import decoded from . import signals diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 1e0f0c17..8b828653 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -9,7 +9,7 @@ from netlib.http.semantics import CONTENT_MISSING from . import common, grideditor, contentview, signals, searchable, tabs from . import flowdetailview from .. import utils, controller -from ..protocol.http import HTTPRequest, HTTPResponse, decoded +from ..models import HTTPRequest, HTTPResponse, decoded class SearchError(Exception): diff --git a/libmproxy/filt.py b/libmproxy/filt.py index 25747bc6..cfd3a1bc 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -35,7 +35,7 @@ from __future__ import absolute_import import re import sys import pyparsing as pp -from .protocol.http import decoded +from .models import decoded class _Token: diff --git a/libmproxy/flow.py b/libmproxy/flow.py index a2f57512..00ec83d2 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,19 +8,18 @@ import Cookie import cookielib import os import re -from libmproxy.protocol.http import HTTPFlow -from libmproxy.protocol2.http_replay import RequestReplayThread +import urlparse + from netlib import odict, wsgi from netlib.http.semantics import CONTENT_MISSING import netlib.http - -from . import controller, protocol, tnetstring, filt, script, version +from . import controller, tnetstring, filt, script, version from .onboarding import app -from .protocol import http from .proxy.config import HostMatcher -from .proxy.connection import ClientConnection, ServerConnection -import urlparse +from .protocol.http_replay import RequestReplayThread +from .protocol import Kill +from .models import ClientConnection, ServerConnection, HTTPResponse, HTTPFlow, HTTPRequest class AppRegistry: @@ -790,7 +789,7 @@ class FlowMaster(controller.Master): rflow = self.server_playback.next_flow(flow) if not rflow: return None - response = http.HTTPResponse.from_state(rflow.response.get_state()) + response = HTTPResponse.from_state(rflow.response.get_state()) response.is_replay = True if self.refresh_server_playback: response.refresh() @@ -836,10 +835,10 @@ class FlowMaster(controller.Master): sni=host, ssl_established=True )) - f = http.HTTPFlow(c, s) + f = HTTPFlow(c, s) headers = odict.ODictCaseless() - req = http.HTTPRequest( + req = HTTPRequest( "absolute", method, scheme, @@ -981,7 +980,7 @@ class FlowMaster(controller.Master): ) if err: self.add_event("Error in wsgi app. %s" % err, "error") - f.reply(protocol.KILL) + f.reply(Kill) return if f not in self.state.flows: # don't add again on replay self.state.add_flow(f) @@ -998,7 +997,7 @@ class FlowMaster(controller.Master): if self.stream_large_bodies: self.stream_large_bodies.run(f, False) except netlib.http.HttpError: - f.reply(protocol.KILL) + f.reply(Kill) return f.reply() diff --git a/libmproxy/models/__init__.py b/libmproxy/models/__init__.py new file mode 100644 index 00000000..3947847c --- /dev/null +++ b/libmproxy/models/__init__.py @@ -0,0 +1,16 @@ +from __future__ import (absolute_import, print_function, division) + +from .http import ( + HTTPFlow, HTTPRequest, HTTPResponse, decoded, + make_error_response, make_connect_request, make_connect_response +) +from .connections import ClientConnection, ServerConnection +from .flow import Flow, Error + +__all__ = [ + "HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded" + "make_error_response", "make_connect_request", + "make_connect_response", + "ClientConnection", "ServerConnection", + "Flow", "Error", +] diff --git a/libmproxy/models/connections.py b/libmproxy/models/connections.py new file mode 100644 index 00000000..98bae3cc --- /dev/null +++ b/libmproxy/models/connections.py @@ -0,0 +1,194 @@ +from __future__ import absolute_import + +import copy +import os + +from netlib import tcp, certutils +from .. import stateobject, utils + + +class ClientConnection(tcp.BaseHandler, stateobject.StateObject): + def __init__(self, client_connection, address, server): + # Eventually, this object is restored from state. We don't have a + # connection then. + if client_connection: + super(ClientConnection, self).__init__(client_connection, address, server) + else: + self.connection = None + self.server = None + self.wfile = None + self.rfile = None + self.address = None + self.clientcert = None + self.ssl_established = None + + self.timestamp_start = utils.timestamp() + self.timestamp_end = None + self.timestamp_ssl_setup = None + self.protocol = None + + def __nonzero__(self): + return bool(self.connection) and not self.finished + + def __repr__(self): + return "".format( + ssl="[ssl] " if self.ssl_established else "", + host=self.address.host, + port=self.address.port + ) + + @property + def tls_established(self): + return self.ssl_established + + _stateobject_attributes = dict( + ssl_established=bool, + timestamp_start=float, + timestamp_end=float, + timestamp_ssl_setup=float + ) + + def get_state(self, short=False): + d = super(ClientConnection, self).get_state(short) + d.update( + address={ + "address": self.address(), + "use_ipv6": self.address.use_ipv6}, + clientcert=self.cert.to_pem() if self.clientcert else None) + return d + + def load_state(self, state): + super(ClientConnection, self).load_state(state) + self.address = tcp.Address( + **state["address"]) if state["address"] else None + self.clientcert = certutils.SSLCert.from_pem( + state["clientcert"]) if state["clientcert"] else None + + def copy(self): + return copy.copy(self) + + def send(self, message): + if isinstance(message, list): + message = b''.join(message) + self.wfile.write(message) + self.wfile.flush() + + @classmethod + def from_state(cls, state): + f = cls(None, tuple(), None) + f.load_state(state) + return f + + def convert_to_ssl(self, *args, **kwargs): + super(ClientConnection, self).convert_to_ssl(*args, **kwargs) + self.timestamp_ssl_setup = utils.timestamp() + + def finish(self): + super(ClientConnection, self).finish() + self.timestamp_end = utils.timestamp() + + +class ServerConnection(tcp.TCPClient, stateobject.StateObject): + def __init__(self, address): + tcp.TCPClient.__init__(self, address) + + self.via = None + self.timestamp_start = None + self.timestamp_end = None + self.timestamp_tcp_setup = None + self.timestamp_ssl_setup = None + self.protocol = None + + def __nonzero__(self): + return bool(self.connection) and not self.finished + + def __repr__(self): + if self.ssl_established and self.sni: + ssl = "[ssl: {0}] ".format(self.sni) + elif self.ssl_established: + ssl = "[ssl] " + else: + ssl = "" + return "".format( + ssl=ssl, + host=self.address.host, + port=self.address.port + ) + + @property + def tls_established(self): + return self.ssl_established + + _stateobject_attributes = dict( + timestamp_start=float, + timestamp_end=float, + timestamp_tcp_setup=float, + timestamp_ssl_setup=float, + address=tcp.Address, + source_address=tcp.Address, + cert=certutils.SSLCert, + ssl_established=bool, + sni=str + ) + _stateobject_long_attributes = {"cert"} + + def get_state(self, short=False): + d = super(ServerConnection, self).get_state(short) + d.update( + address={"address": self.address(), + "use_ipv6": self.address.use_ipv6}, + source_address=({"address": self.source_address(), + "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), + cert=self.cert.to_pem() if self.cert else None + ) + return d + + def load_state(self, state): + super(ServerConnection, self).load_state(state) + + self.address = tcp.Address( + **state["address"]) if state["address"] else None + self.source_address = tcp.Address( + **state["source_address"]) if state["source_address"] else None + self.cert = certutils.SSLCert.from_pem( + state["cert"]) if state["cert"] else None + + @classmethod + def from_state(cls, state): + f = cls(tuple()) + f.load_state(state) + return f + + def copy(self): + return copy.copy(self) + + def connect(self): + self.timestamp_start = utils.timestamp() + tcp.TCPClient.connect(self) + self.timestamp_tcp_setup = utils.timestamp() + + def send(self, message): + if isinstance(message, list): + message = b''.join(message) + self.wfile.write(message) + self.wfile.flush() + + def establish_ssl(self, clientcerts, sni, **kwargs): + clientcert = None + if clientcerts: + path = os.path.join( + clientcerts, + self.address.host.encode("idna")) + ".pem" + if os.path.exists(path): + clientcert = path + + self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) + self.sni = sni + self.timestamp_ssl_setup = utils.timestamp() + + def finish(self): + tcp.TCPClient.finish(self) + self.timestamp_end = utils.timestamp() + + +ServerConnection._stateobject_attributes["via"] = ServerConnection diff --git a/libmproxy/models/flow.py b/libmproxy/models/flow.py new file mode 100644 index 00000000..58287e5b --- /dev/null +++ b/libmproxy/models/flow.py @@ -0,0 +1,166 @@ +from __future__ import absolute_import +import copy +import uuid + +from .. import stateobject, utils, version +from .connections import ClientConnection, ServerConnection + + +class Error(stateobject.StateObject): + """ + An Error. + + This is distinct from an protocol error response (say, a HTTP code 500), + which is represented by a normal HTTPResponse object. This class is + responsible for indicating errors that fall outside of normal protocol + communications, like interrupted connections, timeouts, protocol errors. + + Exposes the following attributes: + + flow: Flow object + msg: Message describing the error + timestamp: Seconds since the epoch + """ + + def __init__(self, msg, timestamp=None): + """ + @type msg: str + @type timestamp: float + """ + self.flow = None # will usually be set by the flow backref mixin + self.msg = msg + self.timestamp = timestamp or utils.timestamp() + + _stateobject_attributes = dict( + msg=str, + timestamp=float + ) + + def __str__(self): + return self.msg + + @classmethod + def from_state(cls, state): + # the default implementation assumes an empty constructor. Override + # accordingly. + f = cls(None) + f.load_state(state) + return f + + def copy(self): + c = copy.copy(self) + return c + + +class Flow(stateobject.StateObject): + """ + A Flow is a collection of objects representing a single transaction. + This class is usually subclassed for each protocol, e.g. HTTPFlow. + """ + + def __init__(self, type, client_conn, server_conn, live=None): + self.type = type + self.id = str(uuid.uuid4()) + self.client_conn = client_conn + """@type: ClientConnection""" + self.server_conn = server_conn + """@type: ServerConnection""" + self.live = live + """@type: LiveConnection""" + + self.error = None + """@type: Error""" + self.intercepted = False + """@type: bool""" + self._backup = None + self.reply = None + + _stateobject_attributes = dict( + id=str, + error=Error, + client_conn=ClientConnection, + server_conn=ServerConnection, + type=str, + intercepted=bool + ) + + def get_state(self, short=False): + d = super(Flow, self).get_state(short) + d.update(version=version.IVERSION) + if self._backup and self._backup != d: + if short: + d.update(modified=True) + else: + d.update(backup=self._backup) + return d + + def __eq__(self, other): + return self is other + + def copy(self): + f = copy.copy(self) + + f.id = str(uuid.uuid4()) + f.live = False + f.client_conn = self.client_conn.copy() + f.server_conn = self.server_conn.copy() + + if self.error: + f.error = self.error.copy() + return f + + def modified(self): + """ + Has this Flow been modified? + """ + if self._backup: + return self._backup != self.get_state() + else: + return False + + def backup(self, force=False): + """ + Save a backup of this Flow, which can be reverted to using a + call to .revert(). + """ + if not self._backup: + self._backup = self.get_state() + + def revert(self): + """ + Revert to the last backed up state. + """ + if self._backup: + self.load_state(self._backup) + self._backup = None + + def kill(self, master): + """ + Kill this request. + """ + from ..protocol import Kill + + self.error = Error("Connection killed") + self.intercepted = False + self.reply(Kill) + master.handle_error(self) + + def intercept(self, master): + """ + Intercept this Flow. Processing will stop until accept_intercept is + called. + """ + if self.intercepted: + return + self.intercepted = True + master.handle_intercept(self) + + def accept_intercept(self, master): + """ + Continue with the flow - called after an intercept(). + """ + if not self.intercepted: + return + self.intercepted = False + self.reply() + master.handle_accept_intercept(self) diff --git a/libmproxy/models/http.py b/libmproxy/models/http.py new file mode 100644 index 00000000..fb2f305b --- /dev/null +++ b/libmproxy/models/http.py @@ -0,0 +1,554 @@ +from __future__ import (absolute_import, print_function, division) +import Cookie +import copy +from email.utils import parsedate_tz, formatdate, mktime_tz +import time + +from libmproxy import utils +from netlib import odict, encoding +from netlib.http import status_codes +from netlib.tcp import Address +from netlib.http.semantics import Request, Response, CONTENT_MISSING +from .. import version, stateobject +from .flow import Flow + + +class MessageMixin(stateobject.StateObject): + _stateobject_attributes = dict( + httpversion=tuple, + headers=odict.ODictCaseless, + body=str, + timestamp_start=float, + timestamp_end=float + ) + _stateobject_long_attributes = {"body"} + + def get_state(self, short=False): + ret = super(MessageMixin, self).get_state(short) + if short: + if self.body: + ret["contentLength"] = len(self.body) + elif self.body == CONTENT_MISSING: + ret["contentLength"] = None + else: + ret["contentLength"] = 0 + return ret + + def get_decoded_content(self): + """ + Returns the decoded content based on the current Content-Encoding + header. + Doesn't change the message iteself or its headers. + """ + ce = self.headers.get_first("content-encoding") + if not self.body or ce not in encoding.ENCODINGS: + return self.body + return encoding.decode(ce, self.body) + + def decode(self): + """ + Decodes body based on the current Content-Encoding header, then + removes the header. If there is no Content-Encoding header, no + action is taken. + + Returns True if decoding succeeded, False otherwise. + """ + ce = self.headers.get_first("content-encoding") + if not self.body or ce not in encoding.ENCODINGS: + return False + data = encoding.decode(ce, self.body) + if data is None: + return False + self.body = data + del self.headers["content-encoding"] + return True + + def encode(self, e): + """ + Encodes body with the encoding e, where e is "gzip", "deflate" + or "identity". + """ + # FIXME: Error if there's an existing encoding header? + self.body = encoding.encode(e, self.body) + self.headers["content-encoding"] = [e] + + def copy(self): + c = copy.copy(self) + c.headers = self.headers.copy() + return c + + def replace(self, pattern, repl, *args, **kwargs): + """ + Replaces a regular expression pattern with repl in both the headers + and the body of the message. Encoded body will be decoded + before replacement, and re-encoded afterwards. + + Returns the number of replacements made. + """ + with decoded(self): + self.body, c = utils.safe_subn( + pattern, repl, self.body, *args, **kwargs + ) + c += self.headers.replace(pattern, repl, *args, **kwargs) + return c + + +class HTTPRequest(MessageMixin, Request): + """ + An HTTP request. + + Exposes the following attributes: + + method: HTTP method + + scheme: URL scheme (http/https) + + host: Target hostname of the request. This is not neccessarily the + directy upstream server (which could be another proxy), but it's always + the target server we want to reach at the end. This attribute is either + inferred from the request itself (absolute-form, authority-form) or from + the connection metadata (e.g. the host in reverse proxy mode). + + port: Destination port + + path: Path portion of the URL (not present in authority-form) + + httpversion: HTTP version tuple, e.g. (1,1) + + headers: odict.ODictCaseless object + + content: Content of the request, None, or CONTENT_MISSING if there + is content associated, but not present. CONTENT_MISSING evaluates + to False to make checking for the presence of content natural. + + form_in: The request form which mitmproxy has received. The following + values are possible: + + - relative (GET /index.html, OPTIONS *) (covers origin form and + asterisk form) + - absolute (GET http://example.com:80/index.html) + - authority-form (CONNECT example.com:443) + Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 + + form_out: The request form which mitmproxy will send out to the + destination + + timestamp_start: Timestamp indicating when request transmission started + + timestamp_end: Timestamp indicating when request transmission ended + """ + + def __init__( + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + body, + timestamp_start=None, + timestamp_end=None, + form_out=None, + ): + Request.__init__( + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + body, + timestamp_start, + timestamp_end, + ) + self.form_out = form_out or form_in + + # Have this request's cookies been modified by sticky cookies or auth? + self.stickycookie = False + self.stickyauth = False + + # Is this request replayed? + self.is_replay = False + + _stateobject_attributes = MessageMixin._stateobject_attributes.copy() + _stateobject_attributes.update( + form_in=str, + method=str, + scheme=str, + host=str, + port=int, + path=str, + form_out=str, + is_replay=bool + ) + + @classmethod + def from_state(cls, state): + f = cls( + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None) + f.load_state(state) + return f + + @classmethod + def from_protocol( + self, + protocol, + *args, + **kwargs + ): + req = protocol.read_request(*args, **kwargs) + return self.wrap(req) + + @classmethod + def wrap(self, request): + req = HTTPRequest( + form_in=request.form_in, + method=request.method, + scheme=request.scheme, + host=request.host, + port=request.port, + path=request.path, + httpversion=request.httpversion, + headers=request.headers, + body=request.body, + timestamp_start=request.timestamp_start, + timestamp_end=request.timestamp_end, + form_out=(request.form_out if hasattr(request, 'form_out') else None), + ) + if hasattr(request, 'stream_id'): + req.stream_id = request.stream_id + return req + + def __hash__(self): + return id(self) + + def replace(self, pattern, repl, *args, **kwargs): + """ + Replaces a regular expression pattern with repl in the headers, the + request path and the body of the request. Encoded content will be + decoded before replacement, and re-encoded afterwards. + + Returns the number of replacements made. + """ + c = MessageMixin.replace(self, pattern, repl, *args, **kwargs) + self.path, pc = utils.safe_subn( + pattern, repl, self.path, *args, **kwargs + ) + c += pc + return c + + +class HTTPResponse(MessageMixin, Response): + """ + An HTTP response. + + Exposes the following attributes: + + httpversion: HTTP version tuple, e.g. (1, 0), (1, 1), or (2, 0) + + status_code: HTTP response status code + + msg: HTTP response message + + headers: ODict Caseless object + + content: Content of the request, None, or CONTENT_MISSING if there + is content associated, but not present. CONTENT_MISSING evaluates + to False to make checking for the presence of content natural. + + timestamp_start: Timestamp indicating when request transmission started + + timestamp_end: Timestamp indicating when request transmission ended + """ + + def __init__( + self, + httpversion, + status_code, + msg, + headers, + body, + timestamp_start=None, + timestamp_end=None, + ): + Response.__init__( + self, + httpversion, + status_code, + msg, + headers, + body, + timestamp_start=timestamp_start, + timestamp_end=timestamp_end, + ) + + # Is this request replayed? + self.is_replay = False + self.stream = False + + _stateobject_attributes = MessageMixin._stateobject_attributes.copy() + _stateobject_attributes.update( + status_code=int, + msg=str + ) + + @classmethod + def from_state(cls, state): + f = cls(None, None, None, None, None) + f.load_state(state) + return f + + @classmethod + def from_protocol( + self, + protocol, + *args, + **kwargs + ): + resp = protocol.read_response(*args, **kwargs) + return self.wrap(resp) + + @classmethod + def wrap(self, response): + resp = HTTPResponse( + httpversion=response.httpversion, + status_code=response.status_code, + msg=response.msg, + headers=response.headers, + body=response.body, + timestamp_start=response.timestamp_start, + timestamp_end=response.timestamp_end, + ) + if hasattr(response, 'stream_id'): + resp.stream_id = response.stream_id + return resp + + def _refresh_cookie(self, c, delta): + """ + Takes a cookie string c and a time delta in seconds, and returns + a refreshed cookie string. + """ + c = Cookie.SimpleCookie(str(c)) + for i in c.values(): + if "expires" in i: + d = parsedate_tz(i["expires"]) + if d: + d = mktime_tz(d) + delta + i["expires"] = formatdate(d) + else: + # This can happen when the expires tag is invalid. + # reddit.com sends a an expires tag like this: "Thu, 31 Dec + # 2037 23:59:59 GMT", which is valid RFC 1123, but not + # strictly correct according to the cookie spec. Browsers + # appear to parse this tolerantly - maybe we should too. + # For now, we just ignore this. + del i["expires"] + return c.output(header="").strip() + + def refresh(self, now=None): + """ + This fairly complex and heuristic function refreshes a server + response for replay. + + - It adjusts date, expires and last-modified headers. + - It adjusts cookie expiration. + """ + if not now: + now = time.time() + delta = now - self.timestamp_start + refresh_headers = [ + "date", + "expires", + "last-modified", + ] + for i in refresh_headers: + if i in self.headers: + d = parsedate_tz(self.headers[i][0]) + if d: + new = mktime_tz(d) + delta + self.headers[i] = [formatdate(new)] + c = [] + for i in self.headers["set-cookie"]: + c.append(self._refresh_cookie(i, delta)) + if c: + self.headers["set-cookie"] = c + + +class HTTPFlow(Flow): + """ + A HTTPFlow is a collection of objects representing a single HTTP + transaction. The main attributes are: + + request: HTTPRequest object + response: HTTPResponse object + error: Error object + server_conn: ServerConnection object + client_conn: ClientConnection object + + Note that it's possible for a Flow to have both a response and an error + object. This might happen, for instance, when a response was received + from the server, but there was an error sending it back to the client. + + The following additional attributes are exposed: + + intercepted: Is this flow currently being intercepted? + live: Does this flow have a live client connection? + """ + + def __init__(self, client_conn, server_conn, live=None): + super(HTTPFlow, self).__init__("http", client_conn, server_conn, live) + self.request = None + """@type: HTTPRequest""" + self.response = None + """@type: HTTPResponse""" + + _stateobject_attributes = Flow._stateobject_attributes.copy() + _stateobject_attributes.update( + request=HTTPRequest, + response=HTTPResponse + ) + + @classmethod + def from_state(cls, state): + f = cls(None, None) + f.load_state(state) + return f + + def __repr__(self): + s = " + + %d %s + + %s + + """.strip() % (status_code, response, message) + + if not headers: + headers = odict.ODictCaseless() + headers["Server"] = [version.NAMEVERSION] + headers["Connection"] = ["close"] + headers["Content-Length"] = [len(body)] + headers["Content-Type"] = ["text/html"] + + return HTTPResponse( + (1, 1), # FIXME: Should be a string. + status_code, + response, + headers, + body, + ) + + +def make_connect_request(address): + address = Address.wrap(address) + return HTTPRequest( + "authority", "CONNECT", None, address.host, address.port, None, (1, 1), + odict.ODictCaseless(), "" + ) + + +def make_connect_response(httpversion): + headers = odict.ODictCaseless([ + ["Content-Length", "0"], + ["Proxy-Agent", version.NAMEVERSION] + ]) + return HTTPResponse( + httpversion, + 200, + "Connection established", + headers, + "", + ) diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index bbc20dba..c582592b 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -1 +1,12 @@ -from .primitives import * +from __future__ import (absolute_import, print_function, division) +from .base import Layer, ServerConnectionMixin, Log, Kill +from .http import Http1Layer, Http2Layer +from .tls import TlsLayer, is_tls_record_magic +from .rawtcp import RawTCPLayer + +__all__ = [ + "Layer", "ServerConnectionMixin", "Log", "Kill", + "Http1Layer", "Http2Layer", + "TlsLayer", "is_tls_record_magic", + "RawTCPLayer" +] diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py new file mode 100644 index 00000000..d22a71c6 --- /dev/null +++ b/libmproxy/protocol/base.py @@ -0,0 +1,152 @@ +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible settings (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentProxy + Http1Layer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + HttpProxy + Http1Layer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentProxy + TLSLayer + Http2Layer + HttpLayer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can call functions provided by lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" +from __future__ import (absolute_import, print_function, division) +from netlib import tcp +from ..models import ServerConnection +from ..exceptions import ProtocolException + + +class _LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + + def __init__(self, *args, **kwargs): + super(_LayerCodeCompletion, self).__init__(*args, **kwargs) + if True: + return + self.config = None + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(_LayerCodeCompletion): + def __init__(self, ctx, *args, **kwargs): + """ + Args: + ctx: The (read-only) higher layer. + """ + super(Layer, self).__init__(*args, **kwargs) + self.ctx = ctx + + def __call__(self): + """ + Logic of the layer. + Raises: + ProtocolException in case of protocol exceptions. + """ + raise NotImplementedError + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "{}: {}".format(repr(self.client_conn.address), msg) + ] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + @property + def layers(self): + return [self] + self.ctx.layers + + def __repr__(self): + return type(self).__name__ + + +class ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + + def __init__(self, server_address=None): + super(ServerConnectionMixin, self).__init__() + self.server_conn = ServerConnection(server_address) + + def reconnect(self): + address = self.server_conn.address + self._disconnect() + self.server_conn.address = address + self.connect() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + if depth == 1: + if self.server_conn: + self._disconnect() + self.log("Set new server address: " + repr(address), "debug") + self.server_conn.address = address + else: + self.ctx.set_server(address, server_tls, sni, depth - 1) + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + self.server_conn.finish() + self.server_conn.close() + # self.channel.tell("serverdisconnect", self) + self.server_conn = ServerConnection(None) + + def connect(self): + if not self.server_conn.address: + raise ProtocolException("Cannot connect to server, no server address given.") + self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProtocolException( + "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) + + +class Log(object): + def __init__(self, msg, level="info"): + self.msg = msg + self.level = level + + +class Kill(Exception): + """ + Kill a connection. + """ diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index bde7b088..fc57f6df 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,92 +1,538 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) -from .primitives import Flow +from netlib import tcp +from netlib.http import http1, HttpErrorConnClosed, HttpError +from netlib.http.semantics import CONTENT_MISSING +from netlib import odict +from netlib.tcp import NetLibError, Address +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol -from .http_wrappers import decoded, HTTPRequest, HTTPResponse +from .. import utils +from ..exceptions import InvalidCredentials, HttpException, ProtocolException +from ..models import ( + HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error +) +from .base import Layer, Kill -class HTTPFlow(Flow): - """ - A HTTPFlow is a collection of objects representing a single HTTP - transaction. The main attributes are: - request: HTTPRequest object - response: HTTPResponse object - error: Error object - server_conn: ServerConnection object - client_conn: ClientConnection object +class _HttpLayer(Layer): + supports_streaming = False + + def read_request(self): + raise NotImplementedError() + + def send_request(self, request): + raise NotImplementedError() + + def read_response(self, request_method): + raise NotImplementedError() + + def send_response(self, response): + raise NotImplementedError() + + +class _StreamingHttpLayer(_HttpLayer): + supports_streaming = True + + def read_response_headers(self): + raise NotImplementedError + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + raise NotImplementedError() + yield "this is a generator" + + def send_response_headers(self, response): + raise NotImplementedError + + def send_response_body(self, response, chunks): + raise NotImplementedError() + + +class Http1Layer(_StreamingHttpLayer): + def __init__(self, ctx, mode): + super(Http1Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP1Protocol(self.client_conn) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def read_request(self): + return HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + + def send_request(self, request): + self.server_conn.send(self.server_protocol.assemble(request)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True + ) + + def send_response(self, response): + self.client_conn.send(self.client_protocol.assemble(response)) + + def read_response_headers(self): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=None, # does not matter if we don't read the body. + body_size_limit=self.config.body_size_limit, + include_body=False + ) + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + return self.server_protocol.read_http_body_chunked( + headers, + self.config.body_size_limit, + request_method, + response_code, + False, + max_chunk_size + ) + + def send_response_headers(self, response): + h = self.client_protocol._assemble_response_first_line(response) + self.client_conn.wfile.write(h + "\r\n") + h = self.client_protocol._assemble_response_headers( + response, + preserve_transfer_encoding=True + ) + self.client_conn.send(h + "\r\n") + + def send_response_body(self, response, chunks): + if self.client_protocol.has_chunked_encoding(response.headers): + chunks = ( + "%d\r\n%s\r\n" % (len(chunk), chunk) + for chunk in chunks + ) + for chunk in chunks: + self.client_conn.send(chunk) - Note that it's possible for a Flow to have both a response and an error - object. This might happen, for instance, when a response was received - from the server, but there was an error sending it back to the client. + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP1Protocol(self.server_conn) - The following additional attributes are exposed: + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP1Protocol(self.server_conn) - intercepted: Is this flow currently being intercepted? - live: Does this flow have a live client connection? + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def __call__(self): + layer = HttpLayer(self, self.mode) + layer() + + +# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. +class Http2Layer(_HttpLayer): + def __init__(self, ctx, mode): + super(Http2Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + + def read_request(self): + request = HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + self._stream_id = request.stream_id + return request + + def send_request(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.server_conn.send(self.server_protocol.assemble(message)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True, + stream_id=self._stream_id + ) + + def send_response(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.client_conn.send(self.client_protocol.assemble(message)) + + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def __call__(self): + self.server_protocol.perform_connection_preface() + layer = HttpLayer(self, self.mode) + layer() + + def handle_unexpected_frame(self, frm): + self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") + + +class ConnectServerConnection(object): + """ + "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. """ - def __init__(self, client_conn, server_conn, live=None): - super(HTTPFlow, self).__init__("http", client_conn, server_conn, live) - self.request = None - """@type: HTTPRequest""" - self.response = None - """@type: HTTPResponse""" - - _stateobject_attributes = Flow._stateobject_attributes.copy() - _stateobject_attributes.update( - request=HTTPRequest, - response=HTTPResponse - ) - - @classmethod - def from_state(cls, state): - f = cls(None, None) - f.load_state(state) - return f - - def __repr__(self): - s = " clientconnect (transparent mode destination known) + # > serverconnect (required for client tls handshake) + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + self.reconnect() + get_response() + + # call the appropriate script hook - this is an opportunity for an + # inline script to set flow.stream = True + flow = self.channel.ask("responseheaders", flow) + if flow is None or flow == Kill: + raise Kill() + + if self.supports_streaming: + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = "".join(self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code + )) + flow.response.timestamp_end = utils.timestamp() + + # no further manipulation of self.server_conn beyond this point + # we can safely set it as the final attribute value here. + flow.server_conn = self.server_conn + + self.log( + "response", + "debug", + [repr(flow.response)] + ) + response_reply = self.channel.ask("response", flow) + if response_reply is None or response_reply == Kill: + raise Kill() + + def process_request_hook(self, flow): + # Determine .scheme, .host and .port attributes for inline scripts. + # For absolute-form requests, they are directly given in the request. + # For authority-form requests, we only need to determine the request scheme. + # For relative-form requests, we need to determine host and port as + # well. + if self.mode == "regular": + pass # only absolute-form at this point, nothing to do here. + elif self.mode == "upstream": + if flow.request.form_in == "authority": + flow.request.scheme = "http" # pseudo value + else: + flow.request.host = self.__original_server_conn.address.host + flow.request.port = self.__original_server_conn.address.port + flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" + + request_reply = self.channel.ask("request", flow) + if request_reply is None or request_reply == Kill: + raise Kill() + if isinstance(request_reply, HTTPResponse): + flow.response = request_reply + return + + def establish_server_connection(self, flow): + address = tcp.Address((flow.request.host, flow.request.port)) + tls = (flow.request.scheme == "https") + + if self.mode == "regular" or self.mode == "transparent": + # If there's an existing connection that doesn't match our expectations, kill it. + if address != self.server_conn.address or tls != self.server_conn.ssl_established: + self.set_server(address, tls, address.host) + # Establish connection is neccessary. + if not self.server_conn: + self.connect() + + # SetServer is not guaranteed to work with TLS: + # If there's not TlsLayer below which could catch the exception, + # TLS will not be established. + if tls and not self.server_conn.tls_established: + raise ProtocolException( + "Cannot upgrade to SSL, no TLS layer on the protocol stack.") + else: + if not self.server_conn: + self.connect() + if tls: + raise HttpException("Cannot change scheme in upstream proxy mode.") + """ + # This is a very ugly (untested) workaround to solve a very ugly problem. + if self.server_conn and self.server_conn.tls_established and not ssl: + self.reconnect() + elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: + if self.server_conn.tls_established: + self.reconnect() + + self.send_request(make_connect_request(address)) + tls_layer = TlsLayer(self, False, True) + tls_layer._establish_tls_with_server() + """ + + def validate_request(self, request): + if request.form_in == "absolute" and request.scheme != "http": + self.send_response( + make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + raise HttpException("Invalid request scheme: %s" % request.scheme) + + expected_request_forms = { + "regular": ("absolute",), # an authority request would already be handled. + "upstream": ("authority", "absolute"), + "transparent": ("relative",) + } + + allowed_request_forms = expected_request_forms[self.mode] + if request.form_in not in allowed_request_forms: + err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( + " or ".join(allowed_request_forms), request.form_in + ) + self.send_response(make_error_response(400, err_message)) + raise HttpException(err_message) + + if self.mode == "regular": + request.form_out = "relative" + + def authenticate(self, request): + if self.config.authenticator: + if self.config.authenticator.authenticate(request.headers): + self.config.authenticator.clean(request.headers) + else: + self.send_response(make_error_response( + 407, + "Proxy Authentication Required", + odict.ODictCaseless( + [ + [k, v] for k, v in + self.config.authenticator.auth_challenge_headers().items() + ]) + )) + raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py new file mode 100644 index 00000000..e0144c93 --- /dev/null +++ b/libmproxy/protocol/http_replay.py @@ -0,0 +1,95 @@ +import threading + +from netlib.http import HttpError +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError +from ..controller import Channel +from ..models import Error, HTTPResponse, ServerConnection, make_connect_request +from .base import Log, Kill + + +# TODO: Doesn't really belong into libmproxy.protocol... + + +class RequestReplayThread(threading.Thread): + name = "RequestReplayThread" + + def __init__(self, config, flow, masterq, should_exit): + """ + masterqueue can be a queue or None, if no scripthooks should be + processed. + """ + self.config, self.flow = config, flow + if masterq: + self.channel = Channel(masterq, should_exit) + else: + self.channel = None + super(RequestReplayThread, self).__init__() + + def run(self): + r = self.flow.request + form_out_backup = r.form_out + try: + self.flow.response = None + + # If we have a channel, run script hooks. + if self.channel: + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == Kill: + raise Kill() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply + + if not self.flow.response: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + server_address = self.config.upstream_server.address + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + connect_request = make_connect_request((r.host, r.port)) + server.send(protocol.assemble(connect_request)) + resp = protocol.read_response("CONNECT") + if resp.code != 200: + raise HttpError(502, "Upstream server refuses CONNECT request") + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + + server.send(protocol.assemble(r)) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_protocol( + protocol, + r.method, + body_size_limit=self.config.body_size_limit, + ) + if self.channel: + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == Kill: + raise Kill() + except (HttpError, NetLibError) as v: + self.flow.error = Error(repr(v)) + if self.channel: + self.channel.ask("error", self.flow) + except Kill: + # KillSignal should only be raised if there's a channel in the + # first place. + self.channel.tell("log", Log("Connection killed", "info")) + finally: + r.form_out = form_out_backup diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py deleted file mode 100644 index a26ddbb4..00000000 --- a/libmproxy/protocol/http_wrappers.py +++ /dev/null @@ -1,413 +0,0 @@ -from __future__ import absolute_import -import Cookie -import copy -import time -from email.utils import parsedate_tz, formatdate, mktime_tz - -from netlib import odict, encoding -from netlib.http import semantics, CONTENT_MISSING -from .. import utils, stateobject - - -class decoded(object): - """ - A context manager that decodes a request or response, and then - re-encodes it with the same encoding after execution of the block. - - Example: - with decoded(request): - request.content = request.content.replace("foo", "bar") - """ - - def __init__(self, o): - self.o = o - ce = o.headers.get_first("content-encoding") - if ce in encoding.ENCODINGS: - self.ce = ce - else: - self.ce = None - - def __enter__(self): - if self.ce: - self.o.decode() - - def __exit__(self, type, value, tb): - if self.ce: - self.o.encode(self.ce) - - -class MessageMixin(stateobject.StateObject): - _stateobject_attributes = dict( - httpversion=tuple, - headers=odict.ODictCaseless, - body=str, - timestamp_start=float, - timestamp_end=float - ) - _stateobject_long_attributes = {"body"} - - def get_state(self, short=False): - ret = super(MessageMixin, self).get_state(short) - if short: - if self.body: - ret["contentLength"] = len(self.body) - elif self.body == CONTENT_MISSING: - ret["contentLength"] = None - else: - ret["contentLength"] = 0 - return ret - - def get_decoded_content(self): - """ - Returns the decoded content based on the current Content-Encoding - header. - Doesn't change the message iteself or its headers. - """ - ce = self.headers.get_first("content-encoding") - if not self.body or ce not in encoding.ENCODINGS: - return self.body - return encoding.decode(ce, self.body) - - def decode(self): - """ - Decodes body based on the current Content-Encoding header, then - removes the header. If there is no Content-Encoding header, no - action is taken. - - Returns True if decoding succeeded, False otherwise. - """ - ce = self.headers.get_first("content-encoding") - if not self.body or ce not in encoding.ENCODINGS: - return False - data = encoding.decode(ce, self.body) - if data is None: - return False - self.body = data - del self.headers["content-encoding"] - return True - - def encode(self, e): - """ - Encodes body with the encoding e, where e is "gzip", "deflate" - or "identity". - """ - # FIXME: Error if there's an existing encoding header? - self.body = encoding.encode(e, self.body) - self.headers["content-encoding"] = [e] - - def copy(self): - c = copy.copy(self) - c.headers = self.headers.copy() - return c - - def replace(self, pattern, repl, *args, **kwargs): - """ - Replaces a regular expression pattern with repl in both the headers - and the body of the message. Encoded body will be decoded - before replacement, and re-encoded afterwards. - - Returns the number of replacements made. - """ - with decoded(self): - self.body, c = utils.safe_subn( - pattern, repl, self.body, *args, **kwargs - ) - c += self.headers.replace(pattern, repl, *args, **kwargs) - return c - - -class HTTPRequest(MessageMixin, semantics.Request): - """ - An HTTP request. - - Exposes the following attributes: - - method: HTTP method - - scheme: URL scheme (http/https) - - host: Target hostname of the request. This is not neccessarily the - directy upstream server (which could be another proxy), but it's always - the target server we want to reach at the end. This attribute is either - inferred from the request itself (absolute-form, authority-form) or from - the connection metadata (e.g. the host in reverse proxy mode). - - port: Destination port - - path: Path portion of the URL (not present in authority-form) - - httpversion: HTTP version tuple, e.g. (1,1) - - headers: odict.ODictCaseless object - - content: Content of the request, None, or CONTENT_MISSING if there - is content associated, but not present. CONTENT_MISSING evaluates - to False to make checking for the presence of content natural. - - form_in: The request form which mitmproxy has received. The following - values are possible: - - - relative (GET /index.html, OPTIONS *) (covers origin form and - asterisk form) - - absolute (GET http://example.com:80/index.html) - - authority-form (CONNECT example.com:443) - Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 - - form_out: The request form which mitmproxy will send out to the - destination - - timestamp_start: Timestamp indicating when request transmission started - - timestamp_end: Timestamp indicating when request transmission ended - """ - - def __init__( - self, - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - body, - timestamp_start=None, - timestamp_end=None, - form_out=None, - ): - semantics.Request.__init__( - self, - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - body, - timestamp_start, - timestamp_end, - ) - self.form_out = form_out or form_in - - # Have this request's cookies been modified by sticky cookies or auth? - self.stickycookie = False - self.stickyauth = False - - # Is this request replayed? - self.is_replay = False - - _stateobject_attributes = MessageMixin._stateobject_attributes.copy() - _stateobject_attributes.update( - form_in=str, - method=str, - scheme=str, - host=str, - port=int, - path=str, - form_out=str, - is_replay=bool - ) - - @classmethod - def from_state(cls, state): - f = cls( - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None) - f.load_state(state) - return f - - @classmethod - def from_protocol( - self, - protocol, - *args, - **kwargs - ): - req = protocol.read_request(*args, **kwargs) - return self.wrap(req) - - @classmethod - def wrap(self, request): - req = HTTPRequest( - form_in=request.form_in, - method=request.method, - scheme=request.scheme, - host=request.host, - port=request.port, - path=request.path, - httpversion=request.httpversion, - headers=request.headers, - body=request.body, - timestamp_start=request.timestamp_start, - timestamp_end=request.timestamp_end, - form_out=(request.form_out if hasattr(request, 'form_out') else None), - ) - if hasattr(request, 'stream_id'): - req.stream_id = request.stream_id - return req - - def __hash__(self): - return id(self) - - def replace(self, pattern, repl, *args, **kwargs): - """ - Replaces a regular expression pattern with repl in the headers, the - request path and the body of the request. Encoded content will be - decoded before replacement, and re-encoded afterwards. - - Returns the number of replacements made. - """ - c = MessageMixin.replace(self, pattern, repl, *args, **kwargs) - self.path, pc = utils.safe_subn( - pattern, repl, self.path, *args, **kwargs - ) - c += pc - return c - - -class HTTPResponse(MessageMixin, semantics.Response): - """ - An HTTP response. - - Exposes the following attributes: - - httpversion: HTTP version tuple, e.g. (1, 0), (1, 1), or (2, 0) - - status_code: HTTP response status code - - msg: HTTP response message - - headers: ODict Caseless object - - content: Content of the request, None, or CONTENT_MISSING if there - is content associated, but not present. CONTENT_MISSING evaluates - to False to make checking for the presence of content natural. - - timestamp_start: Timestamp indicating when request transmission started - - timestamp_end: Timestamp indicating when request transmission ended - """ - - def __init__( - self, - httpversion, - status_code, - msg, - headers, - body, - timestamp_start=None, - timestamp_end=None, - ): - semantics.Response.__init__( - self, - httpversion, - status_code, - msg, - headers, - body, - timestamp_start=timestamp_start, - timestamp_end=timestamp_end, - ) - - # Is this request replayed? - self.is_replay = False - self.stream = False - - _stateobject_attributes = MessageMixin._stateobject_attributes.copy() - _stateobject_attributes.update( - status_code=int, - msg=str - ) - - @classmethod - def from_state(cls, state): - f = cls(None, None, None, None, None) - f.load_state(state) - return f - - @classmethod - def from_protocol( - self, - protocol, - *args, - **kwargs - ): - resp = protocol.read_response(*args, **kwargs) - return self.wrap(resp) - - @classmethod - def wrap(self, response): - resp = HTTPResponse( - httpversion=response.httpversion, - status_code=response.status_code, - msg=response.msg, - headers=response.headers, - body=response.body, - timestamp_start=response.timestamp_start, - timestamp_end=response.timestamp_end, - ) - if hasattr(response, 'stream_id'): - resp.stream_id = response.stream_id - return resp - - def _refresh_cookie(self, c, delta): - """ - Takes a cookie string c and a time delta in seconds, and returns - a refreshed cookie string. - """ - c = Cookie.SimpleCookie(str(c)) - for i in c.values(): - if "expires" in i: - d = parsedate_tz(i["expires"]) - if d: - d = mktime_tz(d) + delta - i["expires"] = formatdate(d) - else: - # This can happen when the expires tag is invalid. - # reddit.com sends a an expires tag like this: "Thu, 31 Dec - # 2037 23:59:59 GMT", which is valid RFC 1123, but not - # strictly correct according to the cookie spec. Browsers - # appear to parse this tolerantly - maybe we should too. - # For now, we just ignore this. - del i["expires"] - return c.output(header="").strip() - - def refresh(self, now=None): - """ - This fairly complex and heuristic function refreshes a server - response for replay. - - - It adjusts date, expires and last-modified headers. - - It adjusts cookie expiration. - """ - if not now: - now = time.time() - delta = now - self.timestamp_start - refresh_headers = [ - "date", - "expires", - "last-modified", - ] - for i in refresh_headers: - if i in self.headers: - d = parsedate_tz(self.headers[i][0]) - if d: - new = mktime_tz(d) + delta - self.headers[i] = [formatdate(new)] - c = [] - for i in self.headers["set-cookie"]: - c.append(self._refresh_cookie(i, delta)) - if c: - self.headers["set-cookie"] = c diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py deleted file mode 100644 index c663f0c5..00000000 --- a/libmproxy/protocol/primitives.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import absolute_import -import copy -import uuid - -from .. import stateobject, utils, version -from ..proxy.connection import ClientConnection, ServerConnection - -KILL = 0 # const for killed requests - - -class Error(stateobject.StateObject): - """ - An Error. - - This is distinct from an protocol error response (say, a HTTP code 500), - which is represented by a normal HTTPResponse object. This class is - responsible for indicating errors that fall outside of normal protocol - communications, like interrupted connections, timeouts, protocol errors. - - Exposes the following attributes: - - flow: Flow object - msg: Message describing the error - timestamp: Seconds since the epoch - """ - - def __init__(self, msg, timestamp=None): - """ - @type msg: str - @type timestamp: float - """ - self.flow = None # will usually be set by the flow backref mixin - self.msg = msg - self.timestamp = timestamp or utils.timestamp() - - _stateobject_attributes = dict( - msg=str, - timestamp=float - ) - - def __str__(self): - return self.msg - - @classmethod - def from_state(cls, state): - # the default implementation assumes an empty constructor. Override - # accordingly. - f = cls(None) - f.load_state(state) - return f - - def copy(self): - c = copy.copy(self) - return c - - -class Flow(stateobject.StateObject): - """ - A Flow is a collection of objects representing a single transaction. - This class is usually subclassed for each protocol, e.g. HTTPFlow. - """ - - def __init__(self, type, client_conn, server_conn, live=None): - self.type = type - self.id = str(uuid.uuid4()) - self.client_conn = client_conn - """@type: ClientConnection""" - self.server_conn = server_conn - """@type: ServerConnection""" - self.live = live - """@type: LiveConnection""" - - self.error = None - """@type: Error""" - self.intercepted = False - """@type: bool""" - self._backup = None - self.reply = None - - _stateobject_attributes = dict( - id=str, - error=Error, - client_conn=ClientConnection, - server_conn=ServerConnection, - type=str, - intercepted=bool - ) - - def get_state(self, short=False): - d = super(Flow, self).get_state(short) - d.update(version=version.IVERSION) - if self._backup and self._backup != d: - if short: - d.update(modified=True) - else: - d.update(backup=self._backup) - return d - - def __eq__(self, other): - return self is other - - def copy(self): - f = copy.copy(self) - - f.id = str(uuid.uuid4()) - f.live = False - f.client_conn = self.client_conn.copy() - f.server_conn = self.server_conn.copy() - - if self.error: - f.error = self.error.copy() - return f - - def modified(self): - """ - Has this Flow been modified? - """ - if self._backup: - return self._backup != self.get_state() - else: - return False - - def backup(self, force=False): - """ - Save a backup of this Flow, which can be reverted to using a - call to .revert(). - """ - if not self._backup: - self._backup = self.get_state() - - def revert(self): - """ - Revert to the last backed up state. - """ - if self._backup: - self.load_state(self._backup) - self._backup = None - - def kill(self, master): - """ - Kill this request. - """ - self.error = Error("Connection killed") - self.intercepted = False - self.reply(KILL) - master.handle_error(self) - - def intercept(self, master): - """ - Intercept this Flow. Processing will stop until accept_intercept is - called. - """ - if self.intercepted: - return - self.intercepted = True - master.handle_intercept(self) - - def accept_intercept(self, master): - """ - Continue with the flow - called after an intercept(). - """ - if not self.intercepted: - return - self.intercepted = False - self.reply() - master.handle_accept_intercept(self) diff --git a/libmproxy/protocol/rawtcp.py b/libmproxy/protocol/rawtcp.py new file mode 100644 index 00000000..86468773 --- /dev/null +++ b/libmproxy/protocol/rawtcp.py @@ -0,0 +1,66 @@ +from __future__ import (absolute_import, print_function, division) +import socket +import select + +from OpenSSL import SSL + +from netlib.tcp import NetLibError +from netlib.utils import cleanBin +from ..exceptions import ProtocolException +from .base import Layer + + +class RawTCPLayer(Layer): + chunk_size = 4096 + + def __init__(self, ctx, logging=True): + self.logging = logging + super(RawTCPLayer, self).__init__(ctx) + + def __call__(self): + self.connect() + + buf = memoryview(bytearray(self.chunk_size)) + + client = self.client_conn.connection + server = self.server_conn.connection + conns = [client, server] + + try: + while True: + r, _, _ = select.select(conns, [], [], 10) + for conn in r: + dst = server if conn == client else client + + size = conn.recv_into(buf, self.chunk_size) + if not size: + conns.remove(conn) + # Shutdown connection to the other peer + if isinstance(conn, SSL.Connection): + # We can't half-close a connection, so we just close everything here. + # Sockets will be cleaned up on a higher level. + return + else: + dst.shutdown(socket.SHUT_WR) + + if len(conns) == 0: + return + continue + + dst.sendall(buf[:size]) + + if self.logging: + # log messages are prepended with the client address, + # hence the "weird" direction string. + if dst == server: + direction = "-> tcp -> {}".format(repr(self.server_conn.address)) + else: + direction = "<- tcp <- {}".format(repr(self.server_conn.address)) + data = cleanBin(buf[:size].tobytes()) + self.log( + "{}\r\n{}".format(direction, data), + "info" + ) + + except (socket.error, NetLibError, SSL.Error) as e: + raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py new file mode 100644 index 00000000..b85a6595 --- /dev/null +++ b/libmproxy/protocol/tls.py @@ -0,0 +1,288 @@ +from __future__ import (absolute_import, print_function, division) + +import struct + +from construct import ConstructError + +from netlib.tcp import NetLibError, NetLibInvalidCertificateError +from netlib.http.http1 import HTTP1Protocol +from ..contrib.tls._constructs import ClientHello +from ..exceptions import ProtocolException +from .base import Layer + + +def is_tls_record_magic(d): + """ + Returns: + True, if the passed bytes start with the TLS record magic bytes. + False, otherwise. + """ + d = d[:3] + + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + return ( + len(d) == 3 and + d[0] == '\x16' and + d[1] == '\x03' and + d[2] in ('\x00', '\x01', '\x02', '\x03') + ) + + +class TlsLayer(Layer): + def __init__(self, ctx, client_tls, server_tls): + self.client_sni = None + self.client_alpn_protocols = None + + super(TlsLayer, self).__init__(ctx) + self._client_tls = client_tls + self._server_tls = server_tls + + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values. + + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + + client_tls_requires_server_cert = ( + self._client_tls and self._server_tls and not self.config.no_upstream_cert + ) + + if self._client_tls: + self._parse_client_hello() + + if client_tls_requires_server_cert: + self._establish_tls_with_client_and_server() + elif self._client_tls: + self._establish_tls_with_client() + + layer = self.ctx.next_layer(self) + layer() + + def __repr__(self): + if self._client_tls and self._server_tls: + return "TlsLayer(client and server)" + elif self._client_tls: + return "TlsLayer(client)" + elif self._server_tls: + return "TlsLayer(server)" + else: + return "TlsLayer(inactive)" + + def _get_client_hello(self): + """ + Peek into the socket and read all records that contain the initial client hello message. + + Returns: + The raw handshake packet bytes, without TLS record header(s). + """ + client_hello = "" + client_hello_size = 1 + offset = 0 + while len(client_hello) < client_hello_size: + record_header = self.client_conn.rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) != 5: + raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) + record_size = struct.unpack("!H", record_header[3:])[0] + 5 + record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) + client_hello += record_body + offset += record_size + client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 + return client_hello + + def _parse_client_hello(self): + """ + Peek into the connection, read the initial client hello and parse it to obtain ALPN values. + """ + try: + raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + except ProtocolException as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + return + + try: + client_hello = ClientHello.parse(raw_client_hello) + except ConstructError as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") + return + + for extension in client_hello.extensions: + if extension.type == 0x00: + if len(extension.server_names) != 1 or extension.server_names[0].type != 0: + self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") + self.client_sni = extension.server_names[0].name + elif extension.type == 0x10: + self.client_alpn_protocols = list(extension.alpn_protocols) + + self.log( + "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), + "debug" + ) + + def connect(self): + if not self.server_conn: + self.ctx.connect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def reconnect(self): + self.ctx.reconnect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + self.ctx.set_server(address, server_tls, sni, depth) + if depth == 1 and server_tls is not None: + self._sni_from_server_change = sni + self._server_tls = server_tls + + @property + def sni_for_server_connection(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self.client_sni + + @property + def alpn_for_client_connection(self): + return self.server_conn.get_alpn_proto_negotiated() + + def __alpn_select_callback(self, conn_, options): + """ + Once the client signals the alternate protocols it supports, + we reconnect upstream with the same list and pass the server's choice down to the client. + """ + + # This gets triggered if we haven't established an upstream connection yet. + default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 + # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 + + if self.alpn_for_client_connection in options: + choice = bytes(self.alpn_for_client_connection) + elif default_alpn in options: + choice = bytes(default_alpn) + else: + choice = options[0] + self.log("ALPN for client: %s" % choice, "debug") + return choice + + def _establish_tls_with_client_and_server(self): + self.ctx.connect() + + # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless + # to send an error message over TLS. + try: + self._establish_tls_with_server() + except Exception as e: + try: + self._establish_tls_with_client() + except: + pass + raise e + + self._establish_tls_with_client() + + def _establish_tls_with_client(self): + self.log("Establish TLS with client", "debug") + cert, key, chain_file = self._find_cert() + + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file, + alpn_select_callback=self.__alpn_select_callback, + ) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) + + def _establish_tls_with_server(self): + self.log("Establish TLS with server", "debug") + try: + # We only support http/1.1 and h2. + # If the server only supports spdy (next to http/1.1), it may select that + # and mitmproxy would enter TCP passthrough mode, which we want to avoid. + deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") + if self.client_alpn_protocols: + alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + else: + alpn = None + + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni_for_server_connection, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + alpn_protos=alpn, + ) + tls_cert_err = self.server_conn.ssl_verification_error + if tls_cert_err is not None: + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except NetLibInvalidCertificateError as e: + tls_cert_err = self.server_conn.ssl_verification_error + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + + self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") + + def _find_cert(self): + host = self.server_conn.address.host + sans = set() + # Incorporate upstream certificate + use_upstream_cert = ( + self.server_conn and + self.server_conn.tls_established and + (not self.config.no_upstream_cert) + ) + if use_upstream_cert: + upstream_cert = self.server_conn.cert + sans.update(upstream_cert.altnames) + if upstream_cert.cn: + sans.add(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self.client_sni: + sans.add(self.client_sni) + if self._sni_from_server_change: + sans.add(self._sni_from_server_change) + + sans.discard(host) + return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py deleted file mode 100644 index 61b9a77e..00000000 --- a/libmproxy/protocol2/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import (absolute_import, print_function, division) -from .root_context import RootContext -from .socks_proxy import Socks5Proxy -from .reverse_proxy import ReverseProxy -from .http_proxy import HttpProxy, HttpUpstreamProxy -from .transparent_proxy import TransparentProxy -from .http import make_error_response - -__all__ = [ - "RootContext", - "Socks5Proxy", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "TransparentProxy", - "make_error_response" -] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py deleted file mode 100644 index a508ae8b..00000000 --- a/libmproxy/protocol2/http.py +++ /dev/null @@ -1,588 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from netlib import tcp -from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError -from netlib.http.semantics import CONTENT_MISSING -from netlib import odict -from netlib.tcp import NetLibError, Address -from netlib.http.http1 import HTTP1Protocol -from netlib.http.http2 import HTTP2Protocol - -from .. import version, utils -from ..exceptions import InvalidCredentials, HttpException, ProtocolException -from .layer import Layer -from ..proxy import Kill -from libmproxy.protocol import KILL, Error -from libmproxy.protocol.http import HTTPFlow -from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest - - -class _HttpLayer(Layer): - supports_streaming = False - - def read_request(self): - raise NotImplementedError() - - def send_request(self, request): - raise NotImplementedError() - - def read_response(self, request_method): - raise NotImplementedError() - - def send_response(self, response): - raise NotImplementedError() - - -class _StreamingHttpLayer(_HttpLayer): - supports_streaming = True - - def read_response_headers(self): - raise NotImplementedError - - def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): - raise NotImplementedError() - yield "this is a generator" - - def send_response_headers(self, response): - raise NotImplementedError - - def send_response_body(self, response, chunks): - raise NotImplementedError() - - -class Http1Layer(_StreamingHttpLayer): - def __init__(self, ctx, mode): - super(Http1Layer, self).__init__(ctx) - self.mode = mode - self.client_protocol = HTTP1Protocol(self.client_conn) - self.server_protocol = HTTP1Protocol(self.server_conn) - - def read_request(self): - return HTTPRequest.from_protocol( - self.client_protocol, - body_size_limit=self.config.body_size_limit - ) - - def send_request(self, request): - self.server_conn.send(self.server_protocol.assemble(request)) - - def read_response(self, request_method): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=request_method, - body_size_limit=self.config.body_size_limit, - include_body=True - ) - - def send_response(self, response): - self.client_conn.send(self.client_protocol.assemble(response)) - - def read_response_headers(self): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=None, # does not matter if we don't read the body. - body_size_limit=self.config.body_size_limit, - include_body=False - ) - - def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): - return self.server_protocol.read_http_body_chunked( - headers, - self.config.body_size_limit, - request_method, - response_code, - False, - max_chunk_size - ) - - def send_response_headers(self, response): - h = self.client_protocol._assemble_response_first_line(response) - self.client_conn.wfile.write(h + "\r\n") - h = self.client_protocol._assemble_response_headers( - response, - preserve_transfer_encoding=True - ) - self.client_conn.send(h + "\r\n") - - def send_response_body(self, response, chunks): - if self.client_protocol.has_chunked_encoding(response.headers): - chunks = ( - "%d\r\n%s\r\n" % (len(chunk), chunk) - for chunk in chunks - ) - for chunk in chunks: - self.client_conn.send(chunk) - - def connect(self): - self.ctx.connect() - self.server_protocol = HTTP1Protocol(self.server_conn) - - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP1Protocol(self.server_conn) - - def set_server(self, *args, **kwargs): - self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP1Protocol(self.server_conn) - - def __call__(self): - layer = HttpLayer(self, self.mode) - layer() - - -# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. -class Http2Layer(_HttpLayer): - def __init__(self, ctx, mode): - super(Http2Layer, self).__init__(ctx) - self.mode = mode - self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - - def read_request(self): - request = HTTPRequest.from_protocol( - self.client_protocol, - body_size_limit=self.config.body_size_limit - ) - self._stream_id = request.stream_id - return request - - def send_request(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames - self.server_conn.send(self.server_protocol.assemble(message)) - - def read_response(self, request_method): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=request_method, - body_size_limit=self.config.body_size_limit, - include_body=True, - stream_id=self._stream_id - ) - - def send_response(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames - self.client_conn.send(self.client_protocol.assemble(message)) - - def connect(self): - self.ctx.connect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def set_server(self, *args, **kwargs): - self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def __call__(self): - self.server_protocol.perform_connection_preface() - layer = HttpLayer(self, self.mode) - layer() - - def handle_unexpected_frame(self, frm): - self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") - - -def make_error_response(status_code, message, headers=None): - response = status_codes.RESPONSES.get(status_code, "Unknown") - body = """ - - - %d %s - - %s - - """.strip() % (status_code, response, message) - - if not headers: - headers = odict.ODictCaseless() - headers["Server"] = [version.NAMEVERSION] - headers["Connection"] = ["close"] - headers["Content-Length"] = [len(body)] - headers["Content-Type"] = ["text/html"] - - return HTTPResponse( - (1, 1), # FIXME: Should be a string. - status_code, - response, - headers, - body, - ) - - -def make_connect_request(address): - address = Address.wrap(address) - return HTTPRequest( - "authority", "CONNECT", None, address.host, address.port, None, (1, 1), - odict.ODictCaseless(), "" - ) - - -def make_connect_response(httpversion): - headers = odict.ODictCaseless([ - ["Content-Length", "0"], - ["Proxy-Agent", version.NAMEVERSION] - ]) - return HTTPResponse( - httpversion, - 200, - "Connection established", - headers, - "", - ) - - -class ConnectServerConnection(object): - """ - "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. - """ - - def __init__(self, address, ctx): - self.address = tcp.Address.wrap(address) - self._ctx = ctx - - @property - def via(self): - return self._ctx.server_conn - - def __getattr__(self, item): - return getattr(self.via, item) - - -class UpstreamConnectLayer(Layer): - def __init__(self, ctx, connect_request): - super(UpstreamConnectLayer, self).__init__(ctx) - self.connect_request = connect_request - self.server_conn = ConnectServerConnection( - (connect_request.host, connect_request.port), - self.ctx - ) - - def __call__(self): - layer = self.ctx.next_layer(self) - layer() - - def connect(self): - if not self.server_conn: - self.ctx.connect() - self.send_request(self.connect_request) - else: - pass # swallow the message - - def reconnect(self): - self.ctx.reconnect() - self.send_request(self.connect_request) - resp = self.read_response("CONNECT") - if resp.code != 200: - raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") - - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1: - if self.ctx.server_conn: - self.ctx.reconnect() - address = Address.wrap(address) - self.connect_request.host = address.host - self.connect_request.port = address.port - self.server_conn.address = address - else: - self.ctx.set_server(address, server_tls, sni, depth - 1) - - -class HttpLayer(Layer): - def __init__(self, ctx, mode): - super(HttpLayer, self).__init__(ctx) - self.mode = mode - self.__original_server_conn = None - "Contains the original destination in transparent mode, which needs to be restored" - "if an inline script modified the target server for a single http request" - - def __call__(self): - if self.mode == "transparent": - self.__original_server_conn = self.server_conn - while True: - try: - flow = HTTPFlow(self.client_conn, self.server_conn, live=self) - - try: - request = self.read_request() - except tcp.NetLibError: - # don't throw an error for disconnects that happen - # before/between requests. - return - - self.log("request", "debug", [repr(request)]) - - # Handle Proxy Authentication - self.authenticate(request) - - # Regular Proxy Mode: Handle CONNECT - if self.mode == "regular" and request.form_in == "authority": - self.handle_regular_mode_connect(request) - return - - # Make sure that the incoming request matches our expectations - self.validate_request(request) - - flow.request = request - self.process_request_hook(flow) - - if not flow.response: - self.establish_server_connection(flow) - self.get_response_from_server(flow) - - self.send_response_to_client(flow) - - if self.check_close_connection(flow): - return - - # TODO: Implement HTTP Upgrade - - # Upstream Proxy Mode: Handle CONNECT - if flow.request.form_in == "authority" and flow.response.code == 200: - self.handle_upstream_mode_connect(flow.request.copy()) - return - - except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: - if flow.request and not flow.response: - flow.error = Error(repr(e)) - self.channel.ask("error", flow) - try: - self.send_response(make_error_response( - getattr(e, "code", 502), - repr(e) - )) - except NetLibError: - pass - if isinstance(e, ProtocolException): - raise e - else: - raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) - finally: - flow.live = False - - def handle_regular_mode_connect(self, request): - self.set_server((request.host, request.port)) - self.send_response(make_connect_response(request.httpversion)) - layer = self.ctx.next_layer(self) - layer() - - def handle_upstream_mode_connect(self, connect_request): - layer = UpstreamConnectLayer(self, connect_request) - layer() - - def check_close_connection(self, flow): - """ - Checks if the connection should be closed depending on the HTTP - semantics. Returns True, if so. - """ - - # TODO: add logic for HTTP/2 - - close_connection = ( - http1.HTTP1Protocol.connection_close( - flow.request.httpversion, - flow.request.headers - ) or http1.HTTP1Protocol.connection_close( - flow.response.httpversion, - flow.response.headers - ) or http1.HTTP1Protocol.expected_http_body_size( - flow.response.headers, - False, - flow.request.method, - flow.response.code) == -1 - ) - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 - # and no Content-Length header - - return False - return close_connection - - def send_response_to_client(self, flow): - if not (self.supports_streaming and flow.response.stream): - # no streaming: - # we already received the full response from the server and can - # send it to the client straight away. - self.send_response(flow.response) - else: - # streaming: - # First send the headers and then transfer the response incrementally - self.send_response_headers(flow.response) - chunks = self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code, - max_chunk_size=4096 - ) - if callable(flow.response.stream): - chunks = flow.response.stream(chunks) - self.send_response_body(flow.response, chunks) - flow.response.timestamp_end = utils.timestamp() - - def get_response_from_server(self, flow): - def get_response(): - self.send_request(flow.request) - if self.supports_streaming: - flow.response = self.read_response_headers() - else: - flow.response = self.read_response() - - try: - get_response() - except (tcp.NetLibError, HttpErrorConnClosed) as v: - self.log( - "server communication error: %s" % repr(v), - level="debug" - ) - # In any case, we try to reconnect at least once. This is - # necessary because it might be possible that we already - # initiated an upstream connection after clientconnect that - # has already been expired, e.g consider the following event - # log: - # > clientconnect (transparent mode destination known) - # > serverconnect (required for client tls handshake) - # > read n% of large request - # > server detects timeout, disconnects - # > read (100-n)% of large request - # > send large request upstream - self.reconnect() - get_response() - - # call the appropriate script hook - this is an opportunity for an - # inline script to set flow.stream = True - flow = self.channel.ask("responseheaders", flow) - if flow is None or flow == KILL: - raise Kill() - - if self.supports_streaming: - if flow.response.stream: - flow.response.content = CONTENT_MISSING - else: - flow.response.content = "".join(self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code - )) - flow.response.timestamp_end = utils.timestamp() - - # no further manipulation of self.server_conn beyond this point - # we can safely set it as the final attribute value here. - flow.server_conn = self.server_conn - - self.log( - "response", - "debug", - [repr(flow.response)] - ) - response_reply = self.channel.ask("response", flow) - if response_reply is None or response_reply == KILL: - raise Kill() - - def process_request_hook(self, flow): - # Determine .scheme, .host and .port attributes for inline scripts. - # For absolute-form requests, they are directly given in the request. - # For authority-form requests, we only need to determine the request scheme. - # For relative-form requests, we need to determine host and port as - # well. - if self.mode == "regular": - pass # only absolute-form at this point, nothing to do here. - elif self.mode == "upstream": - if flow.request.form_in == "authority": - flow.request.scheme = "http" # pseudo value - else: - flow.request.host = self.__original_server_conn.address.host - flow.request.port = self.__original_server_conn.address.port - flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" - - request_reply = self.channel.ask("request", flow) - if request_reply is None or request_reply == KILL: - raise Kill() - if isinstance(request_reply, HTTPResponse): - flow.response = request_reply - return - - def establish_server_connection(self, flow): - address = tcp.Address((flow.request.host, flow.request.port)) - tls = (flow.request.scheme == "https") - - if self.mode == "regular" or self.mode == "transparent": - # If there's an existing connection that doesn't match our expectations, kill it. - if address != self.server_conn.address or tls != self.server_conn.ssl_established: - self.set_server(address, tls, address.host) - # Establish connection is neccessary. - if not self.server_conn: - self.connect() - - # SetServer is not guaranteed to work with TLS: - # If there's not TlsLayer below which could catch the exception, - # TLS will not be established. - if tls and not self.server_conn.tls_established: - raise ProtocolException( - "Cannot upgrade to SSL, no TLS layer on the protocol stack.") - else: - if not self.server_conn: - self.connect() - if tls: - raise HttpException("Cannot change scheme in upstream proxy mode.") - """ - # This is a very ugly (untested) workaround to solve a very ugly problem. - if self.server_conn and self.server_conn.tls_established and not ssl: - self.reconnect() - elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: - if self.server_conn.tls_established: - self.reconnect() - - self.send_request(make_connect_request(address)) - tls_layer = TlsLayer(self, False, True) - tls_layer._establish_tls_with_server() - """ - - def validate_request(self, request): - if request.form_in == "absolute" and request.scheme != "http": - self.send_response( - make_error_response(400, "Invalid request scheme: %s" % request.scheme)) - raise HttpException("Invalid request scheme: %s" % request.scheme) - - expected_request_forms = { - "regular": ("absolute",), # an authority request would already be handled. - "upstream": ("authority", "absolute"), - "transparent": ("relative",) - } - - allowed_request_forms = expected_request_forms[self.mode] - if request.form_in not in allowed_request_forms: - err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( - " or ".join(allowed_request_forms), request.form_in - ) - self.send_response(make_error_response(400, err_message)) - raise HttpException(err_message) - - if self.mode == "regular": - request.form_out = "relative" - - def authenticate(self, request): - if self.config.authenticator: - if self.config.authenticator.authenticate(request.headers): - self.config.authenticator.clean(request.headers) - else: - self.send_response(make_error_response( - 407, - "Proxy Authentication Required", - odict.ODictCaseless( - [ - [k, v] for k, v in - self.config.authenticator.auth_challenge_headers().items() - ]) - )) - raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py deleted file mode 100644 index 2876c022..00000000 --- a/libmproxy/protocol2/http_proxy.py +++ /dev/null @@ -1,26 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from .layer import Layer, ServerConnectionMixin - - -class HttpProxy(Layer, ServerConnectionMixin): - def __call__(self): - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() - - -class HttpUpstreamProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx, server_address): - super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) - - def __call__(self): - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() diff --git a/libmproxy/protocol2/http_replay.py b/libmproxy/protocol2/http_replay.py deleted file mode 100644 index 872ef9cd..00000000 --- a/libmproxy/protocol2/http_replay.py +++ /dev/null @@ -1,95 +0,0 @@ -import threading -from netlib.http import HttpError -from netlib.http.http1 import HTTP1Protocol -from netlib.tcp import NetLibError - -from ..controller import Channel -from ..protocol import KILL, Error -from ..protocol.http_wrappers import HTTPResponse -from ..proxy import Log, Kill -from ..proxy.connection import ServerConnection -from .http import make_connect_request - - -class RequestReplayThread(threading.Thread): - name = "RequestReplayThread" - - def __init__(self, config, flow, masterq, should_exit): - """ - masterqueue can be a queue or None, if no scripthooks should be - processed. - """ - self.config, self.flow = config, flow - if masterq: - self.channel = Channel(masterq, should_exit) - else: - self.channel = None - super(RequestReplayThread, self).__init__() - - def run(self): - r = self.flow.request - form_out_backup = r.form_out - try: - self.flow.response = None - - # If we have a channel, run script hooks. - if self.channel: - request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == KILL: - raise Kill() - elif isinstance(request_reply, HTTPResponse): - self.flow.response = request_reply - - if not self.flow.response: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - server_address = self.config.upstream_server.address - server = ServerConnection(server_address) - server.connect() - protocol = HTTP1Protocol(server) - if r.scheme == "https": - connect_request = make_connect_request((r.host, r.port)) - server.send(protocol.assemble(connect_request)) - resp = protocol.read_response("CONNECT") - if resp.code != 200: - raise HttpError(502, "Upstream server refuses CONNECT request") - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - else: - r.form_out = "absolute" - else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - protocol = HTTP1Protocol(server) - if r.scheme == "https": - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - - server.send(protocol.assemble(r)) - self.flow.server_conn = server - self.flow.response = HTTPResponse.from_protocol( - protocol, - r.method, - body_size_limit=self.config.body_size_limit, - ) - if self.channel: - response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == KILL: - raise Kill() - except (HttpError, NetLibError) as v: - self.flow.error = Error(repr(v)) - if self.channel: - self.channel.ask("error", self.flow) - except Kill: - # KillSignal should only be raised if there's a channel in the - # first place. - self.channel.tell("log", Log("Connection killed", "info")) - finally: - r.form_out = form_out_backup diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py deleted file mode 100644 index 2b47cc26..00000000 --- a/libmproxy/protocol2/layer.py +++ /dev/null @@ -1,138 +0,0 @@ -""" -mitmproxy protocol architecture - -In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. -For example, the following scenarios depict possible scenarios (lowest layer first): - -Transparent HTTP proxy, no SSL: - TransparentModeLayer - HttpLayer - -Regular proxy, CONNECT request with WebSockets over SSL: - RegularModeLayer - HttpLayer - SslLayer - WebsocketLayer (or TcpLayer) - -Automated protocol detection by peeking into the buffer: - TransparentModeLayer - SslLayer - Http2Layer - -Communication between layers is done as follows: - - lower layers provide context information to higher layers - - higher layers can call functions provided by lower layers, - which are propagated until they reach a suitable layer. - -Further goals: - - Connections should always be peekable to make automatic protocol detection work. - - Upstream connections should be established as late as possible; - inline scripts shall have a chance to handle everything locally. -""" -from __future__ import (absolute_import, print_function, division) -from netlib import tcp -from ..proxy import Log -from ..proxy.connection import ServerConnection -from ..exceptions import ProtocolException - - -class _LayerCodeCompletion(object): - """ - Dummy class that provides type hinting in PyCharm, which simplifies development a lot. - """ - - def __init__(self, *args, **kwargs): - super(_LayerCodeCompletion, self).__init__(*args, **kwargs) - if True: - return - self.config = None - """@type: libmproxy.proxy.config.ProxyConfig""" - self.client_conn = None - """@type: libmproxy.proxy.connection.ClientConnection""" - self.channel = None - """@type: libmproxy.controller.Channel""" - - -class Layer(_LayerCodeCompletion): - def __init__(self, ctx, *args, **kwargs): - """ - Args: - ctx: The (read-only) higher layer. - """ - super(Layer, self).__init__(*args, **kwargs) - self.ctx = ctx - - def __call__(self): - """ - Logic of the layer. - Raises: - ProtocolException in case of protocol exceptions. - """ - raise NotImplementedError - - def __getattr__(self, name): - """ - Attributes not present on the current layer may exist on a higher layer. - """ - return getattr(self.ctx, name) - - def log(self, msg, level, subs=()): - full_msg = [ - "{}: {}".format(repr(self.client_conn.address), msg) - ] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - @property - def layers(self): - return [self] + self.ctx.layers - - def __repr__(self): - return type(self).__name__ - - -class ServerConnectionMixin(object): - """ - Mixin that provides a layer with the capabilities to manage a server connection. - """ - - def __init__(self, server_address=None): - super(ServerConnectionMixin, self).__init__() - self.server_conn = ServerConnection(server_address) - - def reconnect(self): - address = self.server_conn.address - self._disconnect() - self.server_conn.address = address - self.connect() - - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1: - if self.server_conn: - self._disconnect() - self.log("Set new server address: " + repr(address), "debug") - self.server_conn.address = address - else: - self.ctx.set_server(address, server_tls, sni, depth - 1) - - def _disconnect(self): - """ - Deletes (and closes) an existing server connection. - """ - self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) - self.server_conn.finish() - self.server_conn.close() - # self.channel.tell("serverdisconnect", self) - self.server_conn = ServerConnection(None) - - def connect(self): - if not self.server_conn.address: - raise ProtocolException("Cannot connect to server, no server address given.") - self.log("serverconnect", "debug", [repr(self.server_conn.address)]) - try: - self.server_conn.connect() - except tcp.NetLibError as e: - raise ProtocolException( - "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py deleted file mode 100644 index b10217f1..00000000 --- a/libmproxy/protocol2/rawtcp.py +++ /dev/null @@ -1,66 +0,0 @@ -from __future__ import (absolute_import, print_function, division) -import socket -import select - -from OpenSSL import SSL - -from netlib.tcp import NetLibError -from netlib.utils import cleanBin -from ..exceptions import ProtocolException -from .layer import Layer - - -class RawTcpLayer(Layer): - chunk_size = 4096 - - def __init__(self, ctx, logging=True): - self.logging = logging - super(RawTcpLayer, self).__init__(ctx) - - def __call__(self): - self.connect() - - buf = memoryview(bytearray(self.chunk_size)) - - client = self.client_conn.connection - server = self.server_conn.connection - conns = [client, server] - - try: - while True: - r, _, _ = select.select(conns, [], [], 10) - for conn in r: - dst = server if conn == client else client - - size = conn.recv_into(buf, self.chunk_size) - if not size: - conns.remove(conn) - # Shutdown connection to the other peer - if isinstance(conn, SSL.Connection): - # We can't half-close a connection, so we just close everything here. - # Sockets will be cleaned up on a higher level. - return - else: - dst.shutdown(socket.SHUT_WR) - - if len(conns) == 0: - return - continue - - dst.sendall(buf[:size]) - - if self.logging: - # log messages are prepended with the client address, - # hence the "weird" direction string. - if dst == server: - direction = "-> tcp -> {}".format(repr(self.server_conn.address)) - else: - direction = "<- tcp <- {}".format(repr(self.server_conn.address)) - data = cleanBin(buf[:size].tobytes()) - self.log( - "{}\r\n{}".format(direction, data), - "info" - ) - - except (socket.error, NetLibError, SSL.Error) as e: - raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py deleted file mode 100644 index 3ca998d5..00000000 --- a/libmproxy/protocol2/reverse_proxy.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from .layer import Layer, ServerConnectionMixin - - -class ReverseProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx, server_address, server_tls): - super(ReverseProxy, self).__init__(ctx, server_address=server_address) - self.server_tls = server_tls - - def __call__(self): - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py deleted file mode 100644 index daea54bd..00000000 --- a/libmproxy/protocol2/root_context.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from netlib.http.http1 import HTTP1Protocol -from netlib.http.http2 import HTTP2Protocol - -from .rawtcp import RawTcpLayer -from .tls import TlsLayer, is_tls_record_magic -from .http import Http1Layer, Http2Layer -from .layer import ServerConnectionMixin -from .http_proxy import HttpProxy, HttpUpstreamProxy -from .reverse_proxy import ReverseProxy - - -class RootContext(object): - """ - The outmost context provided to the root layer. - As a consequence, every layer has .client_conn, .channel, .next_layer() and .config. - """ - - def __init__(self, client_conn, config, channel): - self.client_conn = client_conn # Client Connection - self.channel = channel # provides .ask() method to communicate with FlowMaster - self.config = config # Proxy Configuration - - def next_layer(self, top_layer): - """ - This function determines the next layer in the protocol stack. - - Arguments: - top_layer: the current top layer. - - Returns: - The next layer - """ - - # 1. Check for --ignore. - if self.config.check_ignore(top_layer.server_conn.address): - return RawTcpLayer(top_layer, logging=False) - - d = top_layer.client_conn.rfile.peek(3) - client_tls = is_tls_record_magic(d) - - # 2. Always insert a TLS layer, even if there's neither client nor server tls. - # An inline script may upgrade from http to https, - # in which case we need some form of TLS layer. - if isinstance(top_layer, ReverseProxy): - return TlsLayer(top_layer, client_tls, top_layer.server_tls) - if isinstance(top_layer, ServerConnectionMixin): - return TlsLayer(top_layer, client_tls, client_tls) - - # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. - if isinstance(top_layer, TlsLayer): - if isinstance(top_layer.ctx, HttpProxy): - return Http1Layer(top_layer, "regular") - if isinstance(top_layer.ctx, HttpUpstreamProxy): - return Http1Layer(top_layer, "upstream") - - # 4. Check for other TLS cases (e.g. after CONNECT). - if client_tls: - return TlsLayer(top_layer, True, True) - - # 4. Check for --tcp - if self.config.check_tcp(top_layer.server_conn.address): - return RawTcpLayer(top_layer) - - # 5. Check for TLS ALPN (HTTP1/HTTP2) - if isinstance(top_layer, TlsLayer): - alpn = top_layer.client_conn.get_alpn_proto_negotiated() - if alpn == HTTP2Protocol.ALPN_PROTO_H2: - return Http2Layer(top_layer, 'transparent') - if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: - return Http1Layer(top_layer, 'transparent') - - # 6. Assume HTTP1 by default - return Http1Layer(top_layer, 'transparent') - - # In a future version, we want to implement TCP passthrough as the last fallback, - # but we don't have the UI part ready for that. - # - # d = top_layer.client_conn.rfile.peek(3) - # is_ascii = ( - # len(d) == 3 and - # # better be safe here and don't expect uppercase... - # all(x in string.ascii_letters for x in d) - # ) - # # TODO: This could block if there are not enough bytes available? - # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) - # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) - - @property - def layers(self): - return [] - - def __repr__(self): - return "RootContext" diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py deleted file mode 100644 index 525520e8..00000000 --- a/libmproxy/protocol2/socks_proxy.py +++ /dev/null @@ -1,59 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from netlib import socks -from netlib.tcp import NetLibError -from ..exceptions import Socks5Exception -from .layer import Layer, ServerConnectionMixin - - -class Socks5Proxy(Layer, ServerConnectionMixin): - def __call__(self): - try: - # Parse Client Greeting - client_greet = socks.ClientGreeting.from_file(self.client_conn.rfile, fail_early=True) - client_greet.assert_socks5() - if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: - raise socks.SocksError( - socks.METHOD.NO_ACCEPTABLE_METHODS, - "mitmproxy only supports SOCKS without authentication" - ) - - # Send Server Greeting - server_greet = socks.ServerGreeting( - socks.VERSION.SOCKS5, - socks.METHOD.NO_AUTHENTICATION_REQUIRED - ) - server_greet.to_file(self.client_conn.wfile) - self.client_conn.wfile.flush() - - # Parse Connect Request - connect_request = socks.Message.from_file(self.client_conn.rfile) - connect_request.assert_socks5() - if connect_request.msg != socks.CMD.CONNECT: - raise socks.SocksError( - socks.REP.COMMAND_NOT_SUPPORTED, - "mitmproxy only supports SOCKS5 CONNECT." - ) - - # We always connect lazily, but we need to pretend to the client that we connected. - connect_reply = socks.Message( - socks.VERSION.SOCKS5, - socks.REP.SUCCEEDED, - connect_request.atyp, - # dummy value, we don't have an upstream connection yet. - connect_request.addr - ) - connect_reply.to_file(self.client_conn.wfile) - self.client_conn.wfile.flush() - - except (socks.SocksError, NetLibError) as e: - raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e) - - self.server_conn.address = connect_request.addr - - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py deleted file mode 100644 index 73bb12f3..00000000 --- a/libmproxy/protocol2/tls.py +++ /dev/null @@ -1,288 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -import struct - -from construct import ConstructError - -from netlib.tcp import NetLibError, NetLibInvalidCertificateError -from netlib.http.http1 import HTTP1Protocol -from ..contrib.tls._constructs import ClientHello -from ..exceptions import ProtocolException -from .layer import Layer - - -def is_tls_record_magic(d): - """ - Returns: - True, if the passed bytes start with the TLS record magic bytes. - False, otherwise. - """ - d = d[:3] - - # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 - # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello - return ( - len(d) == 3 and - d[0] == '\x16' and - d[1] == '\x03' and - d[2] in ('\x00', '\x01', '\x02', '\x03') - ) - - -class TlsLayer(Layer): - def __init__(self, ctx, client_tls, server_tls): - self.client_sni = None - self.client_alpn_protocols = None - - super(TlsLayer, self).__init__(ctx) - self._client_tls = client_tls - self._server_tls = server_tls - - self._sni_from_server_change = None - - def __call__(self): - """ - The strategy for establishing SSL is as follows: - First, we determine whether we need the server cert to establish ssl with the client. - If so, we first connect to the server and then to the client. - If not, we only connect to the client and do the server_ssl lazily on a Connect message. - - An additional complexity is that establish ssl with the server may require a SNI value from the client. - In an ideal world, we'd do the following: - 1. Start the SSL handshake with the client - 2. Check if the client sends a SNI. - 3. Pause the client handshake, establish SSL with the server. - 4. Finish the client handshake with the certificate from the server. - There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( - Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values. - - Further notes: - - OpenSSL 1.0.2 introduces a callback that would help here: - https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html - - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 - """ - - client_tls_requires_server_cert = ( - self._client_tls and self._server_tls and not self.config.no_upstream_cert - ) - - if self._client_tls: - self._parse_client_hello() - - if client_tls_requires_server_cert: - self._establish_tls_with_client_and_server() - elif self._client_tls: - self._establish_tls_with_client() - - layer = self.ctx.next_layer(self) - layer() - - def __repr__(self): - if self._client_tls and self._server_tls: - return "TlsLayer(client and server)" - elif self._client_tls: - return "TlsLayer(client)" - elif self._server_tls: - return "TlsLayer(server)" - else: - return "TlsLayer(inactive)" - - def _get_client_hello(self): - """ - Peek into the socket and read all records that contain the initial client hello message. - - Returns: - The raw handshake packet bytes, without TLS record header(s). - """ - client_hello = "" - client_hello_size = 1 - offset = 0 - while len(client_hello) < client_hello_size: - record_header = self.client_conn.rfile.peek(offset + 5)[offset:] - if not is_tls_record_magic(record_header) or len(record_header) != 5: - raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) - record_size = struct.unpack("!H", record_header[3:])[0] + 5 - record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] - if len(record_body) != record_size - 5: - raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) - client_hello += record_body - offset += record_size - client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 - return client_hello - - def _parse_client_hello(self): - """ - Peek into the connection, read the initial client hello and parse it to obtain ALPN values. - """ - try: - raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. - except ProtocolException as e: - self.log("Cannot parse Client Hello: %s" % repr(e), "error") - return - - try: - client_hello = ClientHello.parse(raw_client_hello) - except ConstructError as e: - self.log("Cannot parse Client Hello: %s" % repr(e), "error") - self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") - return - - for extension in client_hello.extensions: - if extension.type == 0x00: - if len(extension.server_names) != 1 or extension.server_names[0].type != 0: - self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") - self.client_sni = extension.server_names[0].name - elif extension.type == 0x10: - self.client_alpn_protocols = list(extension.alpn_protocols) - - self.log( - "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), - "debug" - ) - - def connect(self): - if not self.server_conn: - self.ctx.connect() - if self._server_tls and not self.server_conn.tls_established: - self._establish_tls_with_server() - - def reconnect(self): - self.ctx.reconnect() - if self._server_tls and not self.server_conn.tls_established: - self._establish_tls_with_server() - - def set_server(self, address, server_tls=None, sni=None, depth=1): - self.ctx.set_server(address, server_tls, sni, depth) - if depth == 1 and server_tls is not None: - self._sni_from_server_change = sni - self._server_tls = server_tls - - @property - def sni_for_server_connection(self): - if self._sni_from_server_change is False: - return None - else: - return self._sni_from_server_change or self.client_sni - - @property - def alpn_for_client_connection(self): - return self.server_conn.get_alpn_proto_negotiated() - - def __alpn_select_callback(self, conn_, options): - """ - Once the client signals the alternate protocols it supports, - we reconnect upstream with the same list and pass the server's choice down to the client. - """ - - # This gets triggered if we haven't established an upstream connection yet. - default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 - # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 - - if self.alpn_for_client_connection in options: - choice = bytes(self.alpn_for_client_connection) - elif default_alpn in options: - choice = bytes(default_alpn) - else: - choice = options[0] - self.log("ALPN for client: %s" % choice, "debug") - return choice - - def _establish_tls_with_client_and_server(self): - self.ctx.connect() - - # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless - # to send an error message over TLS. - try: - self._establish_tls_with_server() - except Exception as e: - try: - self._establish_tls_with_client() - except: - pass - raise e - - self._establish_tls_with_client() - - def _establish_tls_with_client(self): - self.log("Establish TLS with client", "debug") - cert, key, chain_file = self._find_cert() - - try: - self.client_conn.convert_to_ssl( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file, - alpn_select_callback=self.__alpn_select_callback, - ) - except NetLibError as e: - raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) - - def _establish_tls_with_server(self): - self.log("Establish TLS with server", "debug") - try: - # We only support http/1.1 and h2. - # If the server only supports spdy (next to http/1.1), it may select that - # and mitmproxy would enter TCP passthrough mode, which we want to avoid. - deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") - if self.client_alpn_protocols: - alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) - else: - alpn = None - - self.server_conn.establish_ssl( - self.config.clientcerts, - self.sni_for_server_connection, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify_options=self.config.openssl_verification_mode_server, - ca_path=self.config.openssl_trusted_cadir_server, - ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, - alpn_protos=alpn, - ) - tls_cert_err = self.server_conn.ssl_verification_error - if tls_cert_err is not None: - self.log( - "TLS verification failed for upstream server at depth %s with error: %s" % - (tls_cert_err['depth'], tls_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") - except NetLibInvalidCertificateError as e: - tls_cert_err = self.server_conn.ssl_verification_error - self.log( - "TLS verification failed for upstream server at depth %s with error: %s" % - (tls_cert_err['depth'], tls_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") - raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) - except NetLibError as e: - raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) - - self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") - - def _find_cert(self): - host = self.server_conn.address.host - sans = set() - # Incorporate upstream certificate - use_upstream_cert = ( - self.server_conn and - self.server_conn.tls_established and - (not self.config.no_upstream_cert) - ) - if use_upstream_cert: - upstream_cert = self.server_conn.cert - sans.update(upstream_cert.altnames) - if upstream_cert.cn: - sans.add(host) - host = upstream_cert.cn.decode("utf8").encode("idna") - # Also add SNI values. - if self.client_sni: - sans.add(self.client_sni) - if self._sni_from_server_change: - sans.add(self._sni_from_server_change) - - sans.discard(host) - return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py deleted file mode 100644 index e6ebf115..00000000 --- a/libmproxy/protocol2/transparent_proxy.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from ..exceptions import ProtocolException -from .. import platform -from .layer import Layer, ServerConnectionMixin - - -class TransparentProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx): - super(TransparentProxy, self).__init__(ctx) - self.resolver = platform.resolver() - - def __call__(self): - try: - self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) - except Exception as e: - raise ProtocolException("Transparent mode failure: %s" % repr(e), e) - - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index 709654cb..d5297cb1 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1,11 +1,9 @@ from __future__ import (absolute_import, print_function, division) -from .primitives import Log, Kill +from .server import ProxyServer, DummyServer from .config import ProxyConfig -from .connection import ClientConnection, ServerConnection __all__ = [ - "Log", "Kill", + "ProxyServer", "DummyServer", "ProxyConfig", - "ClientConnection", "ServerConnection" -] \ No newline at end of file +] diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index b360abbd..65029087 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -6,9 +6,9 @@ from OpenSSL import SSL from netlib import certutils, tcp from netlib.http import authentication +from netlib.tcp import Address, sslversion_choices from .. import utils, platform -from netlib.tcp import Address, sslversion_choices CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py deleted file mode 100644 index 94f318f6..00000000 --- a/libmproxy/proxy/connection.py +++ /dev/null @@ -1,193 +0,0 @@ -from __future__ import absolute_import - -import copy -import os - -from netlib import tcp, certutils -from .. import stateobject, utils - - -class ClientConnection(tcp.BaseHandler, stateobject.StateObject): - def __init__(self, client_connection, address, server): - # Eventually, this object is restored from state. We don't have a - # connection then. - if client_connection: - super(ClientConnection, self).__init__(client_connection, address, server) - else: - self.connection = None - self.server = None - self.wfile = None - self.rfile = None - self.address = None - self.clientcert = None - self.ssl_established = None - - self.timestamp_start = utils.timestamp() - self.timestamp_end = None - self.timestamp_ssl_setup = None - self.protocol = None - - def __nonzero__(self): - return bool(self.connection) and not self.finished - - def __repr__(self): - return "".format( - ssl="[ssl] " if self.ssl_established else "", - host=self.address.host, - port=self.address.port - ) - - @property - def tls_established(self): - return self.ssl_established - - _stateobject_attributes = dict( - ssl_established=bool, - timestamp_start=float, - timestamp_end=float, - timestamp_ssl_setup=float - ) - - def get_state(self, short=False): - d = super(ClientConnection, self).get_state(short) - d.update( - address={ - "address": self.address(), - "use_ipv6": self.address.use_ipv6}, - clientcert=self.cert.to_pem() if self.clientcert else None) - return d - - def load_state(self, state): - super(ClientConnection, self).load_state(state) - self.address = tcp.Address( - **state["address"]) if state["address"] else None - self.clientcert = certutils.SSLCert.from_pem( - state["clientcert"]) if state["clientcert"] else None - - def copy(self): - return copy.copy(self) - - def send(self, message): - if isinstance(message, list): - message = b''.join(message) - self.wfile.write(message) - self.wfile.flush() - - @classmethod - def from_state(cls, state): - f = cls(None, tuple(), None) - f.load_state(state) - return f - - def convert_to_ssl(self, *args, **kwargs): - super(ClientConnection, self).convert_to_ssl(*args, **kwargs) - self.timestamp_ssl_setup = utils.timestamp() - - def finish(self): - super(ClientConnection, self).finish() - self.timestamp_end = utils.timestamp() - - -class ServerConnection(tcp.TCPClient, stateobject.StateObject): - def __init__(self, address): - tcp.TCPClient.__init__(self, address) - - self.via = None - self.timestamp_start = None - self.timestamp_end = None - self.timestamp_tcp_setup = None - self.timestamp_ssl_setup = None - self.protocol = None - - def __nonzero__(self): - return bool(self.connection) and not self.finished - - def __repr__(self): - if self.ssl_established and self.sni: - ssl = "[ssl: {0}] ".format(self.sni) - elif self.ssl_established: - ssl = "[ssl] " - else: - ssl = "" - return "".format( - ssl=ssl, - host=self.address.host, - port=self.address.port - ) - - @property - def tls_established(self): - return self.ssl_established - - _stateobject_attributes = dict( - timestamp_start=float, - timestamp_end=float, - timestamp_tcp_setup=float, - timestamp_ssl_setup=float, - address=tcp.Address, - source_address=tcp.Address, - cert=certutils.SSLCert, - ssl_established=bool, - sni=str - ) - _stateobject_long_attributes = {"cert"} - - def get_state(self, short=False): - d = super(ServerConnection, self).get_state(short) - d.update( - address={"address": self.address(), - "use_ipv6": self.address.use_ipv6}, - source_address=({"address": self.source_address(), - "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), - cert=self.cert.to_pem() if self.cert else None - ) - return d - - def load_state(self, state): - super(ServerConnection, self).load_state(state) - - self.address = tcp.Address( - **state["address"]) if state["address"] else None - self.source_address = tcp.Address( - **state["source_address"]) if state["source_address"] else None - self.cert = certutils.SSLCert.from_pem( - state["cert"]) if state["cert"] else None - - @classmethod - def from_state(cls, state): - f = cls(tuple()) - f.load_state(state) - return f - - def copy(self): - return copy.copy(self) - - def connect(self): - self.timestamp_start = utils.timestamp() - tcp.TCPClient.connect(self) - self.timestamp_tcp_setup = utils.timestamp() - - def send(self, message): - if isinstance(message, list): - message = b''.join(message) - self.wfile.write(message) - self.wfile.flush() - - def establish_ssl(self, clientcerts, sni, **kwargs): - clientcert = None - if clientcerts: - path = os.path.join( - clientcerts, - self.address.host.encode("idna")) + ".pem" - if os.path.exists(path): - clientcert = path - - self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) - self.sni = sni - self.timestamp_ssl_setup = utils.timestamp() - - def finish(self): - tcp.TCPClient.finish(self) - self.timestamp_end = utils.timestamp() - -ServerConnection._stateobject_attributes["via"] = ServerConnection diff --git a/libmproxy/proxy/modes/__init__.py b/libmproxy/proxy/modes/__init__.py new file mode 100644 index 00000000..f014ed98 --- /dev/null +++ b/libmproxy/proxy/modes/__init__.py @@ -0,0 +1,12 @@ +from __future__ import (absolute_import, print_function, division) +from .http_proxy import HttpProxy, HttpUpstreamProxy +from .reverse_proxy import ReverseProxy +from .socks_proxy import Socks5Proxy +from .transparent_proxy import TransparentProxy + +__all__ = [ + "HttpProxy", "HttpUpstreamProxy", + "ReverseProxy", + "Socks5Proxy", + "TransparentProxy" +] diff --git a/libmproxy/proxy/modes/http_proxy.py b/libmproxy/proxy/modes/http_proxy.py new file mode 100644 index 00000000..90c54cc6 --- /dev/null +++ b/libmproxy/proxy/modes/http_proxy.py @@ -0,0 +1,26 @@ +from __future__ import (absolute_import, print_function, division) + +from ...protocol import Layer, ServerConnectionMixin + + +class HttpProxy(Layer, ServerConnectionMixin): + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() + + +class HttpUpstreamProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address): + super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) + + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/modes/reverse_proxy.py b/libmproxy/proxy/modes/reverse_proxy.py new file mode 100644 index 00000000..b57ac5eb --- /dev/null +++ b/libmproxy/proxy/modes/reverse_proxy.py @@ -0,0 +1,17 @@ +from __future__ import (absolute_import, print_function, division) + +from ...protocol import Layer, ServerConnectionMixin + + +class ReverseProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address, server_tls): + super(ReverseProxy, self).__init__(ctx, server_address=server_address) + self.server_tls = server_tls + + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/modes/socks_proxy.py b/libmproxy/proxy/modes/socks_proxy.py new file mode 100644 index 00000000..ebaf939e --- /dev/null +++ b/libmproxy/proxy/modes/socks_proxy.py @@ -0,0 +1,60 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib import socks +from netlib.tcp import NetLibError + +from ...exceptions import Socks5Exception +from ...protocol import Layer, ServerConnectionMixin + + +class Socks5Proxy(Layer, ServerConnectionMixin): + def __call__(self): + try: + # Parse Client Greeting + client_greet = socks.ClientGreeting.from_file(self.client_conn.rfile, fail_early=True) + client_greet.assert_socks5() + if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: + raise socks.SocksError( + socks.METHOD.NO_ACCEPTABLE_METHODS, + "mitmproxy only supports SOCKS without authentication" + ) + + # Send Server Greeting + server_greet = socks.ServerGreeting( + socks.VERSION.SOCKS5, + socks.METHOD.NO_AUTHENTICATION_REQUIRED + ) + server_greet.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + # Parse Connect Request + connect_request = socks.Message.from_file(self.client_conn.rfile) + connect_request.assert_socks5() + if connect_request.msg != socks.CMD.CONNECT: + raise socks.SocksError( + socks.REP.COMMAND_NOT_SUPPORTED, + "mitmproxy only supports SOCKS5 CONNECT." + ) + + # We always connect lazily, but we need to pretend to the client that we connected. + connect_reply = socks.Message( + socks.VERSION.SOCKS5, + socks.REP.SUCCEEDED, + connect_request.atyp, + # dummy value, we don't have an upstream connection yet. + connect_request.addr + ) + connect_reply.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + except (socks.SocksError, NetLibError) as e: + raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e) + + self.server_conn.address = connect_request.addr + + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/modes/transparent_proxy.py b/libmproxy/proxy/modes/transparent_proxy.py new file mode 100644 index 00000000..96ad86c4 --- /dev/null +++ b/libmproxy/proxy/modes/transparent_proxy.py @@ -0,0 +1,24 @@ +from __future__ import (absolute_import, print_function, division) + +from ... import platform +from ...exceptions import ProtocolException +from ...protocol import Layer, ServerConnectionMixin + + +class TransparentProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx): + super(TransparentProxy, self).__init__(ctx) + self.resolver = platform.resolver() + + def __call__(self): + try: + self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) + except Exception as e: + raise ProtocolException("Transparent mode failure: %s" % repr(e), e) + + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py deleted file mode 100644 index 2e440fe8..00000000 --- a/libmproxy/proxy/primitives.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import absolute_import -import collections -from netlib import socks, tcp - - -class Log(object): - def __init__(self, msg, level="info"): - self.msg = msg - self.level = level - - -class Kill(Exception): - """ - Kill a connection. - """ \ No newline at end of file diff --git a/libmproxy/proxy/root_context.py b/libmproxy/proxy/root_context.py new file mode 100644 index 00000000..35909612 --- /dev/null +++ b/libmproxy/proxy/root_context.py @@ -0,0 +1,93 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol + +from ..protocol import ( + RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin +) +from .modes import HttpProxy, HttpUpstreamProxy, ReverseProxy + + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel, .next_layer() and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def next_layer(self, top_layer): + """ + This function determines the next layer in the protocol stack. + + Arguments: + top_layer: the current top layer. + + Returns: + The next layer + """ + + # 1. Check for --ignore. + if self.config.check_ignore(top_layer.server_conn.address): + return RawTCPLayer(top_layer, logging=False) + + d = top_layer.client_conn.rfile.peek(3) + client_tls = is_tls_record_magic(d) + + # 2. Always insert a TLS layer, even if there's neither client nor server tls. + # An inline script may upgrade from http to https, + # in which case we need some form of TLS layer. + if isinstance(top_layer, ReverseProxy): + return TlsLayer(top_layer, client_tls, top_layer.server_tls) + if isinstance(top_layer, ServerConnectionMixin): + return TlsLayer(top_layer, client_tls, client_tls) + + # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. + if isinstance(top_layer, TlsLayer): + if isinstance(top_layer.ctx, HttpProxy): + return Http1Layer(top_layer, "regular") + if isinstance(top_layer.ctx, HttpUpstreamProxy): + return Http1Layer(top_layer, "upstream") + + # 4. Check for other TLS cases (e.g. after CONNECT). + if client_tls: + return TlsLayer(top_layer, True, True) + + # 4. Check for --tcp + if self.config.check_tcp(top_layer.server_conn.address): + return RawTCPLayer(top_layer) + + # 5. Check for TLS ALPN (HTTP1/HTTP2) + if isinstance(top_layer, TlsLayer): + alpn = top_layer.client_conn.get_alpn_proto_negotiated() + if alpn == HTTP2Protocol.ALPN_PROTO_H2: + return Http2Layer(top_layer, 'transparent') + if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: + return Http1Layer(top_layer, 'transparent') + + # 6. Assume HTTP1 by default + return Http1Layer(top_layer, 'transparent') + + # In a future version, we want to implement TCP passthrough as the last fallback, + # but we don't have the UI part ready for that. + # + # d = top_layer.client_conn.rfile.peek(3) + # is_ascii = ( + # len(d) == 3 and + # # better be safe here and don't expect uppercase... + # all(x in string.ascii_letters for x in d) + # ) + # # TODO: This could block if there are not enough bytes available? + # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) + # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) + + @property + def layers(self): + return [] + + def __repr__(self): + return "RootContext" diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 5abd0877..2a451ba1 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -3,14 +3,15 @@ from __future__ import absolute_import, print_function import traceback import sys import socket + from netlib import tcp from netlib.http.http1 import HTTP1Protocol from netlib.tcp import NetLibError - -from .. import protocol2 from ..exceptions import ProtocolException, ServerException -from .primitives import Log, Kill -from .connection import ClientConnection +from ..protocol import Log, Kill +from ..models import ClientConnection, make_error_response +from .modes import HttpUpstreamProxy, HttpProxy, ReverseProxy, TransparentProxy, Socks5Proxy +from .root_context import RootContext class DummyServer: @@ -71,7 +72,7 @@ class ConnectionHandler(object): """@type: libmproxy.controller.Channel""" def _create_root_layer(self): - root_context = protocol2.RootContext( + root_context = RootContext( self.client_conn, self.config, self.channel @@ -79,23 +80,23 @@ class ConnectionHandler(object): mode = self.config.mode if mode == "upstream": - return protocol2.HttpUpstreamProxy( + return HttpUpstreamProxy( root_context, self.config.upstream_server.address ) elif mode == "transparent": - return protocol2.TransparentProxy(root_context) + return TransparentProxy(root_context) elif mode == "reverse": server_tls = self.config.upstream_server.scheme == "https" - return protocol2.ReverseProxy( + return ReverseProxy( root_context, self.config.upstream_server.address, server_tls ) elif mode == "socks5": - return protocol2.Socks5Proxy(root_context) + return Socks5Proxy(root_context) elif mode == "regular": - return protocol2.HttpProxy(root_context) + return HttpProxy(root_context) elif callable(mode): # pragma: nocover return mode(root_context) else: # pragma: nocover @@ -116,7 +117,7 @@ class ConnectionHandler(object): # we send an HTTP error response, which is both # understandable by HTTP clients and humans. try: - error_response = protocol2.make_error_response(502, repr(e)) + error_response = make_error_response(502, repr(e)) self.client_conn.send(HTTP1Protocol().assemble(error_response)) except NetLibError: pass diff --git a/test/test_dump.py b/test/test_dump.py index b05f6a0f..a0ad6cb4 100644 --- a/test/test_dump.py +++ b/test/test_dump.py @@ -1,18 +1,18 @@ import os from cStringIO import StringIO +from libmproxy.models import HTTPResponse import netlib.tutils from netlib.http.semantics import CONTENT_MISSING from libmproxy import dump, flow -from libmproxy.protocol import http_wrappers -from libmproxy.proxy import Log +from libmproxy.protocol import Log import tutils import mock def test_strfuncs(): - t = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + t = HTTPResponse.wrap(netlib.tutils.tresp()) t.is_replay = True dump.str_response(t) @@ -34,7 +34,7 @@ class TestDumpMaster: m.handle_clientconnect(f.client_conn) m.handle_serverconnect(f.server_conn) m.handle_request(f) - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp(content)) + f.response = HTTPResponse.wrap(netlib.tutils.tresp(content)) f = m.handle_response(f) m.handle_clientdisconnect(f.client_conn) return f @@ -71,7 +71,7 @@ class TestDumpMaster: f = tutils.tflow() f.request.content = CONTENT_MISSING m.handle_request(f) - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = HTTPResponse.wrap(netlib.tutils.tresp()) f.response.content = CONTENT_MISSING m.handle_response(f) assert "content missing" in cs.getvalue() diff --git a/test/test_filt.py b/test/test_filt.py index bcdf6e4c..aeec2485 100644 --- a/test/test_filt.py +++ b/test/test_filt.py @@ -2,7 +2,7 @@ import cStringIO from netlib import odict from libmproxy import filt, flow from libmproxy.protocol import http -from libmproxy.protocol.primitives import Error +from libmproxy.models import Error import tutils diff --git a/test/test_flow.py b/test/test_flow.py index 5c49deed..9cce26b3 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -3,21 +3,18 @@ import time import os.path from cStringIO import StringIO import email.utils + import mock -from libmproxy.cmdline import parse_server_spec import netlib.utils from netlib import odict -from netlib.http.semantics import CONTENT_MISSING, HDR_FORM_URLENCODED, HDR_FORM_MULTIPART - -from libmproxy import filt, protocol, controller, utils, tnetstring, flow -from libmproxy.protocol import http_wrappers -from libmproxy.protocol.primitives import Error, Flow -from libmproxy.protocol.http import decoded +from netlib.http.semantics import CONTENT_MISSING, HDR_FORM_URLENCODED +from libmproxy import filt, protocol, controller, tnetstring, flow +from libmproxy.models import Error, Flow, HTTPRequest, HTTPResponse, HTTPFlow, decoded from libmproxy.proxy.config import HostMatcher from libmproxy.proxy import ProxyConfig from libmproxy.proxy.server import DummyServer -from libmproxy.proxy.connection import ClientConnection +from libmproxy.models.connections import ClientConnection import tutils @@ -25,7 +22,7 @@ def test_app_registry(): ar = flow.AppRegistry() ar.add("foo", "domain", 80) - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.host = "domain" r.port = 80 assert ar.get(r) @@ -33,7 +30,7 @@ def test_app_registry(): r.port = 81 assert not ar.get(r) - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.host = "domain2" r.port = 80 assert not ar.get(r) @@ -386,7 +383,7 @@ class TestFlow: def test_backup(self): f = tutils.tflow() - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = HTTPResponse.wrap(netlib.tutils.tresp()) f.request.content = "foo" assert not f.modified() f.backup() @@ -405,13 +402,13 @@ class TestFlow: def test_getset_state(self): f = tutils.tflow(resp=True) state = f.get_state() - assert f.get_state() == protocol.http.HTTPFlow.from_state( + assert f.get_state() == HTTPFlow.from_state( state).get_state() f.response = None f.error = Error("error") state = f.get_state() - assert f.get_state() == protocol.http.HTTPFlow.from_state( + assert f.get_state() == HTTPFlow.from_state( state).get_state() f2 = f.copy() @@ -519,16 +516,16 @@ class TestState: assert c.add_flow(newf) assert c.active_flow_count() == 2 - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = HTTPResponse.wrap(netlib.tutils.tresp()) assert c.update_flow(f) assert c.flow_count() == 2 assert c.active_flow_count() == 1 - _ = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + _ = HTTPResponse.wrap(netlib.tutils.tresp()) assert not c.update_flow(None) assert c.active_flow_count() == 1 - newf.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + newf.response = HTTPResponse.wrap(netlib.tutils.tresp()) assert c.update_flow(newf) assert c.active_flow_count() == 0 @@ -560,7 +557,7 @@ class TestState: c.set_limit("~s") assert c.limit_txt == "~s" assert len(c.view) == 0 - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = HTTPResponse.wrap(netlib.tutils.tresp()) c.update_flow(f) assert len(c.view) == 1 c.set_limit(None) @@ -592,7 +589,7 @@ class TestState: def _add_response(self, state): f = tutils.tflow() state.add_flow(f) - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = HTTPResponse.wrap(netlib.tutils.tresp()) state.update_flow(f) def _add_error(self, state): @@ -807,11 +804,11 @@ class TestFlowMaster: fm.anticomp = True f = tutils.tflow(req=None) fm.handle_clientconnect(f.client_conn) - f.request = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + f.request = HTTPRequest.wrap(netlib.tutils.treq()) fm.handle_request(f) assert s.flow_count() == 1 - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = HTTPResponse.wrap(netlib.tutils.tresp()) fm.handle_response(f) assert not fm.handle_response(None) assert s.flow_count() == 1 @@ -856,7 +853,7 @@ class TestFlowMaster: s = flow.State() f = tutils.tflow() - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp(f.request)) + f.response = HTTPResponse.wrap(netlib.tutils.tresp(f.request)) pb = [f] fm = flow.FlowMaster(None, s) @@ -910,7 +907,7 @@ class TestFlowMaster: def test_server_playback_kill(self): s = flow.State() f = tutils.tflow() - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp(f.request)) + f.response = HTTPResponse.wrap(netlib.tutils.tresp(f.request)) pb = [f] fm = flow.FlowMaster(None, s) fm.refresh_server_playback = True @@ -1009,7 +1006,7 @@ class TestRequest: assert r.get_state() == r2.get_state() def test_get_url(self): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) assert r.url == "http://address:22/path" @@ -1030,7 +1027,7 @@ class TestRequest: assert r.pretty_url(True) == "https://foo.com:22/path" def test_path_components(self): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.path = "/" assert r.get_path_components() == [] r.path = "/foo/bar" @@ -1050,7 +1047,7 @@ class TestRequest: def test_getset_form_urlencoded(self): d = odict.ODict([("one", "two"), ("three", "four")]) - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq(content=netlib.utils.urlencode(d.lst))) + r = HTTPRequest.wrap(netlib.tutils.treq(content=netlib.utils.urlencode(d.lst))) r.headers["content-type"] = [HDR_FORM_URLENCODED] assert r.get_form_urlencoded() == d @@ -1064,7 +1061,7 @@ class TestRequest: def test_getset_query(self): h = odict.ODictCaseless() - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.path = "/foo?x=y&a=b" q = r.get_query() assert q.lst == [("x", "y"), ("a", "b")] @@ -1087,7 +1084,7 @@ class TestRequest: def test_anticache(self): h = odict.ODictCaseless() - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.headers = h h["if-modified-since"] = ["test"] h["if-none-match"] = ["test"] @@ -1096,7 +1093,7 @@ class TestRequest: assert not "if-none-match" in r.headers def test_replace(self): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.path = "path/foo" r.headers["Foo"] = ["fOo"] r.content = "afoob" @@ -1106,31 +1103,31 @@ class TestRequest: assert r.headers["boo"] == ["boo"] def test_constrain_encoding(self): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.headers["accept-encoding"] = ["gzip", "oink"] r.constrain_encoding() assert "oink" not in r.headers["accept-encoding"] def test_decodeencode(self): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" r.decode() assert not r.headers["content-encoding"] assert r.content == "falafel" - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.content = "falafel" assert not r.decode() - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" r.encode("identity") assert r.headers["content-encoding"] == ["identity"] assert r.content == "falafel" - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" r.encode("gzip") @@ -1141,7 +1138,7 @@ class TestRequest: assert r.content == "falafel" def test_get_decoded_content(self): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.content = None r.headers["content-encoding"] = ["identity"] assert r.get_decoded_content() == None @@ -1153,7 +1150,7 @@ class TestRequest: def test_get_content_type(self): h = odict.ODictCaseless() h["Content-Type"] = ["text/plain"] - resp = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + resp = HTTPResponse.wrap(netlib.tutils.tresp()) resp.headers = h assert resp.headers.get_first("content-type") == "text/plain" @@ -1166,7 +1163,7 @@ class TestResponse: assert resp2.get_state() == resp.get_state() def test_refresh(self): - r = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + r = HTTPResponse.wrap(netlib.tutils.tresp()) n = time.time() r.headers["date"] = [email.utils.formatdate(n)] pre = r.headers["date"] @@ -1184,7 +1181,7 @@ class TestResponse: r.refresh() def test_refresh_cookie(self): - r = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + r = HTTPResponse.wrap(netlib.tutils.tresp()) # Invalid expires format, sent to us by Reddit. c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2037 23:59:59 GMT; Path=/" @@ -1194,7 +1191,7 @@ class TestResponse: assert "00:21:38" in r._refresh_cookie(c, 60) def test_replace(self): - r = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["Foo"] = ["fOo"] r.content = "afoob" assert r.replace("foo(?i)", "boo") == 3 @@ -1202,21 +1199,21 @@ class TestResponse: assert r.headers["boo"] == ["boo"] def test_decodeencode(self): - r = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" assert r.decode() assert not r.headers["content-encoding"] assert r.content == "falafel" - r = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" r.encode("identity") assert r.headers["content-encoding"] == ["identity"] assert r.content == "falafel" - r = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" r.encode("gzip") @@ -1233,7 +1230,7 @@ class TestResponse: def test_get_content_type(self): h = odict.ODictCaseless() h["Content-Type"] = ["text/plain"] - resp = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + resp = HTTPResponse.wrap(netlib.tutils.tresp()) resp.headers = h assert resp.headers.get_first("content-type") == "text/plain" @@ -1277,7 +1274,7 @@ class TestClientConnection: def test_decoded(): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) assert r.content == "content" assert not r.headers["content-encoding"] r.encode("gzip") diff --git a/test/test_proxy.py b/test/test_proxy.py index 301ce2ca..b9ca2cce 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -1,14 +1,14 @@ +import mock +from OpenSSL import SSL + from libmproxy import cmdline from libmproxy.proxy import ProxyConfig from libmproxy.proxy.config import process_proxy_options -from libmproxy.proxy.connection import ServerConnection +from libmproxy.models.connections import ServerConnection from libmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler import tutils from libpathod import test from netlib import http, tcp -import mock - -from OpenSSL import SSL class TestServerConnection: diff --git a/test/test_server.py b/test/test_server.py index 66c3a0ae..23d802ca 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -11,7 +11,9 @@ from netlib.http.semantics import CONTENT_MISSING from libpathod import pathoc, pathod from libmproxy.proxy.config import HostMatcher -from libmproxy.protocol import KILL, Error, http_wrappers +from libmproxy.protocol import Kill +from libmproxy.models import Error, HTTPResponse + import tutils import tservers @@ -734,7 +736,7 @@ class TestStreamRequest(tservers.HTTPProxTest): class MasterFakeResponse(tservers.TestMaster): def handle_request(self, f): - resp = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + resp = HTTPResponse.wrap(netlib.tutils.tresp()) f.reply(resp) @@ -759,7 +761,7 @@ class TestServerConnect(tservers.HTTPProxTest): class MasterKillRequest(tservers.TestMaster): def handle_request(self, f): - f.reply(KILL) + f.reply(Kill) class TestKillRequest(tservers.HTTPProxTest): @@ -773,7 +775,7 @@ class TestKillRequest(tservers.HTTPProxTest): class MasterKillResponse(tservers.TestMaster): def handle_response(self, f): - f.reply(KILL) + f.reply(Kill) class TestKillResponse(tservers.HTTPProxTest): @@ -799,7 +801,7 @@ class TestTransparentResolveError(tservers.TransparentProxTest): class MasterIncomplete(tservers.TestMaster): def handle_request(self, f): - resp = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + resp = HTTPResponse.wrap(netlib.tutils.tresp()) resp.content = CONTENT_MISSING f.reply(resp) @@ -938,7 +940,7 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxTest): if not (k[0] in exclude): f.client_conn.finish() f.error = Error("terminated") - f.reply(KILL) + f.reply(Kill) return _func(f) setattr(master, attr, handler) diff --git a/test/tutils.py b/test/tutils.py index 61b1154c..d64388f3 100644 --- a/test/tutils.py +++ b/test/tutils.py @@ -3,22 +3,19 @@ import shutil import tempfile import argparse import sys -import mock_urwid from cStringIO import StringIO from contextlib import contextmanager + from nose.plugins.skip import SkipTest from mock import Mock -from time import time -from netlib import certutils, odict import netlib.tutils - -from libmproxy import flow, utils, controller -from libmproxy.protocol import http, http_wrappers -from libmproxy.proxy.connection import ClientConnection, ServerConnection +from libmproxy import utils, controller +from libmproxy.models import ( + ClientConnection, ServerConnection, Error, HTTPRequest, HTTPResponse, HTTPFlow +) from libmproxy.console.flowview import FlowView from libmproxy.console import ConsoleState -from libmproxy.protocol.primitives import Error def _SkipWindows(): @@ -53,11 +50,11 @@ def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None): err = terr() if req: - req = http_wrappers.HTTPRequest.wrap(req) + req = HTTPRequest.wrap(req) if resp: - resp = http_wrappers.HTTPResponse.wrap(resp) + resp = HTTPResponse.wrap(resp) - f = http.HTTPFlow(client_conn, server_conn) + f = HTTPFlow(client_conn, server_conn) f.request = req f.response = resp f.error = err @@ -91,7 +88,6 @@ def tserver_conn(): return c - def terr(content="error"): """ @return: libmproxy.protocol.primitives.Error @@ -106,7 +102,7 @@ def tflowview(request_contents=None): if request_contents is None: flow = tflow() else: - flow = tflow(req=treq(request_contents)) + flow = tflow(req=netlib.tutils.treq(request_contents)) fv = FlowView(m, cs, flow) return fv @@ -184,4 +180,5 @@ def capture_stderr(command, *args, **kwargs): yield sys.stderr.getvalue() sys.stderr = out + test_data = utils.Data(__name__) -- cgit v1.2.3 From 63ad4a4f5117d34ba6e9692eef1fc88f68b19c3d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 15:59:50 +0200 Subject: coverage++ --- .coveragerc | 6 +++++- libmproxy/models/__init__.py | 4 ++-- libmproxy/protocol/base.py | 4 ++-- libmproxy/protocol/http.py | 2 +- 4 files changed, 10 insertions(+), 6 deletions(-) diff --git a/.coveragerc b/.coveragerc index 70ff48e7..fef1089b 100644 --- a/.coveragerc +++ b/.coveragerc @@ -1,6 +1,10 @@ -[rum] +[run] branch = True [report] omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py include = *libmproxy* +exclude_lines = + pragma: nocover + pragma: no cover + raise NotImplementedError() \ No newline at end of file diff --git a/libmproxy/models/__init__.py b/libmproxy/models/__init__.py index 3947847c..a54f305f 100644 --- a/libmproxy/models/__init__.py +++ b/libmproxy/models/__init__.py @@ -8,8 +8,8 @@ from .connections import ClientConnection, ServerConnection from .flow import Flow, Error __all__ = [ - "HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded" - "make_error_response", "make_connect_request", + "HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded", + "make_error_response", "make_connect_request", "make_connect_response", "ClientConnection", "ServerConnection", "Flow", "Error", diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index d22a71c6..1c9b356c 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -43,7 +43,7 @@ class _LayerCodeCompletion(object): Dummy class that provides type hinting in PyCharm, which simplifies development a lot. """ - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): # pragma: nocover super(_LayerCodeCompletion, self).__init__(*args, **kwargs) if True: return @@ -70,7 +70,7 @@ class Layer(_LayerCodeCompletion): Raises: ProtocolException in case of protocol exceptions. """ - raise NotImplementedError + raise NotImplementedError() def __getattr__(self, name): """ diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index fc57f6df..345b3aa8 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -40,7 +40,7 @@ class _StreamingHttpLayer(_HttpLayer): def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): raise NotImplementedError() - yield "this is a generator" + yield "this is a generator" # pragma: no cover def send_response_headers(self, response): raise NotImplementedError -- cgit v1.2.3 From 1e9aef5b1e3e1d60a2bb94d47be03b780c10a497 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 00:14:42 +0200 Subject: fix upstream proxy server change, update example --- examples/change_upstream_proxy.py | 47 ++++++++++++++++++++++----------------- libmproxy/protocol/base.py | 4 ++++ libmproxy/protocol/http.py | 25 ++++++++++----------- libmproxy/protocol/tls.py | 4 +++- 4 files changed, 45 insertions(+), 35 deletions(-) diff --git a/examples/change_upstream_proxy.py b/examples/change_upstream_proxy.py index 7782dd84..8f58e1f2 100644 --- a/examples/change_upstream_proxy.py +++ b/examples/change_upstream_proxy.py @@ -1,29 +1,34 @@ # This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy # in upstream proxy mode. # -# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s -# "change_upstream_proxy.py host" -from libmproxy.protocol.http import send_connect_request - -alternative_upstream_proxy = ("localhost", 8082) +# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s change_upstream_proxy.py +# +# If you want to change the target server, you should modify flow.request.host and flow.request.port +# flow.live.set_server should only be used by inline scripts to change the upstream proxy. -def should_redirect(flow): - return flow.request.host == "example.com" +def proxy_address(flow): + # Poor man's loadbalancing: route every second domain through the alternative proxy. + if hash(flow.request.host) % 2 == 1: + return ("localhost", 8082) + else: + return ("localhost", 8081) def request(context, flow): - if flow.live and should_redirect(flow): - - # If you want to change the target server, you should modify flow.request.host and flow.request.port - # flow.live.change_server should only be used by inline scripts to change the upstream proxy, - # unless you are sure that you know what you are doing. - server_changed = flow.live.change_server( - alternative_upstream_proxy, - persistent_change=True) - if flow.request.scheme == "https" and server_changed: - send_connect_request( - flow.live.c.server_conn, - flow.request.host, - flow.request.port) - flow.live.c.establish_ssl(server=True) + if flow.request.method == "CONNECT": + # If the decision is done by domain, one could also modify the server address here. + # We do it after CONNECT here to have the request data available as well. + return + address = proxy_address(flow) + if flow.live: + if flow.request.scheme == "http": + # For a normal HTTP request, we just change the proxy server and we're done! + if address != flow.live.server_conn.address: + flow.live.set_server(address, depth=1) + else: + # If we have CONNECTed (and thereby established "destination state"), the story is + # a bit more complex. Now we don't want to change the top level address (which is + # the connect destination) but the address below that. (Notice the `.via` and depth=2). + if address != flow.live.server_conn.via.address: + flow.live.set_server(address, depth=2) diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 1c9b356c..3440cb01 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -116,6 +116,10 @@ class ServerConnectionMixin(object): self._disconnect() self.log("Set new server address: " + repr(address), "debug") self.server_conn.address = address + if server_tls: + raise ProtocolException( + "Cannot upgrade to TLS, no TLS layer on the protocol stack." + ) else: self.ctx.set_server(address, server_tls, sni, depth - 1) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 345b3aa8..3b62c389 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -207,6 +207,9 @@ class ConnectServerConnection(object): def __getattr__(self, item): return getattr(self.via, item) + def __nonzero__(self): + return bool(self.via) + class UpstreamConnectLayer(Layer): def __init__(self, ctx, connect_request): @@ -221,19 +224,22 @@ class UpstreamConnectLayer(Layer): layer = self.ctx.next_layer(self) layer() + def _send_connect_request(self): + self.send_request(self.connect_request) + resp = self.read_response("CONNECT") + if resp.code != 200: + raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") + def connect(self): if not self.server_conn: self.ctx.connect() - self.send_request(self.connect_request) + self._send_connect_request() else: pass # swallow the message def reconnect(self): self.ctx.reconnect() - self.send_request(self.connect_request) - resp = self.read_response("CONNECT") - if resp.code != 200: - raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") + self._send_connect_request() def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: @@ -386,7 +392,7 @@ class HttpLayer(Layer): if self.supports_streaming: flow.response = self.read_response_headers() else: - flow.response = self.read_response() + flow.response = self.read_response(flow.request.method) try: get_response() @@ -473,13 +479,6 @@ class HttpLayer(Layer): # Establish connection is neccessary. if not self.server_conn: self.connect() - - # SetServer is not guaranteed to work with TLS: - # If there's not TlsLayer below which could catch the exception, - # TLS will not be established. - if tls and not self.server_conn.tls_established: - raise ProtocolException( - "Cannot upgrade to SSL, no TLS layer on the protocol stack.") else: if not self.server_conn: self.connect() diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index b85a6595..2646ec4f 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -152,10 +152,12 @@ class TlsLayer(Layer): self._establish_tls_with_server() def set_server(self, address, server_tls=None, sni=None, depth=1): - self.ctx.set_server(address, server_tls, sni, depth) if depth == 1 and server_tls is not None: + self.ctx.set_server(address, None, None, 1) self._sni_from_server_change = sni self._server_tls = server_tls + else: + self.ctx.set_server(address, server_tls, sni, depth) @property def sni_for_server_connection(self): -- cgit v1.2.3 From 7450bef615436d39bcd2a0d2a8892b8f42beea6f Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 13:43:30 +0200 Subject: fix dns_spoofing example, avoid connecting to itself --- examples/dns_spoofing.py | 47 +++++++++++++++++++++++++-------------- libmproxy/models/connections.py | 2 +- libmproxy/models/flow.py | 2 +- libmproxy/protocol/base.py | 21 ++++++++++++++++- libmproxy/protocol/http_replay.py | 1 + libmproxy/proxy/config.py | 2 +- libmproxy/proxy/server.py | 2 +- 7 files changed, 55 insertions(+), 22 deletions(-) diff --git a/examples/dns_spoofing.py b/examples/dns_spoofing.py index dddf172c..98495d45 100644 --- a/examples/dns_spoofing.py +++ b/examples/dns_spoofing.py @@ -8,30 +8,43 @@ Similarly, if there's no Host header or a spoofed Host header, we're out of luck Using transparent mode is the better option most of the time. Usage: - mitmproxy - -p 80 - -R http://example.com/ // Used as the target location if no Host header is present mitmproxy -p 443 - -R https://example.com/ // Used as the target locaction if neither SNI nor host header are present. + -s dns_spoofing.py + # Used as the target location if neither SNI nor host header are present. + -R http://example.com/ + mitmdump + -p 80 + -R http://localhost:443/ -mitmproxy will always connect to the default location first, so it must be reachable. -As a workaround, you can spawn an arbitrary HTTP server and use that for both endpoints, e.g. -mitmproxy -p 80 -R http://localhost:8000 -mitmproxy -p 443 -R https2http://localhost:8000 + (Setting up a single proxy instance and using iptables to redirect to it + works as well) """ +import re + + +# This regex extracts splits the host header into host and port. +# Handles the edge case of IPv6 addresses containing colons. +# https://bugzilla.mozilla.org/show_bug.cgi?id=45891 +parse_host_header = re.compile(r"^(?P[^:]+|\[.+\])(?::(?P\d+))?$") def request(context, flow): if flow.client_conn.ssl_established: - # TLS SNI or Host header - flow.request.host = flow.client_conn.connection.get_servername( - ) or flow.request.pretty_host(hostheader=True) - - # If you use a https2http location as default destination, these - # attributes need to be corrected as well: - flow.request.port = 443 flow.request.scheme = "https" + sni = flow.client_conn.connection.get_servername() + port = 443 else: - # Host header - flow.request.host = flow.request.pretty_host(hostheader=True) + flow.request.scheme = "http" + sni = None + port = 80 + + host_header = flow.request.pretty_host(hostheader=True) + m = parse_host_header.match(host_header) + if m: + host_header = m.group("host").strip("[]") + if m.group("port"): + port = int(m.group("port")) + + flow.request.host = sni or host_header + flow.request.port = port \ No newline at end of file diff --git a/libmproxy/models/connections.py b/libmproxy/models/connections.py index 98bae3cc..f1e10de9 100644 --- a/libmproxy/models/connections.py +++ b/libmproxy/models/connections.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) import copy import os diff --git a/libmproxy/models/flow.py b/libmproxy/models/flow.py index 58287e5b..8eff18f4 100644 --- a/libmproxy/models/flow.py +++ b/libmproxy/models/flow.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) import copy import uuid diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 3440cb01..d1af547f 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -103,6 +103,7 @@ class ServerConnectionMixin(object): def __init__(self, server_address=None): super(ServerConnectionMixin, self).__init__() self.server_conn = ServerConnection(server_address) + self._check_self_connect() def reconnect(self): address = self.server_conn.address @@ -110,12 +111,30 @@ class ServerConnectionMixin(object): self.server_conn.address = address self.connect() + def _check_self_connect(self): + """ + We try to protect the proxy from _accidentally_ connecting to itself, + e.g. because of a failed transparent lookup or an invalid configuration. + """ + address = self.server_conn.address + if address: + self_connect = ( + address.port == self.config.port and + address.host in ("localhost", "127.0.0.1", "::1") + ) + if self_connect: + raise ProtocolException( + "Invalid server address: {}\r\n" + "The proxy shall not connect to itself.".format(repr(address)) + ) + def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: if self.server_conn: self._disconnect() self.log("Set new server address: " + repr(address), "debug") self.server_conn.address = address + self._check_self_connect() if server_tls: raise ProtocolException( "Cannot upgrade to TLS, no TLS layer on the protocol stack." @@ -141,7 +160,7 @@ class ServerConnectionMixin(object): self.server_conn.connect() except tcp.NetLibError as e: raise ProtocolException( - "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) + "Server connection to %s failed: %s" % (repr(self.server_conn.address), e), e) class Log(object): diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py index e0144c93..c37fd131 100644 --- a/libmproxy/protocol/http_replay.py +++ b/libmproxy/protocol/http_replay.py @@ -1,3 +1,4 @@ +from __future__ import (absolute_import, print_function, division) import threading from netlib.http import HttpError diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 65029087..8d2a286d 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) import collections import os import re diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 2a451ba1..b565ef86 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -1,4 +1,4 @@ -from __future__ import absolute_import, print_function +from __future__ import (absolute_import, print_function, division) import traceback import sys -- cgit v1.2.3 From 41e6e538dfa758b7d9f867f85f62e881ae408684 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 13:49:47 +0200 Subject: fix layer initialization --- libmproxy/protocol/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index d1af547f..4eb843e4 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -61,8 +61,8 @@ class Layer(_LayerCodeCompletion): Args: ctx: The (read-only) higher layer. """ - super(Layer, self).__init__(*args, **kwargs) self.ctx = ctx + super(Layer, self).__init__(*args, **kwargs) def __call__(self): """ -- cgit v1.2.3 From b04e6e56ab1e69853abebfb950539e3a3aefbdf2 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 17:05:52 +0200 Subject: update inline script hooks --- examples/stub.py | 13 ++++++++++--- libmproxy/flow.py | 4 ++++ libmproxy/protocol/base.py | 10 +++++++--- libmproxy/protocol/http.py | 6 +++--- libmproxy/protocol/http_replay.py | 4 ++-- libmproxy/proxy/server.py | 5 +++++ test/test_proxy.py | 7 ++++++- test/test_server.py | 4 +--- 8 files changed, 38 insertions(+), 15 deletions(-) diff --git a/examples/stub.py b/examples/stub.py index d5502a47..bd3e7cd0 100644 --- a/examples/stub.py +++ b/examples/stub.py @@ -10,7 +10,7 @@ def start(context, argv): context.log("start") -def clientconnect(context, conn_handler): +def clientconnect(context, root_layer): """ Called when a client initiates a connection to the proxy. Note that a connection can correspond to multiple HTTP requests @@ -18,7 +18,7 @@ def clientconnect(context, conn_handler): context.log("clientconnect") -def serverconnect(context, conn_handler): +def serverconnect(context, server_connection): """ Called when the proxy initiates a connection to the target server. Note that a connection can correspond to multiple HTTP requests @@ -58,7 +58,14 @@ def error(context, flow): context.log("error") -def clientdisconnect(context, conn_handler): +def serverdisconnect(context, server_connection): + """ + Called when the proxy closes the connection to the target server. + """ + context.log("serverdisconnect") + + +def clientdisconnect(context, root_layer): """ Called when a client disconnects from the proxy. """ diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 00ec83d2..5eac8da9 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -961,6 +961,10 @@ class FlowMaster(controller.Master): self.run_script_hook("serverconnect", sc) sc.reply() + def handle_serverdisconnect(self, sc): + self.run_script_hook("serverdisconnect", sc) + sc.reply() + def handle_error(self, f): self.state.update_flow(f) self.run_script_hook("error", f) diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 4eb843e4..40ec0536 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -48,9 +48,11 @@ class _LayerCodeCompletion(object): if True: return self.config = None - """@type: libmproxy.proxy.config.ProxyConfig""" + """@type: libmproxy.proxy.ProxyConfig""" self.client_conn = None - """@type: libmproxy.proxy.connection.ClientConnection""" + """@type: libmproxy.models.ClientConnection""" + self.server_conn = None + """@type: libmproxy.models.ServerConnection""" self.channel = None """@type: libmproxy.controller.Channel""" @@ -62,6 +64,7 @@ class Layer(_LayerCodeCompletion): ctx: The (read-only) higher layer. """ self.ctx = ctx + """@type: libmproxy.protocol.Layer""" super(Layer, self).__init__(*args, **kwargs) def __call__(self): @@ -149,13 +152,14 @@ class ServerConnectionMixin(object): self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) self.server_conn.finish() self.server_conn.close() - # self.channel.tell("serverdisconnect", self) + self.channel.tell("serverdisconnect", self.server_conn) self.server_conn = ServerConnection(None) def connect(self): if not self.server_conn.address: raise ProtocolException("Cannot connect to server, no server address given.") self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + self.channel.ask("serverconnect", self.server_conn) try: self.server_conn.connect() except tcp.NetLibError as e: diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 3b62c389..f0f4ac24 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -418,7 +418,7 @@ class HttpLayer(Layer): # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True flow = self.channel.ask("responseheaders", flow) - if flow is None or flow == Kill: + if flow == Kill: raise Kill() if self.supports_streaming: @@ -442,7 +442,7 @@ class HttpLayer(Layer): [repr(flow.response)] ) response_reply = self.channel.ask("response", flow) - if response_reply is None or response_reply == Kill: + if response_reply == Kill: raise Kill() def process_request_hook(self, flow): @@ -462,7 +462,7 @@ class HttpLayer(Layer): flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" request_reply = self.channel.ask("request", flow) - if request_reply is None or request_reply == Kill: + if request_reply == Kill: raise Kill() if isinstance(request_reply, HTTPResponse): flow.response = request_reply diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py index c37fd131..2759a019 100644 --- a/libmproxy/protocol/http_replay.py +++ b/libmproxy/protocol/http_replay.py @@ -36,7 +36,7 @@ class RequestReplayThread(threading.Thread): # If we have a channel, run script hooks. if self.channel: request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == Kill: + if request_reply == Kill: raise Kill() elif isinstance(request_reply, HTTPResponse): self.flow.response = request_reply @@ -82,7 +82,7 @@ class RequestReplayThread(threading.Thread): ) if self.channel: response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == Kill: + if response_reply == Kill: raise Kill() except (HttpError, NetLibError) as v: self.flow.error = Error(repr(v)) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index b565ef86..e9e8df09 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -106,6 +106,10 @@ class ConnectionHandler(object): self.log("clientconnect", "info") root_layer = self._create_root_layer() + root_layer = self.channel.ask("clientconnect", root_layer) + if root_layer == Kill: + def root_layer(): + raise Kill() try: root_layer() @@ -128,6 +132,7 @@ class ConnectionHandler(object): print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) self.log("clientdisconnect", "info") + self.channel.tell("clientdisconnect", root_layer) self.client_conn.finish() def log(self, msg, level): diff --git a/test/test_proxy.py b/test/test_proxy.py index b9ca2cce..cc6a79d0 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -172,11 +172,16 @@ class TestConnectionHandler: root_layer = mock.Mock() root_layer.side_effect = RuntimeError config.mode.return_value = root_layer + channel = mock.Mock() + + def ask(_, x): + return x + channel.ask = ask c = ConnectionHandler( mock.MagicMock(), ("127.0.0.1", 8080), config, - mock.MagicMock() + channel ) with tutils.capture_stderr(c.handle) as output: assert "mitmproxy has crashed" in output diff --git a/test/test_server.py b/test/test_server.py index 23d802ca..a1259b7f 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -611,13 +611,11 @@ class MasterRedirectRequest(tservers.TestMaster): def handle_request(self, f): if f.request.path == "/p/201": - # This part should have no impact, but it should not cause any exceptions. + # This part should have no impact, but it should also not cause any exceptions. addr = f.live.server_conn.address addr2 = Address(("127.0.0.1", self.redirect_port)) f.live.set_server(addr2) - f.live.connect() f.live.set_server(addr) - f.live.connect() # This is the actual redirection. f.request.port = self.redirect_port -- cgit v1.2.3 From 481cc6ea842dc3c531c45a4bd228bdd6ebcc4229 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 17:29:14 +0200 Subject: we don't support socks auth, refs #738 --- libmproxy/proxy/config.py | 7 +++++++ test/test_proxy.py | 3 +++ 2 files changed, 10 insertions(+) diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 8d2a286d..2a1b84cb 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -136,6 +136,13 @@ def process_proxy_options(parser, options): ) if options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd: + + if options.socks_proxy: + return parser.error( + "Proxy Authentication not supported in SOCKS mode. " + "https://github.com/mitmproxy/mitmproxy/issues/738" + ) + if options.auth_singleuser: if len(options.auth_singleuser.split(':')) != 2: return parser.error( diff --git a/test/test_proxy.py b/test/test_proxy.py index cc6a79d0..3707fabe 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -93,6 +93,9 @@ class TestProcessProxyOptions: self.assert_err("not allowed with", "-R", "http://localhost", "-T") + def test_socks_auth(self): + self.assert_err("Proxy Authentication not supported in SOCKS mode.", "--socks", "--nonanonymous") + def test_client_certs(self): with tutils.tmpdir() as cadir: self.assert_noerr("--client-certs", cadir) -- cgit v1.2.3 From c4d6b357262a5964a8d10ea20b92d22efc9c68a4 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 31 Aug 2015 22:20:12 +0200 Subject: do not log WindowUpdateFrame frames --- libmproxy/protocol/http.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index f0f4ac24..7f57d17c 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -7,6 +7,7 @@ from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol +from netlib.http.http2.frame import WindowUpdateFrame from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException @@ -187,8 +188,15 @@ class Http2Layer(_HttpLayer): layer = HttpLayer(self, self.mode) layer() - def handle_unexpected_frame(self, frm): - self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") + def handle_unexpected_frame(self, frame): + if isinstance(frame, WindowUpdateFrame): + # Clients are sending WindowUpdate frames depending on their flow control algorithm. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should keep track of our own flow control window and + # stall transmission if the outgoing flow control buffer is full. + return + self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") class ConnectServerConnection(object): -- cgit v1.2.3 From b5f1c38e78e6711240e9805798456bb3930ef864 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 1 Sep 2015 02:35:05 +0200 Subject: minor docs improvements --- doc-src/modes.html | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) diff --git a/doc-src/modes.html b/doc-src/modes.html index a878fd82..6bd92167 100644 --- a/doc-src/modes.html +++ b/doc-src/modes.html @@ -173,10 +173,10 @@ on port 80. You can test your app on the example.com domain and get all requests recorded in mitmproxy. - Say you have some toy project that should get SSL support. Simply set up -mitmproxy with SSL termination and you're done (mitmdump -p 443 -R -http://localhost:80/). There are better tools for this specific -task, but mitmproxy is very quick and simple way to set up an SSL-speaking -server. +mitmproxy as a reverse proxy on port 443 and you're done (mitmdump -p 443 -R +http://localhost:80/). mitmproxy auto-detects TLS traffic and intercepts it dynamically. +There are better tools for this specific task, but mitmproxy is very quick and simple way to +set up an SSL-speaking server. - Want to add a non-SSL-capable compression proxy in front of your server? You could even spawn a mitmproxy instance that terminates SSL (-R http://...), -- cgit v1.2.3 From f1c8b47b1eb153d448061c0ddce21030c31af2b7 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 1 Sep 2015 19:24:36 +0200 Subject: better tls error messages, fix #672 --- libmproxy/protocol/tls.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index 2646ec4f..a8dc8bb2 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -259,9 +259,17 @@ class TlsLayer(Layer): (tls_cert_err['depth'], tls_cert_err['errno']), "error") self.log("Aborting connection attempt", "error") - raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + address=repr(self.server_conn.address), + sni=self.sni_for_server_connection, + e=repr(e), + ), e) except NetLibError as e: - raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + address=repr(self.server_conn.address), + sni=self.sni_for_server_connection, + e=repr(e), + ), e) self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") -- cgit v1.2.3