diff options
Diffstat (limited to 'libmproxy')
36 files changed, 2376 insertions, 2041 deletions
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 6125bfbf..7f6f69ef 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -2,8 +2,8 @@ from __future__ import absolute_import import os import re import configargparse +from netlib.tcp import Address, sslversion_choices -from netlib import http import netlib.utils from . import filt, utils, version @@ -108,26 +108,9 @@ def parse_server_spec(url): raise configargparse.ArgumentTypeError( "Invalid server specification: %s" % url ) - - if p[0].lower() == "https": - ssl = [True, True] - else: - ssl = [False, False] - - return ssl + list(p[1:3]) - - -def parse_server_spec_special(url): - """ - Provides additional support for http2https and https2http schemes. - """ - normalized_url = re.sub("^https?2", "", url) - ret = parse_server_spec(normalized_url) - if url.lower().startswith("https2http"): - ret[0] = True - elif url.lower().startswith("http2https"): - ret[0] = False - return ret + address = Address(p[1:3]) + scheme = p[0].lower() + return config.ServerSpec(scheme, address) def get_common_options(options): @@ -192,24 +175,24 @@ def get_common_options(options): outfile=options.outfile, verbosity=options.verbose, nopop=options.nopop, - replay_ignore_content = options.replay_ignore_content, - replay_ignore_params = options.replay_ignore_params, - replay_ignore_payload_params = options.replay_ignore_payload_params, - replay_ignore_host = options.replay_ignore_host + replay_ignore_content=options.replay_ignore_content, + replay_ignore_params=options.replay_ignore_params, + replay_ignore_payload_params=options.replay_ignore_payload_params, + replay_ignore_host=options.replay_ignore_host ) -def common_options(parser): +def basic_options(parser): parser.add_argument( '--version', - action= 'version', - version= "%(prog)s" + " " + version.VERSION + action='version', + version="%(prog)s" + " " + version.VERSION ) parser.add_argument( '--shortversion', - action= 'version', - help = "show program's short version number and exit", - version = version.VERSION + action='version', + help="show program's short version number and exit", + version=version.VERSION ) parser.add_argument( "--anticache", @@ -301,11 +284,42 @@ def common_options(parser): """ ) + +def proxy_modes(parser): + group = parser.add_argument_group("Proxy Modes").add_mutually_exclusive_group() + group.add_argument( + "-R", "--reverse", + action="store", + type=parse_server_spec, + dest="reverse_proxy", + default=None, + help=""" + Forward all requests to upstream HTTP server: + http[s][2http[s]]://host[:port] + """ + ) + group.add_argument( + "--socks", + action="store_true", dest="socks_proxy", default=False, + help="Set SOCKS5 proxy mode." + ) + group.add_argument( + "-T", "--transparent", + action="store_true", dest="transparent_proxy", default=False, + help="Set transparent proxy mode." + ) + group.add_argument( + "-U", "--upstream", + action="store", + type=parse_server_spec, + dest="upstream_proxy", + default=None, + help="Forward all requests to upstream proxy server: http://host[:port]" + ) + + +def proxy_options(parser): group = parser.add_argument_group("Proxy Options") - # We could make a mutually exclusive group out of -R, -U, -T, but we don't - # do that because - --upstream-server should be in that group as well, but - # it's already in a different group. - our own error messages are more - # helpful group.add_argument( "-b", "--bind-address", action="store", type=str, dest="addr", default='', @@ -344,70 +358,78 @@ def common_options(parser): action="store", type=int, dest="port", default=8080, help="Proxy service port." ) + + +def proxy_ssl_options(parser): + # TODO: Agree to consistently either use "upstream" or "server". + group = parser.add_argument_group("SSL") group.add_argument( - "-R", "--reverse", - action="store", - type=parse_server_spec_special, - dest="reverse_proxy", - default=None, - help=""" - Forward all requests to upstream HTTP server: - http[s][2http[s]]://host[:port] - """ - ) + "--cert", + dest='certs', + default=[], + type=str, + metavar="SPEC", + action="append", + help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' + 'The domain may include a wildcard, and is equal to "*" if not specified. ' + 'The file at path is a certificate in PEM format. If a private key is included ' + 'in the PEM, it is used, else the default key in the conf dir is used. ' + 'The PEM file should contain the full certificate chain, with the leaf certificate ' + 'as the first entry. Can be passed multiple times.') group.add_argument( - "--socks", - action="store_true", dest="socks_proxy", default=False, - help="Set SOCKS5 proxy mode." + "--ciphers-client", action="store", + type=str, dest="ciphers_client", default=config.DEFAULT_CLIENT_CIPHERS, + help="Set supported ciphers for client connections. (OpenSSL Syntax)" ) group.add_argument( - "-T", "--transparent", - action="store_true", dest="transparent_proxy", default=False, - help="Set transparent proxy mode." + "--ciphers-server", action="store", + type=str, dest="ciphers_server", default=None, + help="Set supported ciphers for server connections. (OpenSSL Syntax)" ) group.add_argument( - "-U", "--upstream", - action="store", - type=parse_server_spec, - dest="upstream_proxy", - default=None, - help="Forward all requests to upstream proxy server: http://host[:port]" + "--client-certs", action="store", + type=str, dest="clientcerts", default=None, + help="Client certificate directory." ) group.add_argument( - "--spoof", - action="store_true", dest="spoof_mode", default=False, - help="Use Host header to connect to HTTP servers." + "--no-upstream-cert", default=False, + action="store_true", dest="no_upstream_cert", + help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--ssl-spoof", - action="store_true", dest="ssl_spoof_mode", default=False, - help="Use TLS SNI to connect to HTTPS servers." + "--verify-upstream-cert", default=False, + action="store_true", dest="ssl_verify_upstream_cert", + help="Verify upstream server SSL/TLS certificates and fail if invalid " + "or not present." ) group.add_argument( - "--spoofed-port", - action="store", dest="spoofed_ssl_port", type=int, default=443, - help="Port number of upstream HTTPS servers in SSL spoof mode." + "--upstream-trusted-cadir", default=None, action="store", + dest="ssl_verify_upstream_trusted_cadir", + help="Path to a directory of trusted CA certificates for upstream " + "server verification prepared using the c_rehash tool." ) - - group = parser.add_argument_group( - "Advanced Proxy Options", - """ - The following options allow a custom adjustment of the proxy - behavior. Normally, you don't want to use these options directly and - use the provided wrappers instead (-R, -U, -T). - """ + group.add_argument( + "--upstream-trusted-ca", default=None, action="store", + dest="ssl_verify_upstream_trusted_ca", + help="Path to a PEM formatted trusted CA certificate." ) group.add_argument( - "--http-form-in", dest="http_form_in", default=None, - action="store", choices=("relative", "absolute"), - help="Override the HTTP request form accepted by the proxy" + "--ssl-version-client", dest="ssl_version_client", + default="secure", action="store", + choices=sslversion_choices.keys(), + help="Set supported SSL/TLS versions for client connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) group.add_argument( - "--http-form-out", dest="http_form_out", default=None, - action="store", choices=("relative", "absolute"), - help="Override the HTTP request form sent upstream by the proxy" + "--ssl-version-server", dest="ssl_version_server", + default="secure", action="store", + choices=sslversion_choices.keys(), + help="Set supported SSL/TLS versions for server connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) + +def onboarding_app(parser): group = parser.add_argument_group("Onboarding App") group.add_argument( "--noapp", @@ -433,6 +455,8 @@ def common_options(parser): help="Port to serve the onboarding app from." ) + +def client_replay(parser): group = parser.add_argument_group("Client Replay") group.add_argument( "-c", "--client-replay", @@ -440,6 +464,8 @@ def common_options(parser): help="Replay client requests from a saved file." ) + +def server_replay(parser): group = parser.add_argument_group("Server Replay") group.add_argument( "-S", "--server-replay", @@ -504,6 +530,8 @@ def common_options(parser): default=False, help="Ignore request's destination host while searching for a saved flow to replay") + +def replacements(parser): group = parser.add_argument_group( "Replacements", """ @@ -520,14 +548,16 @@ def common_options(parser): ) group.add_argument( "--replace-from-file", - action = "append", type=str, dest="replace_file", default=[], - metavar = "PATH", - help = """ + action="append", type=str, dest="replace_file", default=[], + metavar="PATH", + help=""" Replacement pattern, where the replacement clause is a path to a file. """ ) + +def set_headers(parser): group = parser.add_argument_group( "Set Headers", """ @@ -543,21 +573,22 @@ def common_options(parser): help="Header set pattern." ) + +def proxy_authentication(parser): group = parser.add_argument_group( "Proxy Authentication", """ Specify which users are allowed to access the proxy and the method used for authenticating them. """ - ) - user_specification_group = group.add_mutually_exclusive_group() - user_specification_group.add_argument( + ).add_mutually_exclusive_group() + group.add_argument( "--nonanonymous", action="store_true", dest="auth_nonanonymous", help="Allow access to any user long as a credentials are specified." ) - user_specification_group.add_argument( + group.add_argument( "--singleuser", action="store", dest="auth_singleuser", type=str, metavar="USER", @@ -566,14 +597,25 @@ def common_options(parser): username:password. """ ) - user_specification_group.add_argument( + group.add_argument( "--htpasswd", action="store", dest="auth_htpasswd", type=str, metavar="PATH", help="Allow access to users specified in an Apache htpasswd file." ) - config.ssl_option_group(parser) + +def common_options(parser): + basic_options(parser) + proxy_modes(parser) + proxy_options(parser) + proxy_ssl_options(parser) + onboarding_app(parser) + client_replay(parser) + server_replay(parser) + replacements(parser) + set_headers(parser) + proxy_authentication(parser) def mitmproxy(): @@ -583,13 +625,13 @@ def mitmproxy(): parser = configargparse.ArgumentParser( usage="%(prog)s [options]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmproxy.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) common_options(parser) parser.add_argument( @@ -633,20 +675,20 @@ def mitmproxy(): def mitmdump(): parser = configargparse.ArgumentParser( usage="%(prog)s [options] [filter]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmdump.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) common_options(parser) parser.add_argument( "--keepserving", - action= "store_true", dest="keepserving", default=False, - help= """ + action="store_true", dest="keepserving", default=False, + help=""" Continue serving after client playback or file read. We exit by default. """ @@ -663,13 +705,13 @@ def mitmdump(): def mitmweb(): parser = configargparse.ArgumentParser( usage="%(prog)s [options]", - args_for_setting_config_path = ["--conf"], - default_config_files = [ + args_for_setting_config_path=["--conf"], + default_config_files=[ os.path.join(config.CA_DIR, "common.conf"), os.path.join(config.CA_DIR, "mitmweb.conf") ], - add_config_file_help = True, - add_env_var_help = True + add_config_file_help=True, + add_env_var_help=True ) group = parser.add_argument_group("Mitmweb") diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 1940e390..c25f7267 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -8,7 +8,7 @@ from netlib.http.semantics import CONTENT_MISSING import netlib.utils from .. import utils -from ..protocol.http import decoded +from ..models import decoded from . import signals diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 1e0f0c17..8b828653 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -9,7 +9,7 @@ from netlib.http.semantics import CONTENT_MISSING from . import common, grideditor, contentview, signals, searchable, tabs from . import flowdetailview from .. import utils, controller -from ..protocol.http import HTTPRequest, HTTPResponse, decoded +from ..models import HTTPRequest, HTTPResponse, decoded class SearchError(Exception): diff --git a/libmproxy/console/statusbar.py b/libmproxy/console/statusbar.py index 7eb2131b..ea2dbfa8 100644 --- a/libmproxy/console/statusbar.py +++ b/libmproxy/console/statusbar.py @@ -199,11 +199,12 @@ class StatusBar(urwid.WidgetWrap): r.append("[%s]" % (":".join(opts))) if self.master.server.config.mode in ["reverse", "upstream"]: - dst = self.master.server.config.mode.dst - scheme = "https" if dst[0] else "http" - if dst[1] != dst[0]: - scheme += "2https" if dst[1] else "http" - r.append("[dest:%s]" % utils.unparse_url(scheme, *dst[2:])) + dst = self.master.server.config.upstream_server + r.append("[dest:%s]" % netlib.utils.unparse_url( + dst.scheme, + dst.address.host, + dst.address.port + )) if self.master.scripts: r.append("[") r.append(("heading_key", "s")) diff --git a/libmproxy/contrib/README b/libmproxy/contrib/README index 3b0f7512..e5ce11da 100644 --- a/libmproxy/contrib/README +++ b/libmproxy/contrib/README @@ -8,3 +8,7 @@ jsbeautifier, git checkout 25/03/12, MIT license wbxml - https://github.com/davidpshaw/PyWBXMLDecoder + +tls, BSD license + - https://github.com/mhils/tls/tree/mitmproxy + - limited to required files.
\ No newline at end of file diff --git a/libmproxy/contrib/tls/__init__.py b/libmproxy/contrib/tls/__init__.py new file mode 100644 index 00000000..4b540884 --- /dev/null +++ b/libmproxy/contrib/tls/__init__.py @@ -0,0 +1,5 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function diff --git a/libmproxy/contrib/tls/_constructs.py b/libmproxy/contrib/tls/_constructs.py new file mode 100644 index 00000000..9c57a799 --- /dev/null +++ b/libmproxy/contrib/tls/_constructs.py @@ -0,0 +1,213 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +from construct import (Array, Bytes, Struct, UBInt16, UBInt32, UBInt8, PascalString, Embed, TunnelAdapter, GreedyRange, + Switch, OptionalGreedyRange) + +from .utils import UBInt24 + +ProtocolVersion = Struct( + "version", + UBInt8("major"), + UBInt8("minor"), +) + +TLSPlaintext = Struct( + "TLSPlaintext", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + Bytes("fragment", lambda ctx: ctx.length), +) + +TLSCompressed = Struct( + "TLSCompressed", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + 1024 + Bytes("fragment", lambda ctx: ctx.length), +) + +TLSCiphertext = Struct( + "TLSCiphertext", + UBInt8("type"), + ProtocolVersion, + UBInt16("length"), # TODO: Reject packets with length > 2 ** 14 + 2048 + Bytes("fragment", lambda ctx: ctx.length), +) + +Random = Struct( + "random", + UBInt32("gmt_unix_time"), + Bytes("random_bytes", 28), +) + +SessionID = Struct( + "session_id", + UBInt8("length"), + Bytes("session_id", lambda ctx: ctx.length), +) + +CipherSuites = Struct( + "cipher_suites", + UBInt16("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length // 2, UBInt16("cipher_suites")), +) + +CompressionMethods = Struct( + "compression_methods", + UBInt8("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length, UBInt8("compression_methods")), +) + +ServerName = Struct( + "", + UBInt8("type"), + PascalString("name", length_field=UBInt16("length")), +) + +SNIExtension = Struct( + "", + TunnelAdapter( + PascalString("server_names", length_field=UBInt16("length")), + TunnelAdapter( + PascalString("", length_field=UBInt16("length")), + GreedyRange(ServerName) + ), + ), +) + +ALPNExtension = Struct( + "", + TunnelAdapter( + PascalString("alpn_protocols", length_field=UBInt16("length")), + TunnelAdapter( + PascalString("", length_field=UBInt16("length")), + GreedyRange(PascalString("name")) + ), + ), +) + +UnknownExtension = Struct( + "", + PascalString("bytes", length_field=UBInt16("extensions_length")) +) + +Extension = Struct( + "Extension", + UBInt16("type"), + Embed( + Switch( + "", lambda ctx: ctx.type, + { + 0x00: SNIExtension, + 0x10: ALPNExtension + }, + default=UnknownExtension + ) + ) +) + +extensions = TunnelAdapter( + PascalString("extensions", length_field=UBInt16("extensions_length")), + OptionalGreedyRange(Extension) +) + +ClientHello = Struct( + "ClientHello", + ProtocolVersion, + Random, + SessionID, + CipherSuites, + CompressionMethods, + extensions, +) + +ServerHello = Struct( + "ServerHello", + ProtocolVersion, + Random, + SessionID, + Bytes("cipher_suite", 2), + UBInt8("compression_method"), + extensions, +) + +ClientCertificateType = Struct( + "certificate_types", + UBInt8("length"), # TODO: Reject packets of length 0 + Array(lambda ctx: ctx.length, UBInt8("certificate_types")), +) + +SignatureAndHashAlgorithm = Struct( + "algorithms", + UBInt8("hash"), + UBInt8("signature"), +) + +SupportedSignatureAlgorithms = Struct( + "supported_signature_algorithms", + UBInt16("supported_signature_algorithms_length"), + # TODO: Reject packets of length 0 + Array( + lambda ctx: ctx.supported_signature_algorithms_length / 2, + SignatureAndHashAlgorithm, + ), +) + +DistinguishedName = Struct( + "certificate_authorities", + UBInt16("length"), + Bytes("certificate_authorities", lambda ctx: ctx.length), +) + +CertificateRequest = Struct( + "CertificateRequest", + ClientCertificateType, + SupportedSignatureAlgorithms, + DistinguishedName, +) + +ServerDHParams = Struct( + "ServerDHParams", + UBInt16("dh_p_length"), + Bytes("dh_p", lambda ctx: ctx.dh_p_length), + UBInt16("dh_g_length"), + Bytes("dh_g", lambda ctx: ctx.dh_g_length), + UBInt16("dh_Ys_length"), + Bytes("dh_Ys", lambda ctx: ctx.dh_Ys_length), +) + +PreMasterSecret = Struct( + "pre_master_secret", + ProtocolVersion, + Bytes("random_bytes", 46), +) + +ASN1Cert = Struct( + "ASN1Cert", + UBInt32("length"), # TODO: Reject packets with length not in 1..2^24-1 + Bytes("asn1_cert", lambda ctx: ctx.length), +) + +Certificate = Struct( + "Certificate", # TODO: Reject packets with length > 2 ** 24 - 1 + UBInt32("certificates_length"), + Bytes("certificates_bytes", lambda ctx: ctx.certificates_length), +) + +Handshake = Struct( + "Handshake", + UBInt8("msg_type"), + UBInt24("length"), + Bytes("body", lambda ctx: ctx.length), +) + +Alert = Struct( + "Alert", + UBInt8("level"), + UBInt8("description"), +) diff --git a/libmproxy/contrib/tls/utils.py b/libmproxy/contrib/tls/utils.py new file mode 100644 index 00000000..4c917303 --- /dev/null +++ b/libmproxy/contrib/tls/utils.py @@ -0,0 +1,26 @@ +# This file is dual licensed under the terms of the Apache License, Version +# 2.0, and the BSD License. See the LICENSE file in the root of this repository +# for complete details. + +from __future__ import absolute_import, division, print_function + +import construct + +import six + + +class _UBInt24(construct.Adapter): + def _encode(self, obj, context): + return ( + six.int2byte((obj & 0xFF0000) >> 16) + + six.int2byte((obj & 0x00FF00) >> 8) + + six.int2byte(obj & 0x0000FF) + ) + + def _decode(self, obj, context): + obj = bytearray(obj) + return (obj[0] << 16 | obj[1] << 8 | obj[2]) + + +def UBInt24(name): # noqa + return _UBInt24(construct.Bytes(name, 3)) diff --git a/libmproxy/exceptions.py b/libmproxy/exceptions.py new file mode 100644 index 00000000..f34d9707 --- /dev/null +++ b/libmproxy/exceptions.py @@ -0,0 +1,34 @@ +from __future__ import (absolute_import, print_function, division) + + +class ProxyException(Exception): + """ + Base class for all exceptions thrown by libmproxy. + """ + def __init__(self, message, cause=None): + """ + :param message: Error Message + :param cause: Exception object that caused this exception to be thrown. + """ + super(ProxyException, self).__init__(message) + self.cause = cause + + +class ProtocolException(ProxyException): + pass + + +class Socks5Exception(ProtocolException): + pass + + +class HttpException(ProtocolException): + pass + + +class InvalidCredentials(HttpException): + pass + + +class ServerException(ProxyException): + pass diff --git a/libmproxy/filt.py b/libmproxy/filt.py index bd17a807..cfd3a1bc 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -35,7 +35,7 @@ from __future__ import absolute_import import re import sys import pyparsing as pp -from .protocol.http import decoded +from .models import decoded class _Token: @@ -246,14 +246,14 @@ class FSrc(_Rex): help = "Match source address" def __call__(self, f): - return f.client_conn and re.search(self.expr, repr(f.client_conn.address)) + return f.client_conn.address and re.search(self.expr, repr(f.client_conn.address)) class FDst(_Rex): code = "dst" help = "Match destination address" def __call__(self, f): - return f.server_conn and re.search(self.expr, repr(f.server_conn.address)) + return f.server_conn.address and re.search(self.expr, repr(f.server_conn.address)) class _Int(_Action): def __init__(self, num): diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 3d9ef722..5eac8da9 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,17 +8,18 @@ import Cookie import cookielib import os import re +import urlparse + -from netlib import odict, wsgi, tcp +from netlib import odict, wsgi from netlib.http.semantics import CONTENT_MISSING import netlib.http - -from . import controller, protocol, tnetstring, filt, script, version +from . import controller, tnetstring, filt, script, version from .onboarding import app -from .protocol import http, handle from .proxy.config import HostMatcher -from .proxy.connection import ClientConnection, ServerConnection -import urlparse +from .protocol.http_replay import RequestReplayThread +from .protocol import Kill +from .models import ClientConnection, ServerConnection, HTTPResponse, HTTPFlow, HTTPRequest class AppRegistry: @@ -788,7 +789,7 @@ class FlowMaster(controller.Master): rflow = self.server_playback.next_flow(flow) if not rflow: return None - response = http.HTTPResponse.from_state(rflow.response.get_state()) + response = HTTPResponse.from_state(rflow.response.get_state()) response.is_replay = True if self.refresh_server_playback: response.refresh() @@ -834,10 +835,10 @@ class FlowMaster(controller.Master): sni=host, ssl_established=True )) - f = http.HTTPFlow(c, s) + f = HTTPFlow(c, s) headers = odict.ODictCaseless() - req = http.HTTPRequest( + req = HTTPRequest( "absolute", method, scheme, @@ -859,9 +860,9 @@ class FlowMaster(controller.Master): """ if self.server and self.server.config.mode == "reverse": - f.request.host, f.request.port = self.server.config.mode.dst[2:] - f.request.scheme = "https" if self.server.config.mode.dst[ - 1] else "http" + f.request.host = self.server.config.upstream_server.address.host + f.request.port = self.server.config.upstream_server.address.port + f.request.scheme = re.sub("^https?2", "", self.server.config.upstream_server.scheme) f.reply = controller.DummyReply() if f.request: @@ -934,7 +935,7 @@ class FlowMaster(controller.Master): f.response = None f.error = None self.process_new_request(f) - rt = http.RequestReplayThread( + rt = RequestReplayThread( self.server.config, f, self.masterq if run_scripthooks else False, @@ -960,6 +961,10 @@ class FlowMaster(controller.Master): self.run_script_hook("serverconnect", sc) sc.reply() + def handle_serverdisconnect(self, sc): + self.run_script_hook("serverdisconnect", sc) + sc.reply() + def handle_error(self, f): self.state.update_flow(f) self.run_script_hook("error", f) @@ -979,7 +984,7 @@ class FlowMaster(controller.Master): ) if err: self.add_event("Error in wsgi app. %s" % err, "error") - f.reply(protocol.KILL) + f.reply(Kill) return if f not in self.state.flows: # don't add again on replay self.state.add_flow(f) @@ -996,7 +1001,7 @@ class FlowMaster(controller.Master): if self.stream_large_bodies: self.stream_large_bodies.run(f, False) except netlib.http.HttpError: - f.reply(protocol.KILL) + f.reply(Kill) return f.reply() @@ -1089,7 +1094,7 @@ class FlowReader: "Incompatible serialized data version: %s" % v ) off = self.fo.tell() - yield handle.protocols[data["type"]]["flow"].from_state(data) + yield HTTPFlow.from_state(data) except ValueError as v: # Error is due to EOF if self.fo.tell() == off and self.fo.read() == '': diff --git a/libmproxy/main.py b/libmproxy/main.py index abf3fb9c..23cb487c 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -2,11 +2,11 @@ from __future__ import print_function, absolute_import import os import signal import sys -import netlib.version from netlib.version_check import check_pyopenssl_version, check_mitmproxy_version from . import version, cmdline -from .proxy import process_proxy_options, ProxyServerError +from .exceptions import ServerException from .proxy.server import DummyServer, ProxyServer +from .proxy.config import process_proxy_options def assert_utf8_env(): @@ -31,7 +31,7 @@ def get_server(dummy_server, options): else: try: return ProxyServer(options) - except ProxyServerError as v: + except ServerException as v: print(str(v), file=sys.stderr) sys.exit(1) diff --git a/libmproxy/models/__init__.py b/libmproxy/models/__init__.py new file mode 100644 index 00000000..a54f305f --- /dev/null +++ b/libmproxy/models/__init__.py @@ -0,0 +1,16 @@ +from __future__ import (absolute_import, print_function, division) + +from .http import ( + HTTPFlow, HTTPRequest, HTTPResponse, decoded, + make_error_response, make_connect_request, make_connect_response +) +from .connections import ClientConnection, ServerConnection +from .flow import Flow, Error + +__all__ = [ + "HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded", + "make_error_response", "make_connect_request", + "make_connect_response", + "ClientConnection", "ServerConnection", + "Flow", "Error", +] diff --git a/libmproxy/proxy/connection.py b/libmproxy/models/connections.py index 9e03157a..f1e10de9 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/models/connections.py @@ -1,6 +1,8 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) + import copy import os + from netlib import tcp, certutils from .. import stateobject, utils @@ -10,7 +12,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): # Eventually, this object is restored from state. We don't have a # connection then. if client_connection: - tcp.BaseHandler.__init__(self, client_connection, address, server) + super(ClientConnection, self).__init__(client_connection, address, server) else: self.connection = None self.server = None @@ -25,6 +27,9 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): self.timestamp_ssl_setup = None self.protocol = None + def __nonzero__(self): + return bool(self.connection) and not self.finished + def __repr__(self): return "<ClientConnection: {ssl}{host}:{port}>".format( ssl="[ssl] " if self.ssl_established else "", @@ -32,6 +37,10 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): port=self.address.port ) + @property + def tls_established(self): + return self.ssl_established + _stateobject_attributes = dict( ssl_established=bool, timestamp_start=float, @@ -71,20 +80,11 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): return f def convert_to_ssl(self, *args, **kwargs): - # TODO: read ALPN from server and select same proto for client conn - # alpn_select = 'h2' - # def alpn_select_callback(conn_, options): - # if alpn_select in options: - # return bytes(alpn_select) - # else: # pragma no cover - # return options[0] - # tcp.BaseHandler.convert_to_ssl(self, alpn_select=alpn_select_callback, *args, **kwargs) - - tcp.BaseHandler.convert_to_ssl(self, *args, **kwargs) + super(ClientConnection, self).convert_to_ssl(*args, **kwargs) self.timestamp_ssl_setup = utils.timestamp() def finish(self): - tcp.BaseHandler.finish(self) + super(ClientConnection, self).finish() self.timestamp_end = utils.timestamp() @@ -92,13 +92,16 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def __init__(self, address): tcp.TCPClient.__init__(self, address) - self.state = [] # a list containing (conntype, state) tuples + self.via = None self.timestamp_start = None self.timestamp_end = None self.timestamp_tcp_setup = None self.timestamp_ssl_setup = None self.protocol = None + def __nonzero__(self): + return bool(self.connection) and not self.finished + def __repr__(self): if self.ssl_established and self.sni: ssl = "[ssl: {0}] ".format(self.sni) @@ -112,8 +115,11 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): port=self.address.port ) + @property + def tls_established(self): + return self.ssl_established + _stateobject_attributes = dict( - state=list, timestamp_start=float, timestamp_end=float, timestamp_tcp_setup=float, @@ -131,8 +137,8 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): d.update( address={"address": self.address(), "use_ipv6": self.address.use_ipv6}, - source_address= ({"address": self.source_address(), - "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), + source_address=({"address": self.source_address(), + "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), cert=self.cert.to_pem() if self.cert else None ) return d @@ -176,9 +182,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): if os.path.exists(path): clientcert = path - # TODO: read ALPN from client and use same list for server conn - # self.convert_to_ssl(cert=clientcert, sni=sni, alpn_protos=[netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2], **kwargs) - self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) self.sni = sni self.timestamp_ssl_setup = utils.timestamp() @@ -186,3 +189,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def finish(self): tcp.TCPClient.finish(self) self.timestamp_end = utils.timestamp() + + +ServerConnection._stateobject_attributes["via"] = ServerConnection diff --git a/libmproxy/models/flow.py b/libmproxy/models/flow.py new file mode 100644 index 00000000..8eff18f4 --- /dev/null +++ b/libmproxy/models/flow.py @@ -0,0 +1,166 @@ +from __future__ import (absolute_import, print_function, division) +import copy +import uuid + +from .. import stateobject, utils, version +from .connections import ClientConnection, ServerConnection + + +class Error(stateobject.StateObject): + """ + An Error. + + This is distinct from an protocol error response (say, a HTTP code 500), + which is represented by a normal HTTPResponse object. This class is + responsible for indicating errors that fall outside of normal protocol + communications, like interrupted connections, timeouts, protocol errors. + + Exposes the following attributes: + + flow: Flow object + msg: Message describing the error + timestamp: Seconds since the epoch + """ + + def __init__(self, msg, timestamp=None): + """ + @type msg: str + @type timestamp: float + """ + self.flow = None # will usually be set by the flow backref mixin + self.msg = msg + self.timestamp = timestamp or utils.timestamp() + + _stateobject_attributes = dict( + msg=str, + timestamp=float + ) + + def __str__(self): + return self.msg + + @classmethod + def from_state(cls, state): + # the default implementation assumes an empty constructor. Override + # accordingly. + f = cls(None) + f.load_state(state) + return f + + def copy(self): + c = copy.copy(self) + return c + + +class Flow(stateobject.StateObject): + """ + A Flow is a collection of objects representing a single transaction. + This class is usually subclassed for each protocol, e.g. HTTPFlow. + """ + + def __init__(self, type, client_conn, server_conn, live=None): + self.type = type + self.id = str(uuid.uuid4()) + self.client_conn = client_conn + """@type: ClientConnection""" + self.server_conn = server_conn + """@type: ServerConnection""" + self.live = live + """@type: LiveConnection""" + + self.error = None + """@type: Error""" + self.intercepted = False + """@type: bool""" + self._backup = None + self.reply = None + + _stateobject_attributes = dict( + id=str, + error=Error, + client_conn=ClientConnection, + server_conn=ServerConnection, + type=str, + intercepted=bool + ) + + def get_state(self, short=False): + d = super(Flow, self).get_state(short) + d.update(version=version.IVERSION) + if self._backup and self._backup != d: + if short: + d.update(modified=True) + else: + d.update(backup=self._backup) + return d + + def __eq__(self, other): + return self is other + + def copy(self): + f = copy.copy(self) + + f.id = str(uuid.uuid4()) + f.live = False + f.client_conn = self.client_conn.copy() + f.server_conn = self.server_conn.copy() + + if self.error: + f.error = self.error.copy() + return f + + def modified(self): + """ + Has this Flow been modified? + """ + if self._backup: + return self._backup != self.get_state() + else: + return False + + def backup(self, force=False): + """ + Save a backup of this Flow, which can be reverted to using a + call to .revert(). + """ + if not self._backup: + self._backup = self.get_state() + + def revert(self): + """ + Revert to the last backed up state. + """ + if self._backup: + self.load_state(self._backup) + self._backup = None + + def kill(self, master): + """ + Kill this request. + """ + from ..protocol import Kill + + self.error = Error("Connection killed") + self.intercepted = False + self.reply(Kill) + master.handle_error(self) + + def intercept(self, master): + """ + Intercept this Flow. Processing will stop until accept_intercept is + called. + """ + if self.intercepted: + return + self.intercepted = True + master.handle_intercept(self) + + def accept_intercept(self, master): + """ + Continue with the flow - called after an intercept(). + """ + if not self.intercepted: + return + self.intercepted = False + self.reply() + master.handle_accept_intercept(self) diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/models/http.py index ed5759ea..fb2f305b 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/models/http.py @@ -1,47 +1,16 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) import Cookie import copy -import threading -import time -import urllib -import urlparse from email.utils import parsedate_tz, formatdate, mktime_tz +import time -import netlib -from netlib import http, tcp, odict, utils, encoding -from netlib.http import cookies, semantics, http1 - -from .tcp import TCPHandler -from .primitives import KILL, ProtocolHandler, Flow, Error -from ..proxy.connection import ServerConnection -from .. import utils, controller, stateobject, proxy - - -class decoded(object): - """ - A context manager that decodes a request or response, and then - re-encodes it with the same encoding after execution of the block. - - Example: - with decoded(request): - request.content = request.content.replace("foo", "bar") - """ - - def __init__(self, o): - self.o = o - ce = o.headers.get_first("content-encoding") - if ce in encoding.ENCODINGS: - self.ce = ce - else: - self.ce = None - - def __enter__(self): - if self.ce: - self.o.decode() - - def __exit__(self, type, value, tb): - if self.ce: - self.o.encode(self.ce) +from libmproxy import utils +from netlib import odict, encoding +from netlib.http import status_codes +from netlib.tcp import Address +from netlib.http.semantics import Request, Response, CONTENT_MISSING +from .. import version, stateobject +from .flow import Flow class MessageMixin(stateobject.StateObject): @@ -124,7 +93,7 @@ class MessageMixin(stateobject.StateObject): return c -class HTTPRequest(MessageMixin, semantics.Request): +class HTTPRequest(MessageMixin, Request): """ An HTTP request. @@ -170,21 +139,21 @@ class HTTPRequest(MessageMixin, semantics.Request): """ def __init__( - self, - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - body, - timestamp_start=None, - timestamp_end=None, - form_out=None, + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + body, + timestamp_start=None, + timestamp_end=None, + form_out=None, ): - semantics.Request.__init__( + Request.__init__( self, form_in, method, @@ -240,31 +209,15 @@ class HTTPRequest(MessageMixin, semantics.Request): def from_protocol( self, protocol, - include_body=True, - body_size_limit=None, + *args, + **kwargs ): - req = protocol.read_request( - include_body = include_body, - body_size_limit = body_size_limit, - ) - - return HTTPRequest( - req.form_in, - req.method, - req.scheme, - req.host, - req.port, - req.path, - req.httpversion, - req.headers, - req.body, - req.timestamp_start, - req.timestamp_end, - ) + req = protocol.read_request(*args, **kwargs) + return self.wrap(req) @classmethod def wrap(self, request): - return HTTPRequest( + req = HTTPRequest( form_in=request.form_in, method=request.method, scheme=request.scheme, @@ -278,6 +231,9 @@ class HTTPRequest(MessageMixin, semantics.Request): timestamp_end=request.timestamp_end, form_out=(request.form_out if hasattr(request, 'form_out') else None), ) + if hasattr(request, 'stream_id'): + req.stream_id = request.stream_id + return req def __hash__(self): return id(self) @@ -298,7 +254,7 @@ class HTTPRequest(MessageMixin, semantics.Request): return c -class HTTPResponse(MessageMixin, semantics.Response): +class HTTPResponse(MessageMixin, Response): """ An HTTP response. @@ -331,7 +287,7 @@ class HTTPResponse(MessageMixin, semantics.Response): timestamp_start=None, timestamp_end=None, ): - semantics.Response.__init__( + Response.__init__( self, httpversion, status_code, @@ -362,29 +318,15 @@ class HTTPResponse(MessageMixin, semantics.Response): def from_protocol( self, protocol, - request_method, - include_body=True, - body_size_limit=None + *args, + **kwargs ): - resp = protocol.read_response( - request_method, - body_size_limit, - include_body=include_body - ) - - return HTTPResponse( - resp.httpversion, - resp.status_code, - resp.msg, - resp.headers, - resp.body, - resp.timestamp_start, - resp.timestamp_end, - ) + resp = protocol.read_response(*args, **kwargs) + return self.wrap(resp) @classmethod def wrap(self, response): - return HTTPResponse( + resp = HTTPResponse( httpversion=response.httpversion, status_code=response.status_code, msg=response.msg, @@ -393,6 +335,9 @@ class HTTPResponse(MessageMixin, semantics.Response): timestamp_start=response.timestamp_start, timestamp_end=response.timestamp_end, ) + if hasattr(response, 'stream_id'): + resp.stream_id = response.stream_id + return resp def _refresh_cookie(self, c, delta): """ @@ -443,3 +388,167 @@ class HTTPResponse(MessageMixin, semantics.Response): c.append(self._refresh_cookie(i, delta)) if c: self.headers["set-cookie"] = c + + +class HTTPFlow(Flow): + """ + A HTTPFlow is a collection of objects representing a single HTTP + transaction. The main attributes are: + + request: HTTPRequest object + response: HTTPResponse object + error: Error object + server_conn: ServerConnection object + client_conn: ClientConnection object + + Note that it's possible for a Flow to have both a response and an error + object. This might happen, for instance, when a response was received + from the server, but there was an error sending it back to the client. + + The following additional attributes are exposed: + + intercepted: Is this flow currently being intercepted? + live: Does this flow have a live client connection? + """ + + def __init__(self, client_conn, server_conn, live=None): + super(HTTPFlow, self).__init__("http", client_conn, server_conn, live) + self.request = None + """@type: HTTPRequest""" + self.response = None + """@type: HTTPResponse""" + + _stateobject_attributes = Flow._stateobject_attributes.copy() + _stateobject_attributes.update( + request=HTTPRequest, + response=HTTPResponse + ) + + @classmethod + def from_state(cls, state): + f = cls(None, None) + f.load_state(state) + return f + + def __repr__(self): + s = "<HTTPFlow" + for a in ("request", "response", "error", "client_conn", "server_conn"): + if getattr(self, a, False): + s += "\r\n %s = {flow.%s}" % (a, a) + s += ">" + return s.format(flow=self) + + def copy(self): + f = super(HTTPFlow, self).copy() + if self.request: + f.request = self.request.copy() + if self.response: + f.response = self.response.copy() + return f + + def match(self, f): + """ + Match this flow against a compiled filter expression. Returns True + if matched, False if not. + + If f is a string, it will be compiled as a filter expression. If + the expression is invalid, ValueError is raised. + """ + if isinstance(f, basestring): + from .. import filt + + f = filt.parse(f) + if not f: + raise ValueError("Invalid filter expression.") + if f: + return f(self) + return True + + def replace(self, pattern, repl, *args, **kwargs): + """ + Replaces a regular expression pattern with repl in both request and + response of the flow. Encoded content will be decoded before + replacement, and re-encoded afterwards. + + Returns the number of replacements made. + """ + c = self.request.replace(pattern, repl, *args, **kwargs) + if self.response: + c += self.response.replace(pattern, repl, *args, **kwargs) + return c + + +class decoded(object): + """ + A context manager that decodes a request or response, and then + re-encodes it with the same encoding after execution of the block. + + Example: + with decoded(request): + request.content = request.content.replace("foo", "bar") + """ + + def __init__(self, o): + self.o = o + ce = o.headers.get_first("content-encoding") + if ce in encoding.ENCODINGS: + self.ce = ce + else: + self.ce = None + + def __enter__(self): + if self.ce: + self.o.decode() + + def __exit__(self, type, value, tb): + if self.ce: + self.o.encode(self.ce) + + +def make_error_response(status_code, message, headers=None): + response = status_codes.RESPONSES.get(status_code, "Unknown") + body = """ + <html> + <head> + <title>%d %s</title> + </head> + <body>%s</body> + </html> + """.strip() % (status_code, response, message) + + if not headers: + headers = odict.ODictCaseless() + headers["Server"] = [version.NAMEVERSION] + headers["Connection"] = ["close"] + headers["Content-Length"] = [len(body)] + headers["Content-Type"] = ["text/html"] + + return HTTPResponse( + (1, 1), # FIXME: Should be a string. + status_code, + response, + headers, + body, + ) + + +def make_connect_request(address): + address = Address.wrap(address) + return HTTPRequest( + "authority", "CONNECT", None, address.host, address.port, None, (1, 1), + odict.ODictCaseless(), "" + ) + + +def make_connect_response(httpversion): + headers = odict.ODictCaseless([ + ["Content-Length", "0"], + ["Proxy-Agent", version.NAMEVERSION] + ]) + return HTTPResponse( + httpversion, + 200, + "Connection established", + headers, + "", + ) diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index bbc20dba..c582592b 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -1 +1,12 @@ -from .primitives import * +from __future__ import (absolute_import, print_function, division) +from .base import Layer, ServerConnectionMixin, Log, Kill +from .http import Http1Layer, Http2Layer +from .tls import TlsLayer, is_tls_record_magic +from .rawtcp import RawTCPLayer + +__all__ = [ + "Layer", "ServerConnectionMixin", "Log", "Kill", + "Http1Layer", "Http2Layer", + "TlsLayer", "is_tls_record_magic", + "RawTCPLayer" +] diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py new file mode 100644 index 00000000..40ec0536 --- /dev/null +++ b/libmproxy/protocol/base.py @@ -0,0 +1,179 @@ +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible settings (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentProxy + Http1Layer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + HttpProxy + Http1Layer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentProxy + TLSLayer + Http2Layer + HttpLayer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can call functions provided by lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" +from __future__ import (absolute_import, print_function, division) +from netlib import tcp +from ..models import ServerConnection +from ..exceptions import ProtocolException + + +class _LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + + def __init__(self, *args, **kwargs): # pragma: nocover + super(_LayerCodeCompletion, self).__init__(*args, **kwargs) + if True: + return + self.config = None + """@type: libmproxy.proxy.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.models.ClientConnection""" + self.server_conn = None + """@type: libmproxy.models.ServerConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(_LayerCodeCompletion): + def __init__(self, ctx, *args, **kwargs): + """ + Args: + ctx: The (read-only) higher layer. + """ + self.ctx = ctx + """@type: libmproxy.protocol.Layer""" + super(Layer, self).__init__(*args, **kwargs) + + def __call__(self): + """ + Logic of the layer. + Raises: + ProtocolException in case of protocol exceptions. + """ + raise NotImplementedError() + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "{}: {}".format(repr(self.client_conn.address), msg) + ] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + @property + def layers(self): + return [self] + self.ctx.layers + + def __repr__(self): + return type(self).__name__ + + +class ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + + def __init__(self, server_address=None): + super(ServerConnectionMixin, self).__init__() + self.server_conn = ServerConnection(server_address) + self._check_self_connect() + + def reconnect(self): + address = self.server_conn.address + self._disconnect() + self.server_conn.address = address + self.connect() + + def _check_self_connect(self): + """ + We try to protect the proxy from _accidentally_ connecting to itself, + e.g. because of a failed transparent lookup or an invalid configuration. + """ + address = self.server_conn.address + if address: + self_connect = ( + address.port == self.config.port and + address.host in ("localhost", "127.0.0.1", "::1") + ) + if self_connect: + raise ProtocolException( + "Invalid server address: {}\r\n" + "The proxy shall not connect to itself.".format(repr(address)) + ) + + def set_server(self, address, server_tls=None, sni=None, depth=1): + if depth == 1: + if self.server_conn: + self._disconnect() + self.log("Set new server address: " + repr(address), "debug") + self.server_conn.address = address + self._check_self_connect() + if server_tls: + raise ProtocolException( + "Cannot upgrade to TLS, no TLS layer on the protocol stack." + ) + else: + self.ctx.set_server(address, server_tls, sni, depth - 1) + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + self.server_conn.finish() + self.server_conn.close() + self.channel.tell("serverdisconnect", self.server_conn) + self.server_conn = ServerConnection(None) + + def connect(self): + if not self.server_conn.address: + raise ProtocolException("Cannot connect to server, no server address given.") + self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + self.channel.ask("serverconnect", self.server_conn) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProtocolException( + "Server connection to %s failed: %s" % (repr(self.server_conn.address), e), e) + + +class Log(object): + def __init__(self, msg, level="info"): + self.msg = msg + self.level = level + + +class Kill(Exception): + """ + Kill a connection. + """ diff --git a/libmproxy/protocol/handle.py b/libmproxy/protocol/handle.py deleted file mode 100644 index 49cb3c1b..00000000 --- a/libmproxy/protocol/handle.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import absolute_import -from . import http, tcp - -protocols = { - 'http': dict(handler=http.HTTPHandler, flow=http.HTTPFlow), - 'tcp': dict(handler=tcp.TCPHandler) -} - - -def protocol_handler(protocol): - """ - @type protocol: str - @returns: libmproxy.protocol.primitives.ProtocolHandler - """ - if protocol in protocols: - return protocols[protocol]["handler"] - - raise NotImplementedError( - "Unknown Protocol: %s" % - protocol) # pragma: nocover diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4c15c80d..7f57d17c 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,606 +1,347 @@ -from __future__ import absolute_import -import Cookie -import copy -import threading -import time -import urllib -import urlparse -from email.utils import parsedate_tz, formatdate, mktime_tz - -import netlib -from netlib import http, tcp, odict, utils, encoding -from netlib.http import cookies, http1, http2 +from __future__ import (absolute_import, print_function, division) + +from netlib import tcp +from netlib.http import http1, HttpErrorConnClosed, HttpError from netlib.http.semantics import CONTENT_MISSING +from netlib import odict +from netlib.tcp import NetLibError, Address +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol +from netlib.http.http2.frame import WindowUpdateFrame -from .tcp import TCPHandler -from .primitives import KILL, ProtocolHandler, Flow, Error -from ..proxy.connection import ServerConnection -from .. import utils, controller, stateobject, proxy - -from .http_wrappers import decoded, HTTPRequest, HTTPResponse - - -class KillSignal(Exception): - pass - - -def send_connect_request(conn, host, port, update_state=True): - upstream_request = HTTPRequest( - "authority", - "CONNECT", - None, - host, - port, - None, - (1, 1), - odict.ODictCaseless(), - "" - ) - - # we currently only support HTTP/1 CONNECT requests - protocol = http1.HTTP1Protocol(conn) - - conn.send(protocol.assemble(upstream_request)) - resp = HTTPResponse.from_protocol(protocol, upstream_request.method) - if resp.status_code != 200: - raise proxy.ProxyError(resp.status_code, - "Cannot establish SSL " + - "connection with upstream proxy: \r\n" + - repr(resp)) - if update_state: - conn.state.append(("http", { - "state": "connect", - "host": host, - "port": port} - )) - return resp - - -class HTTPFlow(Flow): - """ - A HTTPFlow is a collection of objects representing a single HTTP - transaction. The main attributes are: +from .. import utils +from ..exceptions import InvalidCredentials, HttpException, ProtocolException +from ..models import ( + HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error +) +from .base import Layer, Kill - request: HTTPRequest object - response: HTTPResponse object - error: Error object - server_conn: ServerConnection object - client_conn: ClientConnection object - Note that it's possible for a Flow to have both a response and an error - object. This might happen, for instance, when a response was received - from the server, but there was an error sending it back to the client. +class _HttpLayer(Layer): + supports_streaming = False - The following additional attributes are exposed: + def read_request(self): + raise NotImplementedError() - intercepted: Is this flow currently being intercepted? - live: Does this flow have a live client connection? - """ + def send_request(self, request): + raise NotImplementedError() - def __init__(self, client_conn, server_conn, live=None): - super(HTTPFlow, self).__init__("http", client_conn, server_conn, live) - self.request = None - """@type: HTTPRequest""" - self.response = None - """@type: HTTPResponse""" - - _stateobject_attributes = Flow._stateobject_attributes.copy() - _stateobject_attributes.update( - request=HTTPRequest, - response=HTTPResponse - ) - - @classmethod - def from_state(cls, state): - f = cls(None, None) - f.load_state(state) - return f - - def __repr__(self): - s = "<HTTPFlow" - for a in ("request", "response", "error", "client_conn", "server_conn"): - if getattr(self, a, False): - s += "\r\n %s = {flow.%s}" % (a, a) - s += ">" - return s.format(flow=self) - - def copy(self): - f = super(HTTPFlow, self).copy() - if self.request: - f.request = self.request.copy() - if self.response: - f.response = self.response.copy() - return f - - def match(self, f): - """ - Match this flow against a compiled filter expression. Returns True - if matched, False if not. + def read_response(self, request_method): + raise NotImplementedError() - If f is a string, it will be compiled as a filter expression. If - the expression is invalid, ValueError is raised. - """ - if isinstance(f, basestring): - from .. import filt + def send_response(self, response): + raise NotImplementedError() - f = filt.parse(f) - if not f: - raise ValueError("Invalid filter expression.") - if f: - return f(self) - return True - def replace(self, pattern, repl, *args, **kwargs): - """ - Replaces a regular expression pattern with repl in both request and - response of the flow. Encoded content will be decoded before - replacement, and re-encoded afterwards. +class _StreamingHttpLayer(_HttpLayer): + supports_streaming = True - Returns the number of replacements made. - """ - c = self.request.replace(pattern, repl, *args, **kwargs) - if self.response: - c += self.response.replace(pattern, repl, *args, **kwargs) - return c + def read_response_headers(self): + raise NotImplementedError + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + raise NotImplementedError() + yield "this is a generator" # pragma: no cover -class HTTPHandler(ProtocolHandler): - """ - HTTPHandler implements mitmproxys understanding of the HTTP protocol. + def send_response_headers(self, response): + raise NotImplementedError - """ + def send_response_body(self, response, chunks): + raise NotImplementedError() - def __init__(self, c): - super(HTTPHandler, self).__init__(c) - self.expected_form_in = c.config.mode.http_form_in - self.expected_form_out = c.config.mode.http_form_out - self.skip_authentication = False - def handle_messages(self): - while self.handle_flow(): - pass +class Http1Layer(_StreamingHttpLayer): + def __init__(self, ctx, mode): + super(Http1Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP1Protocol(self.client_conn) + self.server_protocol = HTTP1Protocol(self.server_conn) - def get_response_from_server(self, flow): - self.c.establish_server_connection() + def read_request(self): + return HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) - for attempt in (0, 1): - try: - if not self.c.server_conn.protocol: - # instantiate new protocol if connection does not have one yet - # TODO: select correct protocol based on ALPN (?) - self.c.server_conn.protocol = http1.HTTP1Protocol(self.c.server_conn) - # self.c.server_conn.protocol = http2.HTTP2Protocol(self.c.server_conn) - # self.c.server_conn.protocol.perform_connection_preface() - - self.c.server_conn.send(self.c.server_conn.protocol.assemble(flow.request)) - - # Only get the headers at first... - flow.response = HTTPResponse.from_protocol( - self.c.server_conn.protocol, - flow.request.method, - body_size_limit=self.c.config.body_size_limit, - include_body=False, - ) - break - except (tcp.NetLibError, http.HttpErrorConnClosed) as v: - self.c.log( - "error in server communication: %s" % repr(v), - level="debug" - ) - if attempt == 0: - # In any case, we try to reconnect at least once. This is - # necessary because it might be possible that we already - # initiated an upstream connection after clientconnect that - # has already been expired, e.g consider the following event - # log: - # > clientconnect (transparent mode destination known) - # > serverconnect - # > read n% of large request - # > server detects timeout, disconnects - # > read (100-n)% of large request - # > send large request upstream - self.c.server_reconnect() - else: - raise + def send_request(self, request): + self.server_conn.send(self.server_protocol.assemble(request)) - # call the appropriate script hook - this is an opportunity for an - # inline script to set flow.stream = True - flow = self.c.channel.ask("responseheaders", flow) - if flow is None or flow == KILL: - raise KillSignal() - else: - # now get the rest of the request body, if body still needs to be - # read but not streaming this response - if flow.response.stream: - flow.response.content = CONTENT_MISSING - else: - if isinstance(self.c.server_conn.protocol, http1.HTTP1Protocol): - # streaming is only supported with HTTP/1 at the moment - flow.response.content = self.c.server_conn.protocol.read_http_body( - flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, - flow.response.code, - False - ) - flow.response.timestamp_end = utils.timestamp() - - def handle_flow(self): - flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live) + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True + ) - try: - try: - if not flow.client_conn.protocol: - # instantiate new protocol if connection does not have one yet - # the first request might be a CONNECT - which is currently only supported with HTTP/1 - flow.client_conn.protocol = http1.HTTP1Protocol(self.c.client_conn) - - req = HTTPRequest.from_protocol( - flow.client_conn.protocol, - body_size_limit=self.c.config.body_size_limit - ) - except tcp.NetLibError: - # don't throw an error for disconnects that happen - # before/between requests. - return False - - self.c.log( - "request", - "debug", - [repr(req)] - ) - ret = self.process_request(flow, req) - if ret: - # instantiate new protocol if connection does not have one yet - # TODO: select correct protocol based on ALPN (?) - flow.client_conn.protocol = http1.HTTP1Protocol(self.c.client_conn) - # flow.client_conn.protocol = http2.HTTP2Protocol(self.c.client_conn, is_server=True) - if ret is not None: - return ret - - # Be careful NOT to assign the request to the flow before - # process_request completes. This is because the call can raise an - # exception. If the request object is already attached, this results - # in an Error object that has an attached request that has not been - # sent through to the Master. - flow.request = req - request_reply = self.c.channel.ask("request", flow) - if request_reply is None or request_reply == KILL: - raise KillSignal() - - # The inline script may have changed request.host - self.process_server_address(flow) - - if isinstance(request_reply, HTTPResponse): - flow.response = request_reply - else: - self.get_response_from_server(flow) + def send_response(self, response): + self.client_conn.send(self.client_protocol.assemble(response)) - # no further manipulation of self.c.server_conn beyond this point - # we can safely set it as the final attribute value here. - flow.server_conn = self.c.server_conn + def read_response_headers(self): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=None, # does not matter if we don't read the body. + body_size_limit=self.config.body_size_limit, + include_body=False + ) - self.c.log( - "response", - "debug", - [repr(flow.response)] - ) - response_reply = self.c.channel.ask("response", flow) - if response_reply is None or response_reply == KILL: - raise KillSignal() - - self.send_response_to_client(flow) - - if self.check_close_connection(flow): - return False - - # We sent a CONNECT request to an upstream proxy. - if flow.request.form_in == "authority" and flow.response.code == 200: - # TODO: Possibly add headers (memory consumption/usefulness - # tradeoff) Make sure to add state info before the actual - # processing of the CONNECT request happens. During an SSL - # upgrade, we may receive an SNI indication from the client, - # which resets the upstream connection. If this is the case, we - # must already re-issue the CONNECT request at this point. - self.c.server_conn.state.append( - ( - "http", { - "state": "connect", - "host": flow.request.host, - "port": flow.request.port - } - ) - ) - if not self.process_connect_request( - (flow.request.host, flow.request.port)): - return False - - # If the user has changed the target server on this connection, - # restore the original target server - flow.live.restore_server() - - return True # Next flow please. - except ( - http.HttpAuthenticationError, - http.HttpError, - proxy.ProxyError, - tcp.NetLibError, - ) as e: - self.handle_error(e, flow) - except KillSignal: - self.c.log("Connection killed", "info") - finally: - flow.live = None # Connection is not live anymore. - return False - - def handle_server_reconnect(self, state): - if state["state"] == "connect": - send_connect_request( - self.c.server_conn, - state["host"], - state["port"], - update_state=False - ) - else: # pragma: nocover - raise RuntimeError("Unknown State: %s" % state["state"]) - - def handle_error(self, error, flow=None): - message = repr(error) - message_debug = None - - if isinstance(error, tcp.NetLibError): - message = None - message_debug = "TCP connection closed unexpectedly." - elif "tlsv1 alert unknown ca" in message: - message = "TLSv1 Alert Unknown CA: The client does not trust the proxy's certificate." - elif "handshake error" in message: - message_debug = message - message = "SSL handshake error: The client may not trust the proxy's certificate." - - if message: - self.c.log(message, level="info") - if message_debug: - self.c.log(message_debug, level="debug") - - if flow: - # TODO: no flows without request or with both request and response - # at the moment. - if flow.request and not flow.response: - flow.error = Error(message or message_debug) - self.c.channel.ask("error", flow) - try: - status_code = getattr(error, "code", 502) - headers = getattr(error, "headers", None) - - html_message = message or "" - if message_debug: - html_message += "<pre>%s</pre>" % message_debug - self.send_error(status_code, html_message, headers) - except: - pass - - def send_error(self, status_code, message, headers): - response = http.status_codes.RESPONSES.get(status_code, "Unknown") - body = """ - <html> - <head> - <title>%d %s</title> - </head> - <body>%s</body> - </html> - """ % (status_code, response, message) - - if not headers: - headers = odict.ODictCaseless() - assert isinstance(headers, odict.ODictCaseless) - - headers["Server"] = [self.c.config.server_version] - headers["Connection"] = ["close"] - headers["Content-Length"] = [len(body)] - headers["Content-Type"] = ["text/html"] - - resp = HTTPResponse( - (1, 1), # if HTTP/2 is used, this value is ignored anyway - status_code, - response, + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + return self.server_protocol.read_http_body_chunked( headers, - body, + self.config.body_size_limit, + request_method, + response_code, + False, + max_chunk_size ) - # if no protocol is assigned yet - just assume HTTP/1 - # TODO: maybe check ALPN and use HTTP/2 if required? - protocol = self.c.client_conn.protocol or http1.HTTP1Protocol(self.c.client_conn) - self.c.client_conn.send(protocol.assemble(resp)) - - def process_request(self, flow, request): - """ - @returns: - True, if the request should not be sent upstream - False, if the connection should be aborted - None, if the request should be sent upstream - (a status code != None should be returned directly by handle_flow) - """ - - if not self.skip_authentication: - self.authenticate(request) + def send_response_headers(self, response): + h = self.client_protocol._assemble_response_first_line(response) + self.client_conn.wfile.write(h + "\r\n") + h = self.client_protocol._assemble_response_headers( + response, + preserve_transfer_encoding=True + ) + self.client_conn.send(h + "\r\n") - # Determine .scheme, .host and .port attributes - # For absolute-form requests, they are directly given in the request. - # For authority-form requests, we only need to determine the request scheme. - # For relative-form requests, we need to determine host and port as - # well. - if not request.scheme: - request.scheme = "https" if flow.server_conn and flow.server_conn.ssl_established else "http" - if not request.host: - # Host/Port Complication: In upstream mode, use the server we CONNECTed to, - # not the upstream proxy. - if flow.server_conn: - for s in flow.server_conn.state: - if s[0] == "http" and s[1]["state"] == "connect": - request.host, request.port = s[1]["host"], s[1]["port"] - if not request.host and flow.server_conn: - request.host, request.port = flow.server_conn.address.host, flow.server_conn.address.port - - - # Now we can process the request. - if request.form_in == "authority": - if self.c.client_conn.ssl_established: - raise http.HttpError( - 400, - "Must not CONNECT on already encrypted connection" - ) - - if self.c.config.mode == "regular": - self.c.set_server_address((request.host, request.port)) - # Update server_conn attribute on the flow - flow.server_conn = self.c.server_conn - - # since we currently only support HTTP/1 CONNECT requests - # the response must be HTTP/1 as well - self.c.client_conn.send( - ('HTTP/%s.%s 200 ' % (request.httpversion[0], request.httpversion[1])) + - 'Connection established\r\n' + - 'Content-Length: 0\r\n' + - ('Proxy-agent: %s\r\n' % self.c.config.server_version) + - '\r\n' - ) - return self.process_connect_request(self.c.server_conn.address) - elif self.c.config.mode == "upstream": - return None - else: - # CONNECT should never occur if we don't expect absolute-form - # requests - pass - - elif request.form_in == self.expected_form_in: - request.form_out = self.expected_form_out - if request.form_in == "absolute": - if request.scheme != "http": - raise http.HttpError( - 400, - "Invalid request scheme: %s" % request.scheme - ) - if self.c.config.mode == "regular": - # Update info so that an inline script sees the correct - # value at flow.server_conn - self.c.set_server_address((request.host, request.port)) - flow.server_conn = self.c.server_conn - - elif request.form_in == "relative": - if self.c.config.mode == "spoof": - # Host header - h = request.pretty_host(hostheader=True) - if h is None: - raise http.HttpError( - 400, - "Invalid request: No host information" - ) - p = netlib.utils.parse_url("http://" + h) - request.scheme = p[0] - request.host = p[1] - request.port = p[2] - self.c.set_server_address((request.host, request.port)) - flow.server_conn = self.c.server_conn - - if self.c.config.mode == "sslspoof": - # SNI is processed in server.py - if not (flow.server_conn and flow.server_conn.ssl_established): - raise http.HttpError( - 400, - "Invalid request: No host information" - ) - - return None - - raise http.HttpError( - 400, "Invalid HTTP request form (expected: %s, got: %s)" % ( - self.expected_form_in, request.form_in + def send_response_body(self, response, chunks): + if self.client_protocol.has_chunked_encoding(response.headers): + chunks = ( + "%d\r\n%s\r\n" % (len(chunk), chunk) + for chunk in chunks ) + for chunk in chunks: + self.client_conn.send(chunk) + + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP1Protocol(self.server_conn) + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP1Protocol(self.server_conn) + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def __call__(self): + layer = HttpLayer(self, self.mode) + layer() + + +# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. +class Http2Layer(_HttpLayer): + def __init__(self, ctx, mode): + super(Http2Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + + def read_request(self): + request = HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + self._stream_id = request.stream_id + return request + + def send_request(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.server_conn.send(self.server_protocol.assemble(message)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True, + stream_id=self._stream_id ) - def process_server_address(self, flow): - # Depending on the proxy mode, server handling is entirely different - # We provide a mostly unified API to the user, which needs to be - # unfiddled here - # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 ) - address = tcp.Address((flow.request.host, flow.request.port)) + def send_response(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.client_conn.send(self.client_protocol.assemble(message)) + + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def __call__(self): + self.server_protocol.perform_connection_preface() + layer = HttpLayer(self, self.mode) + layer() + + def handle_unexpected_frame(self, frame): + if isinstance(frame, WindowUpdateFrame): + # Clients are sending WindowUpdate frames depending on their flow control algorithm. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should keep track of our own flow control window and + # stall transmission if the outgoing flow control buffer is full. + return + self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") + + +class ConnectServerConnection(object): + """ + "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. + """ - ssl = (flow.request.scheme == "https") - - if self.c.config.mode == "upstream": - # The connection to the upstream proxy may have a state we may need - # to take into account. - connected_to = None - for s in flow.server_conn.state: - if s[0] == "http" and s[1]["state"] == "connect": - connected_to = tcp.Address((s[1]["host"], s[1]["port"])) - - # We need to reconnect if the current flow either requires a - # (possibly impossible) change to the connection state, e.g. the - # host has changed but we already CONNECTed somewhere else. - needs_server_change = ( - ssl != self.c.server_conn.ssl_established - or - # HTTP proxying is "stateless", CONNECT isn't. - (connected_to and address != connected_to) - ) + def __init__(self, address, ctx): + self.address = tcp.Address.wrap(address) + self._ctx = ctx - if needs_server_change: - # force create new connection to the proxy server to reset - # state - self.live.change_server(self.c.server_conn.address, force=True) - if ssl: - send_connect_request( - self.c.server_conn, - address.host, - address.port - ) - self.c.establish_ssl(server=True) - else: - # If we're not in upstream mode, we just want to update the host - # and possibly establish TLS. This is a no op if the addresses - # match. - self.live.change_server(address, ssl=ssl) + @property + def via(self): + return self._ctx.server_conn - flow.server_conn = self.c.server_conn + def __getattr__(self, item): + return getattr(self.via, item) - def send_response_to_client(self, flow): - if not flow.response.stream: - # no streaming: - # we already received the full response from the server and can - # send it to the client straight away. - self.c.client_conn.send(self.c.client_conn.protocol.assemble(flow.response)) + def __nonzero__(self): + return bool(self.via) + + +class UpstreamConnectLayer(Layer): + def __init__(self, ctx, connect_request): + super(UpstreamConnectLayer, self).__init__(ctx) + self.connect_request = connect_request + self.server_conn = ConnectServerConnection( + (connect_request.host, connect_request.port), + self.ctx + ) + + def __call__(self): + layer = self.ctx.next_layer(self) + layer() + + def _send_connect_request(self): + self.send_request(self.connect_request) + resp = self.read_response("CONNECT") + if resp.code != 200: + raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") + + def connect(self): + if not self.server_conn: + self.ctx.connect() + self._send_connect_request() else: - if isinstance(self.c.client_conn.protocol, http2.HTTP2Protocol): - raise NotImplementedError("HTTP streaming with HTTP/2 is currently not supported.") + pass # swallow the message + + def reconnect(self): + self.ctx.reconnect() + self._send_connect_request() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + if depth == 1: + if self.ctx.server_conn: + self.ctx.reconnect() + address = Address.wrap(address) + self.connect_request.host = address.host + self.connect_request.port = address.port + self.server_conn.address = address + else: + self.ctx.set_server(address, server_tls, sni, depth - 1) - # streaming: - # First send the headers and then transfer the response - # incrementally: - h = self.c.client_conn.protocol._assemble_response_first_line(flow.response) - self.c.client_conn.send(h + "\r\n") - h = self.c.client_conn.protocol._assemble_response_headers(flow.response, preserve_transfer_encoding=True) - self.c.client_conn.send(h + "\r\n") - - chunks = self.c.server_conn.protocol.read_http_body_chunked( - flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, - flow.response.code, - False, - 4096 - ) +class HttpLayer(Layer): + def __init__(self, ctx, mode): + super(HttpLayer, self).__init__(ctx) + self.mode = mode + self.__original_server_conn = None + "Contains the original destination in transparent mode, which needs to be restored" + "if an inline script modified the target server for a single http request" - if callable(flow.response.stream): - chunks = flow.response.stream(chunks) + def __call__(self): + if self.mode == "transparent": + self.__original_server_conn = self.server_conn + while True: + try: + flow = HTTPFlow(self.client_conn, self.server_conn, live=self) + + try: + request = self.read_request() + except tcp.NetLibError: + # don't throw an error for disconnects that happen + # before/between requests. + return + + self.log("request", "debug", [repr(request)]) + + # Handle Proxy Authentication + self.authenticate(request) + + # Regular Proxy Mode: Handle CONNECT + if self.mode == "regular" and request.form_in == "authority": + self.handle_regular_mode_connect(request) + return + + # Make sure that the incoming request matches our expectations + self.validate_request(request) + + flow.request = request + self.process_request_hook(flow) + + if not flow.response: + self.establish_server_connection(flow) + self.get_response_from_server(flow) + + self.send_response_to_client(flow) + + if self.check_close_connection(flow): + return + + # TODO: Implement HTTP Upgrade + + # Upstream Proxy Mode: Handle CONNECT + if flow.request.form_in == "authority" and flow.response.code == 200: + self.handle_upstream_mode_connect(flow.request.copy()) + return + + except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: + if flow.request and not flow.response: + flow.error = Error(repr(e)) + self.channel.ask("error", flow) + try: + self.send_response(make_error_response( + getattr(e, "code", 502), + repr(e) + )) + except NetLibError: + pass + if isinstance(e, ProtocolException): + raise e + else: + raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) + finally: + flow.live = False - for chunk in chunks: - for part in chunk: - self.c.client_conn.wfile.write(part) - self.c.client_conn.wfile.flush() + def handle_regular_mode_connect(self, request): + self.set_server((request.host, request.port)) + self.send_response(make_connect_response(request.httpversion)) + layer = self.ctx.next_layer(self) + layer() - flow.response.timestamp_end = utils.timestamp() + def handle_upstream_mode_connect(self, connect_request): + layer = UpstreamConnectLayer(self, connect_request) + layer() def check_close_connection(self, flow): """ @@ -622,157 +363,183 @@ class HTTPHandler(ProtocolHandler): False, flow.request.method, flow.response.code) == -1 - ) - if close_connection: - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 - # and no Content-Length header - pass - else: - return True - return False + ) + if flow.request.form_in == "authority" and flow.response.code == 200: + # Workaround for + # https://github.com/mitmproxy/mitmproxy/issues/313: Some + # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 + # and no Content-Length header - def process_connect_request(self, address): - """ - Process a CONNECT request. - Returns True if the CONNECT request has been processed successfully. - Returns False, if the connection should be closed immediately. - """ - address = tcp.Address.wrap(address) - if self.c.config.check_ignore(address): - self.c.log("Ignore host: %s:%s" % address(), "info") - TCPHandler(self.c, log=False).handle_messages() return False + return close_connection + + def send_response_to_client(self, flow): + if not (self.supports_streaming and flow.response.stream): + # no streaming: + # we already received the full response from the server and can + # send it to the client straight away. + self.send_response(flow.response) else: - self.expected_form_in = "relative" - self.expected_form_out = "relative" - self.skip_authentication = True - - # In practice, nobody issues a CONNECT request to send unencrypted - # HTTP requests afterwards. If we don't delegate to TCP mode, we - # should always negotiate a SSL connection. - # - # FIXME: Turns out the previous statement isn't entirely true. - # Chrome on Windows CONNECTs to :80 if an explicit proxy is - # configured and a websocket connection should be established. We - # don't support websocket at the moment, so it fails anyway, but we - # should come up with a better solution to this if we start to - # support WebSockets. - should_establish_ssl = ( - address.port in self.c.config.ssl_ports - or - not self.c.config.check_tcp(address) + # streaming: + # First send the headers and then transfer the response incrementally + self.send_response_headers(flow.response) + chunks = self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code, + max_chunk_size=4096 ) + if callable(flow.response.stream): + chunks = flow.response.stream(chunks) + self.send_response_body(flow.response, chunks) + flow.response.timestamp_end = utils.timestamp() - if should_establish_ssl: - self.c.log( - "Received CONNECT request to SSL port. " - "Upgrading to SSL...", "debug" - ) - server_ssl = not self.c.config.no_upstream_cert - if server_ssl: - self.c.establish_server_connection() - self.c.establish_ssl(server=server_ssl, client=True) - self.c.log("Upgrade to SSL completed.", "debug") - - if self.c.config.check_tcp(address): - self.c.log( - "Generic TCP mode for host: %s:%s" % address(), - "info" - ) - TCPHandler(self.c).handle_messages() - return False - - return True + def get_response_from_server(self, flow): + def get_response(): + self.send_request(flow.request) + if self.supports_streaming: + flow.response = self.read_response_headers() + else: + flow.response = self.read_response(flow.request.method) - def authenticate(self, request): - if self.c.config.authenticator: - if self.c.config.authenticator.authenticate(request.headers): - self.c.config.authenticator.clean(request.headers) + try: + get_response() + except (tcp.NetLibError, HttpErrorConnClosed) as v: + self.log( + "server communication error: %s" % repr(v), + level="debug" + ) + # In any case, we try to reconnect at least once. This is + # necessary because it might be possible that we already + # initiated an upstream connection after clientconnect that + # has already been expired, e.g consider the following event + # log: + # > clientconnect (transparent mode destination known) + # > serverconnect (required for client tls handshake) + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + self.reconnect() + get_response() + + # call the appropriate script hook - this is an opportunity for an + # inline script to set flow.stream = True + flow = self.channel.ask("responseheaders", flow) + if flow == Kill: + raise Kill() + + if self.supports_streaming: + if flow.response.stream: + flow.response.content = CONTENT_MISSING else: - raise http.HttpAuthenticationError( - self.c.config.authenticator.auth_challenge_headers()) - return request.headers + flow.response.content = "".join(self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code + )) + flow.response.timestamp_end = utils.timestamp() + # no further manipulation of self.server_conn beyond this point + # we can safely set it as the final attribute value here. + flow.server_conn = self.server_conn -class RequestReplayThread(threading.Thread): - name = "RequestReplayThread" + self.log( + "response", + "debug", + [repr(flow.response)] + ) + response_reply = self.channel.ask("response", flow) + if response_reply == Kill: + raise Kill() - def __init__(self, config, flow, masterq, should_exit): - """ - masterqueue can be a queue or None, if no scripthooks should be - processed. - """ - self.config, self.flow = config, flow - if masterq: - self.channel = controller.Channel(masterq, should_exit) + def process_request_hook(self, flow): + # Determine .scheme, .host and .port attributes for inline scripts. + # For absolute-form requests, they are directly given in the request. + # For authority-form requests, we only need to determine the request scheme. + # For relative-form requests, we need to determine host and port as + # well. + if self.mode == "regular": + pass # only absolute-form at this point, nothing to do here. + elif self.mode == "upstream": + if flow.request.form_in == "authority": + flow.request.scheme = "http" # pseudo value + else: + flow.request.host = self.__original_server_conn.address.host + flow.request.port = self.__original_server_conn.address.port + flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" + + request_reply = self.channel.ask("request", flow) + if request_reply == Kill: + raise Kill() + if isinstance(request_reply, HTTPResponse): + flow.response = request_reply + return + + def establish_server_connection(self, flow): + address = tcp.Address((flow.request.host, flow.request.port)) + tls = (flow.request.scheme == "https") + + if self.mode == "regular" or self.mode == "transparent": + # If there's an existing connection that doesn't match our expectations, kill it. + if address != self.server_conn.address or tls != self.server_conn.ssl_established: + self.set_server(address, tls, address.host) + # Establish connection is neccessary. + if not self.server_conn: + self.connect() else: - self.channel = None - super(RequestReplayThread, self).__init__() + if not self.server_conn: + self.connect() + if tls: + raise HttpException("Cannot change scheme in upstream proxy mode.") + """ + # This is a very ugly (untested) workaround to solve a very ugly problem. + if self.server_conn and self.server_conn.tls_established and not ssl: + self.reconnect() + elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: + if self.server_conn.tls_established: + self.reconnect() + + self.send_request(make_connect_request(address)) + tls_layer = TlsLayer(self, False, True) + tls_layer._establish_tls_with_server() + """ + + def validate_request(self, request): + if request.form_in == "absolute" and request.scheme != "http": + self.send_response( + make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + raise HttpException("Invalid request scheme: %s" % request.scheme) + + expected_request_forms = { + "regular": ("absolute",), # an authority request would already be handled. + "upstream": ("authority", "absolute"), + "transparent": ("relative",) + } + + allowed_request_forms = expected_request_forms[self.mode] + if request.form_in not in allowed_request_forms: + err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( + " or ".join(allowed_request_forms), request.form_in + ) + self.send_response(make_error_response(400, err_message)) + raise HttpException(err_message) - def run(self): - r = self.flow.request - form_out_backup = r.form_out - try: - self.flow.response = None - - # If we have a channel, run script hooks. - if self.channel: - request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == KILL: - raise KillSignal() - elif isinstance(request_reply, HTTPResponse): - self.flow.response = request_reply - - if not self.flow.response: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - server_address = self.config.mode.get_upstream_server( - self.flow.client_conn - )[2:] - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - send_connect_request(server, r.host, r.port) - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - else: - r.form_out = "absolute" - else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - - server.send(self.flow.server_conn.protocol.assemble(r)) - self.flow.server_conn = server - self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) - self.flow.response = HTTPResponse.from_protocol( - self.flow.server_conn.protocol, - r.method, - body_size_limit=self.config.body_size_limit, - ) - if self.channel: - response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == KILL: - raise KillSignal() - except (proxy.ProxyError, http.HttpError, tcp.NetLibError) as v: - self.flow.error = Error(repr(v)) - if self.channel: - self.channel.ask("error", self.flow) - except KillSignal: - # KillSignal should only be raised if there's a channel in the - # first place. - self.channel.tell("log", proxy.Log("Connection killed", "info")) - finally: - r.form_out = form_out_backup + if self.mode == "regular": + request.form_out = "relative" + + def authenticate(self, request): + if self.config.authenticator: + if self.config.authenticator.authenticate(request.headers): + self.config.authenticator.clean(request.headers) + else: + self.send_response(make_error_response( + 407, + "Proxy Authentication Required", + odict.ODictCaseless( + [ + [k, v] for k, v in + self.config.authenticator.auth_challenge_headers().items() + ]) + )) + raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py new file mode 100644 index 00000000..2759a019 --- /dev/null +++ b/libmproxy/protocol/http_replay.py @@ -0,0 +1,96 @@ +from __future__ import (absolute_import, print_function, division) +import threading + +from netlib.http import HttpError +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError +from ..controller import Channel +from ..models import Error, HTTPResponse, ServerConnection, make_connect_request +from .base import Log, Kill + + +# TODO: Doesn't really belong into libmproxy.protocol... + + +class RequestReplayThread(threading.Thread): + name = "RequestReplayThread" + + def __init__(self, config, flow, masterq, should_exit): + """ + masterqueue can be a queue or None, if no scripthooks should be + processed. + """ + self.config, self.flow = config, flow + if masterq: + self.channel = Channel(masterq, should_exit) + else: + self.channel = None + super(RequestReplayThread, self).__init__() + + def run(self): + r = self.flow.request + form_out_backup = r.form_out + try: + self.flow.response = None + + # If we have a channel, run script hooks. + if self.channel: + request_reply = self.channel.ask("request", self.flow) + if request_reply == Kill: + raise Kill() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply + + if not self.flow.response: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + server_address = self.config.upstream_server.address + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + connect_request = make_connect_request((r.host, r.port)) + server.send(protocol.assemble(connect_request)) + resp = protocol.read_response("CONNECT") + if resp.code != 200: + raise HttpError(502, "Upstream server refuses CONNECT request") + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + + server.send(protocol.assemble(r)) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_protocol( + protocol, + r.method, + body_size_limit=self.config.body_size_limit, + ) + if self.channel: + response_reply = self.channel.ask("response", self.flow) + if response_reply == Kill: + raise Kill() + except (HttpError, NetLibError) as v: + self.flow.error = Error(repr(v)) + if self.channel: + self.channel.ask("error", self.flow) + except Kill: + # KillSignal should only be raised if there's a channel in the + # first place. + self.channel.tell("log", Log("Connection killed", "info")) + finally: + r.form_out = form_out_backup diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py deleted file mode 100644 index 92fc95e5..00000000 --- a/libmproxy/protocol/primitives.py +++ /dev/null @@ -1,294 +0,0 @@ -from __future__ import absolute_import -import copy -import uuid -import netlib.tcp -from .. import stateobject, utils, version -from ..proxy.connection import ClientConnection, ServerConnection - - -KILL = 0 # const for killed requests - - -class Error(stateobject.StateObject): - """ - An Error. - - This is distinct from an protocol error response (say, a HTTP code 500), - which is represented by a normal HTTPResponse object. This class is - responsible for indicating errors that fall outside of normal protocol - communications, like interrupted connections, timeouts, protocol errors. - - Exposes the following attributes: - - flow: Flow object - msg: Message describing the error - timestamp: Seconds since the epoch - """ - - def __init__(self, msg, timestamp=None): - """ - @type msg: str - @type timestamp: float - """ - self.flow = None # will usually be set by the flow backref mixin - self.msg = msg - self.timestamp = timestamp or utils.timestamp() - - _stateobject_attributes = dict( - msg=str, - timestamp=float - ) - - def __str__(self): - return self.msg - - @classmethod - def from_state(cls, state): - # the default implementation assumes an empty constructor. Override - # accordingly. - f = cls(None) - f.load_state(state) - return f - - def copy(self): - c = copy.copy(self) - return c - - -class Flow(stateobject.StateObject): - """ - A Flow is a collection of objects representing a single transaction. - This class is usually subclassed for each protocol, e.g. HTTPFlow. - """ - - def __init__(self, type, client_conn, server_conn, live=None): - self.type = type - self.id = str(uuid.uuid4()) - self.client_conn = client_conn - """@type: ClientConnection""" - self.server_conn = server_conn - """@type: ServerConnection""" - self.live = live - """@type: LiveConnection""" - - self.error = None - """@type: Error""" - self.intercepted = False - """@type: bool""" - self._backup = None - self.reply = None - - _stateobject_attributes = dict( - id=str, - error=Error, - client_conn=ClientConnection, - server_conn=ServerConnection, - type=str, - intercepted=bool - ) - - def get_state(self, short=False): - d = super(Flow, self).get_state(short) - d.update(version=version.IVERSION) - if self._backup and self._backup != d: - if short: - d.update(modified=True) - else: - d.update(backup=self._backup) - return d - - def __eq__(self, other): - return self is other - - def copy(self): - f = copy.copy(self) - - f.id = str(uuid.uuid4()) - f.live = False - f.client_conn = self.client_conn.copy() - f.server_conn = self.server_conn.copy() - - if self.error: - f.error = self.error.copy() - return f - - def modified(self): - """ - Has this Flow been modified? - """ - if self._backup: - return self._backup != self.get_state() - else: - return False - - def backup(self, force=False): - """ - Save a backup of this Flow, which can be reverted to using a - call to .revert(). - """ - if not self._backup: - self._backup = self.get_state() - - def revert(self): - """ - Revert to the last backed up state. - """ - if self._backup: - self.load_state(self._backup) - self._backup = None - - def kill(self, master): - """ - Kill this request. - """ - self.error = Error("Connection killed") - self.intercepted = False - self.reply(KILL) - master.handle_error(self) - - def intercept(self, master): - """ - Intercept this Flow. Processing will stop until accept_intercept is - called. - """ - if self.intercepted: - return - self.intercepted = True - master.handle_intercept(self) - - def accept_intercept(self, master): - """ - Continue with the flow - called after an intercept(). - """ - if not self.intercepted: - return - self.intercepted = False - self.reply() - master.handle_accept_intercept(self) - - - -class ProtocolHandler(object): - """ - A ProtocolHandler implements an application-layer protocol, e.g. HTTP. - See: libmproxy.protocol.http.HTTPHandler - """ - - def __init__(self, c): - self.c = c - """@type: libmproxy.proxy.server.ConnectionHandler""" - self.live = LiveConnection(c) - """@type: LiveConnection""" - - def handle_messages(self): - """ - This method gets called if a client connection has been made. Depending - on the proxy settings, a server connection might already exist as well. - """ - raise NotImplementedError # pragma: nocover - - def handle_server_reconnect(self, state): - """ - This method gets called if a server connection needs to reconnect and - there's a state associated with the server connection (e.g. a - previously-sent CONNECT request or a SOCKS proxy request). This method - gets called after the connection has been restablished but before SSL is - established. - """ - raise NotImplementedError # pragma: nocover - - def handle_error(self, error): - """ - This method gets called should there be an uncaught exception during the - connection. This might happen outside of handle_messages, e.g. if the - initial SSL handshake fails in transparent mode. - """ - raise error # pragma: nocover - - -class LiveConnection(object): - """ - This facade allows interested parties (FlowMaster, inline scripts) to - interface with a live connection, without exposing the internals - of the ConnectionHandler. - """ - - def __init__(self, c): - self.c = c - """@type: libmproxy.proxy.server.ConnectionHandler""" - self._backup_server_conn = None - """@type: libmproxy.proxy.connection.ServerConnection""" - - def change_server( - self, - address, - ssl=None, - sni=None, - force=False, - persistent_change=False): - """ - Change the server connection to the specified address. - @returns: - True, if a new connection has been established, - False, if an existing connection has been used - """ - address = netlib.tcp.Address.wrap(address) - - ssl_mismatch = ( - ssl is not None and - ( - (self.c.server_conn.connection and ssl != self.c.server_conn.ssl_established) - or - (sni is not None and sni != self.c.server_conn.sni) - ) - ) - address_mismatch = (address != self.c.server_conn.address) - - if persistent_change: - self._backup_server_conn = None - - if ssl_mismatch or address_mismatch or force: - - self.c.log( - "Change server connection: %s:%s -> %s:%s [persistent: %s]" % ( - self.c.server_conn.address.host, - self.c.server_conn.address.port, - address.host, - address.port, - persistent_change - ), - "debug" - ) - - if not self._backup_server_conn and not persistent_change: - self._backup_server_conn = self.c.server_conn - self.c.server_conn = None - else: - # This is at least the second temporary change. We can kill the - # current connection. - self.c.del_server_connection() - - self.c.set_server_address(address) - self.c.establish_server_connection(ask=False) - if ssl: - self.c.establish_ssl(server=True, sni=sni) - return True - return False - - def restore_server(self): - # TODO: Similar to _backup_server_conn, introduce _cache_server_conn, - # which keeps the changed connection open This may be beneficial if a - # user is rewriting all requests from http to https or similar. - if not self._backup_server_conn: - return - - self.c.log("Restore original server connection: %s:%s -> %s:%s" % ( - self.c.server_conn.address.host, - self.c.server_conn.address.port, - self._backup_server_conn.address.host, - self._backup_server_conn.address.port - ), "debug") - - self.c.del_server_connection() - self.c.server_conn = self._backup_server_conn - self._backup_server_conn = None diff --git a/libmproxy/protocol/rawtcp.py b/libmproxy/protocol/rawtcp.py new file mode 100644 index 00000000..86468773 --- /dev/null +++ b/libmproxy/protocol/rawtcp.py @@ -0,0 +1,66 @@ +from __future__ import (absolute_import, print_function, division) +import socket +import select + +from OpenSSL import SSL + +from netlib.tcp import NetLibError +from netlib.utils import cleanBin +from ..exceptions import ProtocolException +from .base import Layer + + +class RawTCPLayer(Layer): + chunk_size = 4096 + + def __init__(self, ctx, logging=True): + self.logging = logging + super(RawTCPLayer, self).__init__(ctx) + + def __call__(self): + self.connect() + + buf = memoryview(bytearray(self.chunk_size)) + + client = self.client_conn.connection + server = self.server_conn.connection + conns = [client, server] + + try: + while True: + r, _, _ = select.select(conns, [], [], 10) + for conn in r: + dst = server if conn == client else client + + size = conn.recv_into(buf, self.chunk_size) + if not size: + conns.remove(conn) + # Shutdown connection to the other peer + if isinstance(conn, SSL.Connection): + # We can't half-close a connection, so we just close everything here. + # Sockets will be cleaned up on a higher level. + return + else: + dst.shutdown(socket.SHUT_WR) + + if len(conns) == 0: + return + continue + + dst.sendall(buf[:size]) + + if self.logging: + # log messages are prepended with the client address, + # hence the "weird" direction string. + if dst == server: + direction = "-> tcp -> {}".format(repr(self.server_conn.address)) + else: + direction = "<- tcp <- {}".format(repr(self.server_conn.address)) + data = cleanBin(buf[:size].tobytes()) + self.log( + "{}\r\n{}".format(direction, data), + "info" + ) + + except (socket.error, NetLibError, SSL.Error) as e: + raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) diff --git a/libmproxy/protocol/tcp.py b/libmproxy/protocol/tcp.py deleted file mode 100644 index 0feb77c6..00000000 --- a/libmproxy/protocol/tcp.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import absolute_import -import select -import socket -from .primitives import ProtocolHandler -from netlib.utils import cleanBin -from netlib.tcp import NetLibError - - -class TCPHandler(ProtocolHandler): - """ - TCPHandler acts as a generic TCP forwarder. - Data will be .log()ed, but not stored any further. - """ - - chunk_size = 4096 - - def __init__(self, c, log=True): - super(TCPHandler, self).__init__(c) - self.log = log - - def handle_messages(self): - self.c.establish_server_connection() - - server = "%s:%s" % self.c.server_conn.address()[:2] - buf = memoryview(bytearray(self.chunk_size)) - conns = [self.c.client_conn.rfile, self.c.server_conn.rfile] - - try: - while True: - r, _, _ = select.select(conns, [], [], 10) - for rfile in r: - if self.c.client_conn.rfile == rfile: - src, dst = self.c.client_conn, self.c.server_conn - direction = "-> tcp ->" - src_str, dst_str = "client", server - else: - dst, src = self.c.client_conn, self.c.server_conn - direction = "<- tcp <-" - dst_str, src_str = "client", server - - closed = False - if src.ssl_established: - # Unfortunately, pyOpenSSL lacks a recv_into function. - # We need to read a single byte before .pending() - # becomes usable - contents = src.rfile.read(1) - contents += src.rfile.read(src.connection.pending()) - if not contents: - closed = True - else: - size = src.connection.recv_into(buf) - if not size: - closed = True - - if closed: - conns.remove(src.rfile) - # Shutdown connection to the other peer - if dst.ssl_established: - # We can't half-close a connection, so we just close everything here. - # Sockets will be cleaned up on a higher level. - return - else: - dst.connection.shutdown(socket.SHUT_WR) - - if len(conns) == 0: - return - continue - - if src.ssl_established or dst.ssl_established: - # if one of the peers is over SSL, we need to send - # bytes/strings - if not src.ssl_established: - # we revc'd into buf but need bytes/string now. - contents = buf[:size].tobytes() - if self.log: - self.c.log( - "%s %s\r\n%s" % ( - direction, dst_str, cleanBin(contents) - ), - "info" - ) - # Do not use dst.connection.send here, which may raise - # OpenSSL-specific errors. - dst.send(contents) - else: - # socket.socket.send supports raw bytearrays/memoryviews - if self.log: - self.c.log( - "%s %s\r\n%s" % ( - direction, dst_str, cleanBin(buf.tobytes()) - ), - "info" - ) - dst.connection.send(buf[:size]) - except (socket.error, NetLibError) as e: - self.c.log("TCP connection closed unexpectedly.", "debug") - return diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py new file mode 100644 index 00000000..a8dc8bb2 --- /dev/null +++ b/libmproxy/protocol/tls.py @@ -0,0 +1,298 @@ +from __future__ import (absolute_import, print_function, division) + +import struct + +from construct import ConstructError + +from netlib.tcp import NetLibError, NetLibInvalidCertificateError +from netlib.http.http1 import HTTP1Protocol +from ..contrib.tls._constructs import ClientHello +from ..exceptions import ProtocolException +from .base import Layer + + +def is_tls_record_magic(d): + """ + Returns: + True, if the passed bytes start with the TLS record magic bytes. + False, otherwise. + """ + d = d[:3] + + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + return ( + len(d) == 3 and + d[0] == '\x16' and + d[1] == '\x03' and + d[2] in ('\x00', '\x01', '\x02', '\x03') + ) + + +class TlsLayer(Layer): + def __init__(self, ctx, client_tls, server_tls): + self.client_sni = None + self.client_alpn_protocols = None + + super(TlsLayer, self).__init__(ctx) + self._client_tls = client_tls + self._server_tls = server_tls + + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values. + + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + + client_tls_requires_server_cert = ( + self._client_tls and self._server_tls and not self.config.no_upstream_cert + ) + + if self._client_tls: + self._parse_client_hello() + + if client_tls_requires_server_cert: + self._establish_tls_with_client_and_server() + elif self._client_tls: + self._establish_tls_with_client() + + layer = self.ctx.next_layer(self) + layer() + + def __repr__(self): + if self._client_tls and self._server_tls: + return "TlsLayer(client and server)" + elif self._client_tls: + return "TlsLayer(client)" + elif self._server_tls: + return "TlsLayer(server)" + else: + return "TlsLayer(inactive)" + + def _get_client_hello(self): + """ + Peek into the socket and read all records that contain the initial client hello message. + + Returns: + The raw handshake packet bytes, without TLS record header(s). + """ + client_hello = "" + client_hello_size = 1 + offset = 0 + while len(client_hello) < client_hello_size: + record_header = self.client_conn.rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) != 5: + raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) + record_size = struct.unpack("!H", record_header[3:])[0] + 5 + record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) + client_hello += record_body + offset += record_size + client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 + return client_hello + + def _parse_client_hello(self): + """ + Peek into the connection, read the initial client hello and parse it to obtain ALPN values. + """ + try: + raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + except ProtocolException as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + return + + try: + client_hello = ClientHello.parse(raw_client_hello) + except ConstructError as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") + return + + for extension in client_hello.extensions: + if extension.type == 0x00: + if len(extension.server_names) != 1 or extension.server_names[0].type != 0: + self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") + self.client_sni = extension.server_names[0].name + elif extension.type == 0x10: + self.client_alpn_protocols = list(extension.alpn_protocols) + + self.log( + "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), + "debug" + ) + + def connect(self): + if not self.server_conn: + self.ctx.connect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def reconnect(self): + self.ctx.reconnect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + if depth == 1 and server_tls is not None: + self.ctx.set_server(address, None, None, 1) + self._sni_from_server_change = sni + self._server_tls = server_tls + else: + self.ctx.set_server(address, server_tls, sni, depth) + + @property + def sni_for_server_connection(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self.client_sni + + @property + def alpn_for_client_connection(self): + return self.server_conn.get_alpn_proto_negotiated() + + def __alpn_select_callback(self, conn_, options): + """ + Once the client signals the alternate protocols it supports, + we reconnect upstream with the same list and pass the server's choice down to the client. + """ + + # This gets triggered if we haven't established an upstream connection yet. + default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 + # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 + + if self.alpn_for_client_connection in options: + choice = bytes(self.alpn_for_client_connection) + elif default_alpn in options: + choice = bytes(default_alpn) + else: + choice = options[0] + self.log("ALPN for client: %s" % choice, "debug") + return choice + + def _establish_tls_with_client_and_server(self): + self.ctx.connect() + + # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless + # to send an error message over TLS. + try: + self._establish_tls_with_server() + except Exception as e: + try: + self._establish_tls_with_client() + except: + pass + raise e + + self._establish_tls_with_client() + + def _establish_tls_with_client(self): + self.log("Establish TLS with client", "debug") + cert, key, chain_file = self._find_cert() + + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file, + alpn_select_callback=self.__alpn_select_callback, + ) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) + + def _establish_tls_with_server(self): + self.log("Establish TLS with server", "debug") + try: + # We only support http/1.1 and h2. + # If the server only supports spdy (next to http/1.1), it may select that + # and mitmproxy would enter TCP passthrough mode, which we want to avoid. + deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") + if self.client_alpn_protocols: + alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + else: + alpn = None + + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni_for_server_connection, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + alpn_protos=alpn, + ) + tls_cert_err = self.server_conn.ssl_verification_error + if tls_cert_err is not None: + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except NetLibInvalidCertificateError as e: + tls_cert_err = self.server_conn.ssl_verification_error + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + address=repr(self.server_conn.address), + sni=self.sni_for_server_connection, + e=repr(e), + ), e) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + address=repr(self.server_conn.address), + sni=self.sni_for_server_connection, + e=repr(e), + ), e) + + self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") + + def _find_cert(self): + host = self.server_conn.address.host + sans = set() + # Incorporate upstream certificate + use_upstream_cert = ( + self.server_conn and + self.server_conn.tls_established and + (not self.config.no_upstream_cert) + ) + if use_upstream_cert: + upstream_cert = self.server_conn.cert + sans.update(upstream_cert.altnames) + if upstream_cert.cn: + sans.add(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self.client_sni: + sans.add(self.client_sni) + if self._sni_from_server_change: + sans.add(self._sni_from_server_change) + + sans.discard(host) + return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index f33d323b..d5297cb1 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1,2 +1,9 @@ -from .primitives import * -from .config import ProxyConfig, process_proxy_options +from __future__ import (absolute_import, print_function, division) + +from .server import ProxyServer, DummyServer +from .config import ProxyConfig + +__all__ = [ + "ProxyServer", "DummyServer", + "ProxyConfig", +] diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index ec91a6e0..2a1b84cb 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,26 +1,31 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) +import collections import os import re from OpenSSL import SSL -import netlib -from netlib import http, certutils, tcp +from netlib import certutils, tcp from netlib.http import authentication +from netlib.tcp import Address, sslversion_choices -from .. import utils, platform, version -from .primitives import RegularProxyMode, SpoofMode, SSLSpoofMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode +from .. import utils, platform -TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" +# We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. +# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old +DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" + class HostMatcher(object): - def __init__(self, patterns=[]): + def __init__(self, patterns=tuple()): self.patterns = list(patterns) self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] def __call__(self, address): + if not address: + return False address = tcp.Address.wrap(address) host = "%s:%s" % (address.host, address.port) if any(rex.search(host) for rex in self.regexes): @@ -32,61 +37,44 @@ class HostMatcher(object): return bool(self.patterns) +ServerSpec = collections.namedtuple("ServerSpec", "scheme address") + + class ProxyConfig: def __init__( self, host='', port=8080, - server_version=version.NAMEVERSION, cadir=CA_DIR, clientcerts=None, no_upstream_cert=False, body_size_limit=None, - mode=None, + mode="regular", upstream_server=None, - http_form_in=None, - http_form_out=None, authenticator=None, - ignore_hosts=[], - tcp_hosts=[], + ignore_hosts=tuple(), + tcp_hosts=tuple(), ciphers_client=None, ciphers_server=None, - certs=[], - ssl_version_client=tcp.SSL_DEFAULT_METHOD, - ssl_version_server=tcp.SSL_DEFAULT_METHOD, - ssl_ports=TRANSPARENT_SSL_PORTS, - spoofed_ssl_port=None, + certs=tuple(), + ssl_version_client="secure", + ssl_version_server="secure", ssl_verify_upstream_cert=False, - ssl_upstream_trusted_cadir=None, - ssl_upstream_trusted_ca=None + ssl_verify_upstream_trusted_cadir=None, + ssl_verify_upstream_trusted_ca=None, ): self.host = host self.port = port - self.server_version = server_version self.ciphers_client = ciphers_client self.ciphers_server = ciphers_server self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit - - if mode == "transparent": - self.mode = TransparentProxyMode(platform.resolver(), ssl_ports) - elif mode == "socks5": - self.mode = Socks5ProxyMode(ssl_ports) - elif mode == "reverse": - self.mode = ReverseProxyMode(upstream_server) - elif mode == "upstream": - self.mode = UpstreamProxyMode(upstream_server) - elif mode == "spoof": - self.mode = SpoofMode() - elif mode == "sslspoof": - self.mode = SSLSpoofMode(spoofed_ssl_port) + self.mode = mode + if upstream_server: + self.upstream_server = ServerSpec(upstream_server[0], Address.wrap(upstream_server[1])) else: - self.mode = RegularProxyMode() - - # Handle manual overrides of the http forms - self.mode.http_form_in = http_form_in or self.mode.http_form_in - self.mode.http_form_out = http_form_out or self.mode.http_form_out + self.upstream_server = None self.check_ignore = HostMatcher(ignore_hosts) self.check_tcp = HostMatcher(tcp_hosts) @@ -94,41 +82,33 @@ class ProxyConfig: self.cadir = os.path.expanduser(cadir) self.certstore = certutils.CertStore.from_store( self.cadir, - CONF_BASENAME) + CONF_BASENAME + ) for spec, cert in certs: self.certstore.add_cert_file(spec, cert) - self.ssl_ports = ssl_ports - if isinstance(ssl_version_client, int): - self.openssl_method_client = ssl_version_client - else: - self.openssl_method_client = tcp.SSL_VERSIONS[ssl_version_client] - if isinstance(ssl_version_server, int): - self.openssl_method_server = ssl_version_server - else: - self.openssl_method_server = tcp.SSL_VERSIONS[ssl_version_server] + self.openssl_method_client, self.openssl_options_client = \ + sslversion_choices[ssl_version_client] + self.openssl_method_server, self.openssl_options_server = \ + sslversion_choices[ssl_version_server] if ssl_verify_upstream_cert: self.openssl_verification_mode_server = SSL.VERIFY_PEER else: self.openssl_verification_mode_server = SSL.VERIFY_NONE - self.openssl_trusted_cadir_server = ssl_upstream_trusted_cadir - self.openssl_trusted_ca_server = ssl_upstream_trusted_ca - - self.openssl_options_client = tcp.SSL_DEFAULT_OPTIONS - self.openssl_options_server = tcp.SSL_DEFAULT_OPTIONS + self.openssl_trusted_cadir_server = ssl_verify_upstream_trusted_cadir + self.openssl_trusted_ca_server = ssl_verify_upstream_trusted_ca def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) c = 0 - mode, upstream_server, spoofed_ssl_port = None, None, None + mode, upstream_server = "regular", None if options.transparent_proxy: c += 1 if not platform.resolver: - return parser.error( - "Transparent mode not supported on this platform.") + return parser.error("Transparent mode not supported on this platform.") mode = "transparent" if options.socks_proxy: c += 1 @@ -141,32 +121,33 @@ def process_proxy_options(parser, options): c += 1 mode = "upstream" upstream_server = options.upstream_proxy - if options.spoof_mode: - c += 1 - mode = "spoof" - if options.ssl_spoof_mode: - c += 1 - mode = "sslspoof" - spoofed_ssl_port = options.spoofed_ssl_port if c > 1: return parser.error( "Transparent, SOCKS5, reverse and upstream proxy mode " - "are mutually exclusive.") + "are mutually exclusive. Read the docs on proxy modes to understand why." + ) if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) - if not os.path.exists( - options.clientcerts) or not os.path.isdir( - options.clientcerts): + if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): return parser.error( "Client certificate directory does not exist or is not a directory: %s" % - options.clientcerts) + options.clientcerts + ) + + if options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd: + + if options.socks_proxy: + return parser.error( + "Proxy Authentication not supported in SOCKS mode. " + "https://github.com/mitmproxy/mitmproxy/issues/738" + ) - if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd): if options.auth_singleuser: if len(options.auth_singleuser.split(':')) != 2: return parser.error( - "Invalid single-user specification. Please use the format username:password") + "Invalid single-user specification. Please use the format username:password" + ) username, password = options.auth_singleuser.split(':') password_manager = authentication.PassManSingleUser(username, password) elif options.auth_nonanonymous: @@ -191,12 +172,6 @@ def process_proxy_options(parser, options): parser.error("Certificate file does not exist: %s" % parts[1]) certs.append(parts) - ssl_ports = options.ssl_ports - if options.ssl_ports != TRANSPARENT_SSL_PORTS: - # arparse appends to default value by default, strip that off. - # see http://bugs.python.org/issue16399 - ssl_ports = ssl_ports[len(TRANSPARENT_SSL_PORTS):] - return ProxyConfig( host=options.addr, port=options.port, @@ -206,99 +181,15 @@ def process_proxy_options(parser, options): body_size_limit=body_size_limit, mode=mode, upstream_server=upstream_server, - http_form_in=options.http_form_in, - http_form_out=options.http_form_out, ignore_hosts=options.ignore_hosts, tcp_hosts=options.tcp_hosts, authenticator=authenticator, ciphers_client=options.ciphers_client, ciphers_server=options.ciphers_server, - certs=certs, + certs=tuple(certs), ssl_version_client=options.ssl_version_client, ssl_version_server=options.ssl_version_server, - ssl_ports=ssl_ports, - spoofed_ssl_port=spoofed_ssl_port, ssl_verify_upstream_cert=options.ssl_verify_upstream_cert, - ssl_upstream_trusted_cadir=options.ssl_upstream_trusted_cadir, - ssl_upstream_trusted_ca=options.ssl_upstream_trusted_ca - ) - - -def ssl_option_group(parser): - group = parser.add_argument_group("SSL") - group.add_argument( - "--cert", - dest='certs', - default=[], - type=str, - metavar="SPEC", - action="append", - help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' - 'The domain may include a wildcard, and is equal to "*" if not specified. ' - 'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' - 'it is used, else the default key in the conf dir is used. ' - 'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. ' - 'Can be passed multiple times.') - group.add_argument( - "--ciphers-client", action="store", - type=str, dest="ciphers_client", default=None, - help="Set supported ciphers for client connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--ciphers-server", action="store", - type=str, dest="ciphers_server", default=None, - help="Set supported ciphers for server connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--client-certs", action="store", - type=str, dest="clientcerts", default=None, - help="Client certificate directory." - ) - group.add_argument( - "--no-upstream-cert", default=False, - action="store_true", dest="no_upstream_cert", - help="Don't connect to upstream server to look up certificate details." - ) - group.add_argument( - "--verify-upstream-cert", default=False, - action="store_true", dest="ssl_verify_upstream_cert", - help="Verify upstream server SSL/TLS certificates and fail if invalid " - "or not present." - ) - group.add_argument( - "--upstream-trusted-cadir", default=None, action="store", - dest="ssl_upstream_trusted_cadir", - help="Path to a directory of trusted CA certificates for upstream " - "server verification prepared using the c_rehash tool." - ) - group.add_argument( - "--upstream-trusted-ca", default=None, action="store", - dest="ssl_upstream_trusted_ca", - help="Path to a PEM formatted trusted CA certificate." - ) - group.add_argument( - "--ssl-port", - action="append", - type=int, - dest="ssl_ports", - default=list(TRANSPARENT_SSL_PORTS), - metavar="PORT", - help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " - "Defaults to %s." % - str(TRANSPARENT_SSL_PORTS)) - group.add_argument( - "--ssl-version-client", dest="ssl_version_client", type=str, default=tcp.SSL_DEFAULT_VERSION, - choices=tcp.SSL_VERSIONS.keys(), - help="""" - Use a specified protocol for client connections: - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. - Default to SSLv23.""" - ) - group.add_argument( - "--ssl-version-server", dest="ssl_version_server", type=str, default=tcp.SSL_DEFAULT_VERSION, - choices=tcp.SSL_VERSIONS.keys(), - help="""" - Use a specified protocol for server connections: - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23. - Default to SSLv23.""" - ) + ssl_verify_upstream_trusted_cadir=options.ssl_verify_upstream_trusted_cadir, + ssl_verify_upstream_trusted_ca=options.ssl_verify_upstream_trusted_ca + )
\ No newline at end of file diff --git a/libmproxy/proxy/modes/__init__.py b/libmproxy/proxy/modes/__init__.py new file mode 100644 index 00000000..f014ed98 --- /dev/null +++ b/libmproxy/proxy/modes/__init__.py @@ -0,0 +1,12 @@ +from __future__ import (absolute_import, print_function, division) +from .http_proxy import HttpProxy, HttpUpstreamProxy +from .reverse_proxy import ReverseProxy +from .socks_proxy import Socks5Proxy +from .transparent_proxy import TransparentProxy + +__all__ = [ + "HttpProxy", "HttpUpstreamProxy", + "ReverseProxy", + "Socks5Proxy", + "TransparentProxy" +] diff --git a/libmproxy/proxy/modes/http_proxy.py b/libmproxy/proxy/modes/http_proxy.py new file mode 100644 index 00000000..90c54cc6 --- /dev/null +++ b/libmproxy/proxy/modes/http_proxy.py @@ -0,0 +1,26 @@ +from __future__ import (absolute_import, print_function, division) + +from ...protocol import Layer, ServerConnectionMixin + + +class HttpProxy(Layer, ServerConnectionMixin): + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() + + +class HttpUpstreamProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address): + super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) + + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/modes/reverse_proxy.py b/libmproxy/proxy/modes/reverse_proxy.py new file mode 100644 index 00000000..b57ac5eb --- /dev/null +++ b/libmproxy/proxy/modes/reverse_proxy.py @@ -0,0 +1,17 @@ +from __future__ import (absolute_import, print_function, division) + +from ...protocol import Layer, ServerConnectionMixin + + +class ReverseProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address, server_tls): + super(ReverseProxy, self).__init__(ctx, server_address=server_address) + self.server_tls = server_tls + + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/modes/socks_proxy.py b/libmproxy/proxy/modes/socks_proxy.py new file mode 100644 index 00000000..ebaf939e --- /dev/null +++ b/libmproxy/proxy/modes/socks_proxy.py @@ -0,0 +1,60 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib import socks +from netlib.tcp import NetLibError + +from ...exceptions import Socks5Exception +from ...protocol import Layer, ServerConnectionMixin + + +class Socks5Proxy(Layer, ServerConnectionMixin): + def __call__(self): + try: + # Parse Client Greeting + client_greet = socks.ClientGreeting.from_file(self.client_conn.rfile, fail_early=True) + client_greet.assert_socks5() + if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: + raise socks.SocksError( + socks.METHOD.NO_ACCEPTABLE_METHODS, + "mitmproxy only supports SOCKS without authentication" + ) + + # Send Server Greeting + server_greet = socks.ServerGreeting( + socks.VERSION.SOCKS5, + socks.METHOD.NO_AUTHENTICATION_REQUIRED + ) + server_greet.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + # Parse Connect Request + connect_request = socks.Message.from_file(self.client_conn.rfile) + connect_request.assert_socks5() + if connect_request.msg != socks.CMD.CONNECT: + raise socks.SocksError( + socks.REP.COMMAND_NOT_SUPPORTED, + "mitmproxy only supports SOCKS5 CONNECT." + ) + + # We always connect lazily, but we need to pretend to the client that we connected. + connect_reply = socks.Message( + socks.VERSION.SOCKS5, + socks.REP.SUCCEEDED, + connect_request.atyp, + # dummy value, we don't have an upstream connection yet. + connect_request.addr + ) + connect_reply.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + except (socks.SocksError, NetLibError) as e: + raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e) + + self.server_conn.address = connect_request.addr + + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/modes/transparent_proxy.py b/libmproxy/proxy/modes/transparent_proxy.py new file mode 100644 index 00000000..96ad86c4 --- /dev/null +++ b/libmproxy/proxy/modes/transparent_proxy.py @@ -0,0 +1,24 @@ +from __future__ import (absolute_import, print_function, division) + +from ... import platform +from ...exceptions import ProtocolException +from ...protocol import Layer, ServerConnectionMixin + + +class TransparentProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx): + super(TransparentProxy, self).__init__(ctx) + self.resolver = platform.resolver() + + def __call__(self): + try: + self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) + except Exception as e: + raise ProtocolException("Transparent mode failure: %s" % repr(e), e) + + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py deleted file mode 100644 index 923f84ca..00000000 --- a/libmproxy/proxy/primitives.py +++ /dev/null @@ -1,178 +0,0 @@ -from __future__ import absolute_import -from netlib import socks, tcp - - -class ProxyError(Exception): - def __init__(self, code, message, headers=None): - super(ProxyError, self).__init__(message) - self.code, self.headers = code, headers - - -class ProxyServerError(Exception): - pass - - -class ProxyMode(object): - http_form_in = None - http_form_out = None - - def get_upstream_server(self, client_conn): - """ - Returns the address of the server to connect to. - Returns None if the address needs to be determined on the protocol level (regular proxy mode) - """ - raise NotImplementedError() # pragma: nocover - - @property - def name(self): - return self.__class__.__name__.replace("ProxyMode", "").lower() - - def __str__(self): - return self.name - - def __eq__(self, other): - """ - Allow comparisions with "regular" etc. - """ - if isinstance(other, ProxyMode): - return self is other - else: - return self.name == other - - def __ne__(self, other): - return not self.__eq__(other) - - -class RegularProxyMode(ProxyMode): - http_form_in = "absolute" - http_form_out = "relative" - - def get_upstream_server(self, client_conn): - return None - - -class SpoofMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def get_upstream_server(self, client_conn): - return None - - @property - def name(self): - return "spoof" - - -class SSLSpoofMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, sslport): - self.sslport = sslport - - def get_upstream_server(self, client_conn): - return None - - @property - def name(self): - return "sslspoof" - - -class TransparentProxyMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, resolver, sslports): - self.resolver = resolver - self.sslports = sslports - - def get_upstream_server(self, client_conn): - try: - dst = self.resolver.original_addr(client_conn.connection) - except Exception as e: - raise ProxyError(502, "Transparent mode failure: %s" % str(e)) - - if dst[1] in self.sslports: - ssl = True - else: - ssl = False - return [ssl, ssl] + list(dst) - - -class Socks5ProxyMode(ProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - def __init__(self, sslports): - self.sslports = sslports - - def get_upstream_server(self, client_conn): - try: - # Parse Client Greeting - client_greet = socks.ClientGreeting.from_file(client_conn.rfile, fail_early=True) - client_greet.assert_socks5() - if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: - raise socks.SocksError( - socks.METHOD.NO_ACCEPTABLE_METHODS, - "mitmproxy only supports SOCKS without authentication" - ) - - # Send Server Greeting - server_greet = socks.ServerGreeting( - socks.VERSION.SOCKS5, - socks.METHOD.NO_AUTHENTICATION_REQUIRED - ) - server_greet.to_file(client_conn.wfile) - client_conn.wfile.flush() - - # Parse Connect Request - connect_request = socks.Message.from_file(client_conn.rfile) - connect_request.assert_socks5() - if connect_request.msg != socks.CMD.CONNECT: - raise socks.SocksError( - socks.REP.COMMAND_NOT_SUPPORTED, - "mitmproxy only supports SOCKS5 CONNECT." - ) - - # We do not connect here yet, as the clientconnect event has not - # been handled yet. - - connect_reply = socks.Message( - socks.VERSION.SOCKS5, - socks.REP.SUCCEEDED, - connect_request.atyp, - # dummy value, we don't have an upstream connection yet. - connect_request.addr - ) - connect_reply.to_file(client_conn.wfile) - client_conn.wfile.flush() - - ssl = bool(connect_request.addr.port in self.sslports) - return ssl, ssl, connect_request.addr.host, connect_request.addr.port - - except (socks.SocksError, tcp.NetLibError) as e: - raise ProxyError(502, "SOCKS5 mode failure: %s" % str(e)) - - -class _ConstDestinationProxyMode(ProxyMode): - def __init__(self, dst): - self.dst = dst - - def get_upstream_server(self, client_conn): - return self.dst - - -class ReverseProxyMode(_ConstDestinationProxyMode): - http_form_in = "relative" - http_form_out = "relative" - - -class UpstreamProxyMode(_ConstDestinationProxyMode): - http_form_in = "absolute" - http_form_out = "absolute" - - -class Log: - def __init__(self, msg, level="info"): - self.msg = msg - self.level = level diff --git a/libmproxy/proxy/root_context.py b/libmproxy/proxy/root_context.py new file mode 100644 index 00000000..35909612 --- /dev/null +++ b/libmproxy/proxy/root_context.py @@ -0,0 +1,93 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol + +from ..protocol import ( + RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin +) +from .modes import HttpProxy, HttpUpstreamProxy, ReverseProxy + + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel, .next_layer() and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def next_layer(self, top_layer): + """ + This function determines the next layer in the protocol stack. + + Arguments: + top_layer: the current top layer. + + Returns: + The next layer + """ + + # 1. Check for --ignore. + if self.config.check_ignore(top_layer.server_conn.address): + return RawTCPLayer(top_layer, logging=False) + + d = top_layer.client_conn.rfile.peek(3) + client_tls = is_tls_record_magic(d) + + # 2. Always insert a TLS layer, even if there's neither client nor server tls. + # An inline script may upgrade from http to https, + # in which case we need some form of TLS layer. + if isinstance(top_layer, ReverseProxy): + return TlsLayer(top_layer, client_tls, top_layer.server_tls) + if isinstance(top_layer, ServerConnectionMixin): + return TlsLayer(top_layer, client_tls, client_tls) + + # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. + if isinstance(top_layer, TlsLayer): + if isinstance(top_layer.ctx, HttpProxy): + return Http1Layer(top_layer, "regular") + if isinstance(top_layer.ctx, HttpUpstreamProxy): + return Http1Layer(top_layer, "upstream") + + # 4. Check for other TLS cases (e.g. after CONNECT). + if client_tls: + return TlsLayer(top_layer, True, True) + + # 4. Check for --tcp + if self.config.check_tcp(top_layer.server_conn.address): + return RawTCPLayer(top_layer) + + # 5. Check for TLS ALPN (HTTP1/HTTP2) + if isinstance(top_layer, TlsLayer): + alpn = top_layer.client_conn.get_alpn_proto_negotiated() + if alpn == HTTP2Protocol.ALPN_PROTO_H2: + return Http2Layer(top_layer, 'transparent') + if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: + return Http1Layer(top_layer, 'transparent') + + # 6. Assume HTTP1 by default + return Http1Layer(top_layer, 'transparent') + + # In a future version, we want to implement TCP passthrough as the last fallback, + # but we don't have the UI part ready for that. + # + # d = top_layer.client_conn.rfile.peek(3) + # is_ascii = ( + # len(d) == 3 and + # # better be safe here and don't expect uppercase... + # all(x in string.ascii_letters for x in d) + # ) + # # TODO: This could block if there are not enough bytes available? + # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) + # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) + + @property + def layers(self): + return [] + + def __repr__(self): + return "RootContext" diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 2f6ee061..e9e8df09 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -1,13 +1,17 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) +import traceback +import sys import socket -from OpenSSL import SSL from netlib import tcp -from .primitives import ProxyServerError, Log, ProxyError -from .connection import ClientConnection, ServerConnection -from ..protocol.handle import protocol_handler -from .. import version +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError +from ..exceptions import ProtocolException, ServerException +from ..protocol import Log, Kill +from ..models import ClientConnection, make_error_response +from .modes import HttpUpstreamProxy, HttpProxy, ReverseProxy, TransparentProxy, Socks5Proxy +from .root_context import RootContext class DummyServer: @@ -33,9 +37,9 @@ class ProxyServer(tcp.TCPServer): """ self.config = config try: - tcp.TCPServer.__init__(self, (config.host, config.port)) - except socket.error as v: - raise ProxyServerError('Error starting proxy server: ' + repr(v)) + super(ProxyServer, self).__init__((config.host, config.port)) + except socket.error as e: + raise ServerException('Error starting proxy server: ' + repr(e), e) self.channel = None def start_slave(self, klass, channel): @@ -47,340 +51,90 @@ class ProxyServer(tcp.TCPServer): def handle_client_connection(self, conn, client_address): h = ConnectionHandler( - self.config, conn, client_address, - self, - self.channel) + self.config, + self.channel + ) h.handle() - h.finish() -class ConnectionHandler: - def __init__( - self, - config, - client_connection, - client_address, - server, - channel): +class ConnectionHandler(object): + def __init__(self, client_conn, client_address, config, channel): self.config = config """@type: libmproxy.proxy.config.ProxyConfig""" self.client_conn = ClientConnection( - client_connection, + client_conn, client_address, - server) + None) """@type: libmproxy.proxy.connection.ClientConnection""" - self.server_conn = None - """@type: libmproxy.proxy.connection.ServerConnection""" self.channel = channel + """@type: libmproxy.controller.Channel""" - self.conntype = "http" + def _create_root_layer(self): + root_context = RootContext( + self.client_conn, + self.config, + self.channel + ) + + mode = self.config.mode + if mode == "upstream": + return HttpUpstreamProxy( + root_context, + self.config.upstream_server.address + ) + elif mode == "transparent": + return TransparentProxy(root_context) + elif mode == "reverse": + server_tls = self.config.upstream_server.scheme == "https" + return ReverseProxy( + root_context, + self.config.upstream_server.address, + server_tls + ) + elif mode == "socks5": + return Socks5Proxy(root_context) + elif mode == "regular": + return HttpProxy(root_context) + elif callable(mode): # pragma: nocover + return mode(root_context) + else: # pragma: nocover + raise ValueError("Unknown proxy mode: %s" % mode) def handle(self): - try: - self.log("clientconnect", "info") - - # Can we already identify the target server and connect to it? - client_ssl, server_ssl = False, False - conn_kwargs = dict() - upstream_info = self.config.mode.get_upstream_server( - self.client_conn) - if upstream_info: - self.set_server_address(upstream_info[2:]) - client_ssl, server_ssl = upstream_info[:2] - if self.config.check_ignore(self.server_conn.address): - self.log( - "Ignore host: %s:%s" % - self.server_conn.address(), - "info") - self.conntype = "tcp" - conn_kwargs["log"] = False - client_ssl, server_ssl = False, False - else: - # No upstream info from the metadata: upstream info in the - # protocol (e.g. HTTP absolute-form) - pass - - self.channel.ask("clientconnect", self) - - # Check for existing connection: If an inline script already established a - # connection, do not apply client_ssl or server_ssl. - if self.server_conn and not self.server_conn.connection: - self.establish_server_connection() - if client_ssl or server_ssl: - self.establish_ssl(client=client_ssl, server=server_ssl) - - if self.config.check_tcp(self.server_conn.address): - self.log( - "Generic TCP mode for host: %s:%s" % - self.server_conn.address(), - "info") - self.conntype = "tcp" - - elif not self.server_conn and self.config.mode == "sslspoof": - port = self.config.mode.sslport - self.set_server_address(("-", port)) - self.establish_ssl(client=True) - host = self.client_conn.connection.get_servername() - if host: - self.set_server_address((host, port)) - self.establish_server_connection() - self.establish_ssl(server=True, sni=host) - - # Delegate handling to the protocol handler - protocol_handler( - self.conntype)( - self, - **conn_kwargs).handle_messages() - - self.log("clientdisconnect", "info") - self.channel.tell("clientdisconnect", self) - - except ProxyError as e: - protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e) - except Exception: - import traceback - import sys - - self.log(traceback.format_exc(), "error") - print >> sys.stderr, traceback.format_exc() - print >> sys.stderr, "mitmproxy has crashed!" - print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy" - finally: - # Make sure that we close the server connection in any case. - # The client connection is closed by the ProxyServer and does not - # have be handled here. - self.del_server_connection() - - def del_server_connection(self): - """ - Deletes (and closes) an existing server connection. - """ - if self.server_conn and self.server_conn.connection: - self.server_conn.finish() - self.server_conn.close() - self.log( - "serverdisconnect", "debug", [ - "%s:%s" % - (self.server_conn.address.host, self.server_conn.address.port)]) - self.channel.tell("serverdisconnect", self) - self.server_conn = None - - def set_server_address(self, addr): - """ - Sets a new server address with the given priority. - Does not re-establish either connection or SSL handshake. - """ - address = tcp.Address.wrap(addr) - - # Don't reconnect to the same destination. - if self.server_conn and self.server_conn.address == address: - return + self.log("clientconnect", "info") - if self.server_conn: - self.del_server_connection() + root_layer = self._create_root_layer() + root_layer = self.channel.ask("clientconnect", root_layer) + if root_layer == Kill: + def root_layer(): + raise Kill() - self.log( - "Set new server address: %s:%s" % - (address.host, address.port), "debug") - self.server_conn = ServerConnection(address) - - def establish_server_connection(self, ask=True): - """ - Establishes a new server connection. - If there is already an existing server connection, the function returns immediately. - - By default, this function ".ask"s the proxy master. This is deadly if this function is already called from the - master (e.g. via change_server), because this navigates us in a simple deadlock (the master is single-threaded). - In these scenarios, ask=False can be passed to suppress the call to the master. - """ - if self.server_conn.connection: - return - self.log( - "serverconnect", "debug", [ - "%s:%s" % - self.server_conn.address()[ - :2]]) - if ask: - self.channel.ask("serverconnect", self) try: - self.server_conn.connect() - except tcp.NetLibError as v: - raise ProxyError(502, v) - - def establish_ssl(self, client=False, server=False, sni=None): - """ - Establishes SSL on the existing connection(s) to the server or the client, - as specified by the parameters. - """ - - # Logging - if client or server: - subs = [] - if client: - subs.append("with client") - if server: - subs.append("with server (sni: %s)" % sni) - self.log("Establish SSL", "debug", subs) - - if server: - if not self.server_conn or not self.server_conn.connection: - raise ProxyError(502, "No server connection.") - if self.server_conn.ssl_established: - raise ProxyError(502, "SSL to Server already established.") + root_layer() + except Kill: + self.log("Connection killed", "info") + except ProtocolException as e: + self.log(e, "info") + # If an error propagates to the topmost level, + # we send an HTTP error response, which is both + # understandable by HTTP clients and humans. try: - self.server_conn.establish_ssl( - self.config.clientcerts, - sni, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify_options=self.config.openssl_verification_mode_server, - ca_path=self.config.openssl_trusted_cadir_server, - ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, - ) - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") - except tcp.NetLibError as v: - e = ProxyError(502, repr(v)) - # Workaround for https://github.com/mitmproxy/mitmproxy/issues/427 - # The upstream server may reject connections without SNI, which means we need to - # establish SSL with the client first, hope for a SNI (which triggers a reconnect which replaces the - # ServerConnection object) and see whether that worked. - if client and "handshake failure" in e.message: - self.server_conn.may_require_sni = e - else: - ssl_cert_err = self.server_conn.ssl_verification_error - if ssl_cert_err is not None: - self.log( - "SSL verification failed for upstream server at depth %s with error: %s" % - (ssl_cert_err['depth'], ssl_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") - raise e - if client: - if self.client_conn.ssl_established: - raise ProxyError(502, "SSL to Client already established.") - cert, key, chain_file = self.find_cert() - try: - self.client_conn.convert_to_ssl( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - handle_sni=self.handle_sni, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - except tcp.NetLibError as v: - raise ProxyError(400, repr(v)) - - # Workaround for #427 part 2 - if server and hasattr(self.server_conn, "may_require_sni"): - raise self.server_conn.may_require_sni - - def server_reconnect(self, new_sni=False): - address = self.server_conn.address - had_ssl = self.server_conn.ssl_established - state = self.server_conn.state - sni = new_sni or self.server_conn.sni - self.log("(server reconnect follows)", "debug") - self.del_server_connection() - self.set_server_address(address) - self.establish_server_connection() - - for s in state: - protocol_handler(s[0])(self).handle_server_reconnect(s[1]) - self.server_conn.state = state - - # Receiving new_sni where had_ssl is False is a weird case that happens when the workaround for - # https://github.com/mitmproxy/mitmproxy/issues/427 is active. In this - # case, we want to establish SSL as well. - if had_ssl or new_sni: - self.establish_ssl(server=True, sni=sni) + error_response = make_error_response(502, repr(e)) + self.client_conn.send(HTTP1Protocol().assemble(error_response)) + except NetLibError: + pass + except Exception: + self.log(traceback.format_exc(), "error") + print(traceback.format_exc(), file=sys.stderr) + print("mitmproxy has crashed!", file=sys.stderr) + print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) - def finish(self): + self.log("clientdisconnect", "info") + self.channel.tell("clientdisconnect", root_layer) self.client_conn.finish() - def log(self, msg, level, subs=()): - full_msg = [ - "%s:%s: %s" % - (self.client_conn.address.host, - self.client_conn.address.port, - msg)] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - def find_cert(self): - host = self.server_conn.address.host - sans = [] - if self.server_conn.ssl_established and ( - not self.config.no_upstream_cert): - upstream_cert = self.server_conn.cert - sans.extend(upstream_cert.altnames) - if upstream_cert.cn: - sans.append(host) - host = upstream_cert.cn.decode("utf8").encode("idna") - if self.server_conn.sni: - sans.append(self.server_conn.sni) - # for ssl spoof mode - if hasattr(self.client_conn, "sni"): - sans.append(self.client_conn.sni) - - ret = self.config.certstore.get_cert(host, sans) - if not ret: - raise ProxyError(502, "Unable to generate dummy cert.") - return ret - - def handle_sni(self, connection): - """ - This callback gets called during the SSL handshake with the client. - The client has just sent the Sever Name Indication (SNI). We now connect upstream to - figure out which certificate needs to be served. - """ - try: - sn = connection.get_servername() - if not sn: - return - sni = sn.decode("utf8").encode("idna") - # for ssl spoof mode - self.client_conn.sni = sni - - if sni != self.server_conn.sni: - self.log("SNI received: %s" % sni, "debug") - # We should only re-establish upstream SSL if one of the following conditions is true: - # - We established SSL with the server previously - # - We initially wanted to establish SSL with the server, - # but the server refused to negotiate without SNI. - if self.server_conn.ssl_established or hasattr( - self.server_conn, - "may_require_sni"): - # reconnect to upstream server with SNI - self.server_reconnect(sni) - # Now, change client context to reflect changed certificate: - cert, key, chain_file = self.find_cert() - new_context = self.client_conn.create_ssl_context( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file - ) - connection.set_context(new_context) - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except: # pragma: no cover - import traceback - self.log( - "Error in handle_sni:\r\n" + - traceback.format_exc(), - "error") + def log(self, msg, level): + msg = "{}: {}".format(repr(self.client_conn.address), msg) + self.channel.tell("log", Log(msg, level))
\ No newline at end of file diff --git a/libmproxy/utils.py b/libmproxy/utils.py index 3ac3cc01..a6ca55f7 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -1,14 +1,10 @@ from __future__ import absolute_import import os import datetime -import urllib import re import time -import functools -import cgi import json -import netlib.utils def timestamp(): """ |