diff options
| -rw-r--r-- | doc-src/features/reverseproxy.html | 2 | ||||
| -rw-r--r-- | doc-src/scripting/inlinescripts.html | 28 | ||||
| -rw-r--r-- | libmproxy/cmdline.py | 79 | ||||
| -rw-r--r-- | libmproxy/console/__init__.py | 25 | ||||
| -rw-r--r-- | libmproxy/console/flowview.py | 4 | ||||
| -rw-r--r-- | libmproxy/console/help.py | 1 | ||||
| -rw-r--r-- | libmproxy/protocol/__init__.py | 2 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 76 | ||||
| -rw-r--r-- | libmproxy/proxy/config.py | 60 | ||||
| -rw-r--r-- | libmproxy/proxy/primitives.py | 43 | ||||
| -rw-r--r-- | libmproxy/proxy/server.py | 72 | ||||
| -rw-r--r-- | libmproxy/utils.py | 9 | ||||
| -rw-r--r-- | test/test_cmdline.py | 11 | ||||
| -rw-r--r-- | test/test_protocol_http.py | 4 | ||||
| -rw-r--r-- | test/test_proxy.py | 26 | ||||
| -rw-r--r-- | test/test_utils.py | 8 | ||||
| -rw-r--r-- | test/tservers.py | 32 | ||||
| -rw-r--r-- | test/tutils.py | 11 |
18 files changed, 260 insertions, 233 deletions
diff --git a/doc-src/features/reverseproxy.html b/doc-src/features/reverseproxy.html index d399cdc0..7be02b87 100644 --- a/doc-src/features/reverseproxy.html +++ b/doc-src/features/reverseproxy.html @@ -10,7 +10,7 @@ __Host__ header field from the request, not the reverse proxy server. <table class="table"> <tbody> <tr> - <th width="20%">command-line</th> <td>-P http[s]://hostname[:port]</td> + <th width="20%">command-line</th> <td>-R http[s]://hostname[:port]</td> </tr> <tr> <th>mitmproxy shortcut</th> <td><b>P</b></td> diff --git a/doc-src/scripting/inlinescripts.html b/doc-src/scripting/inlinescripts.html index 7ab1c101..32a98e99 100644 --- a/doc-src/scripting/inlinescripts.html +++ b/doc-src/scripting/inlinescripts.html @@ -76,26 +76,26 @@ The main classes you will deal with in writing mitmproxy scripts are: <table class="table"> <tr> - <th>libmproxy.flow.ClientConnection</th> - <td>Describes a client connection.</td> + <th>libmproxy.proxy.server.ConnectionHandler</th> + <td>Describes a proxy client connection session. Always has a client_conn attribute, might have a server_conn attribute.</td> </tr> <tr> - <th>libmproxy.flow.ClientDisconnection</th> - <td>Describes a client disconnection.</td> + <th>libmproxy.proxy.connection.ClientConnection</th> + <td>Describes a client connection.</td> + </tr> + <tr> + <th>libmproxy.proxy.connection.ServerConnection</th> + <td>Describes a server connection.</td> </tr> <tr> - <th>libmproxy.flow.Error</th> + <th>libmproxy.protocol.primitives.Error</th> <td>A communications error.</td> </tr> <tr> - <th>libmproxy.flow.Flow</th> + <th>libmproxy.protocol.http.HTTPFlow</th> <td>A collection of objects representing a single HTTP transaction.</td> </tr> <tr> - <th>libmproxy.flow.Headers</th> - <td>HTTP headers for a request or response.</td> - </tr> - <tr> <th>libmproxy.flow.ODict</th> <td>A dictionary-like object for managing sets of key/value data. There @@ -103,15 +103,15 @@ The main classes you will deal with in writing mitmproxy scripts are: calls (used mainly for headers).</td> </tr> <tr> - <th>libmproxy.flow.Response</th> + <th>libmproxy.protocol.http.HTTPResponse</th> <td>An HTTP response.</td> </tr> <tr> - <th>libmproxy.flow.Request</th> + <th>libmproxy.protocol.http.HTTPRequest</th> <td>An HTTP request.</td> </tr> <tr> - <th>libmproxy.flow.ScriptContext</th> + <th>libmproxy.script.ScriptContext</th> <td> A handle for interacting with mitmproxy's from within scripts. </td> </tr> <tr> @@ -124,7 +124,7 @@ The canonical API documentation is the code. You can view the API documentation using pydoc (which is installed with Python by default), like this: <pre class="terminal"> -> pydoc libmproxy.flow.Request +> pydoc libmproxy.protocol.http.HTTPRequest </pre> diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 72c13769..bee4aa60 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -1,12 +1,16 @@ from . import proxy import re, filt import argparse +from argparse import ArgumentTypeError +from netlib import http APP_HOST = "mitm.it" APP_PORT = 80 -class ParseException(Exception): pass -class OptionException(Exception): pass + +class ParseException(Exception): + pass + def _parse_hook(s): sep, rem = s[0], s[1:] @@ -91,6 +95,26 @@ def parse_setheader(s): return _parse_hook(s) +def parse_server_spec(url): + + normalized_url = re.sub("^https?2", "", url) + + p = http.parse_url(normalized_url) + if not p or not p[1]: + raise ArgumentTypeError("Invalid server specification: %s" % url) + + if url.lower().startswith("https2http"): + ssl = [True, False] + elif url.lower().startswith("http2https"): + ssl = [False, True] + elif url.lower().startswith("https"): + ssl = [True, True] + else: + ssl = [False, False] + + return ssl + list(p[1:3]) + + def get_common_options(options): stickycookie, stickyauth = None, None if options.stickycookie_filt: @@ -104,17 +128,17 @@ def get_common_options(options): try: p = parse_replace_hook(i) except ParseException, e: - raise OptionException(e.message) + raise ArgumentTypeError(e.message) reps.append(p) for i in options.replace_file: try: patt, rex, path = parse_replace_hook(i) except ParseException, e: - raise OptionException(e.message) + raise ArgumentTypeError(e.message) try: v = open(path, "rb").read() except IOError, e: - raise OptionException("Could not read replace file: %s"%path) + raise ArgumentTypeError("Could not read replace file: %s"%path) reps.append((patt, rex, v)) @@ -123,7 +147,7 @@ def get_common_options(options): try: p = parse_setheader(i) except ParseException, e: - raise OptionException(e.message) + raise ArgumentTypeError(e.message) setheaders.append(p) return dict( @@ -185,17 +209,49 @@ def common_options(parser): action="store", type = int, dest="port", default=8080, help = "Proxy service port." ) + # We could make a mutually exclusive group out of -R, -F, -T, but we don't do that because + # - --upstream-server should be in that group as well, but it's already in a different group. + # - our own error messages are more helpful parser.add_argument( - "-P", - action="store", dest="reverse_proxy", default=None, + "-R", + action="store", type=parse_server_spec, dest="reverse_proxy", default=None, help="Reverse proxy to upstream server: http[s]://host[:port]" ) parser.add_argument( "-F", - action="store", dest="forward_proxy", default=None, + action="store", type=parse_server_spec, dest="forward_proxy", default=None, help="Proxy to unconditionally forward to: http[s]://host[:port]" ) parser.add_argument( + "-T", + action="store_true", dest="transparent_proxy", default=False, + help="Set transparent proxy mode." + ) + + group = parser.add_argument_group( + "Advanced Proxy Options", + """ + The following options allow a custom adjustment of the proxy behavior. + Normally, you don't want to use these options directly and use the provided wrappers instead (-R, -F, -T). + """.strip() + ) + group.add_argument( + "--http-form-in", dest="http_form_in", default=None, + action="store", choices=("relative", "absolute"), + help="Override the HTTP request form accepted by the proxy" + ) + group.add_argument( + "--http-form-out", dest="http_form_out", default=None, + action="store", choices=("relative", "absolute"), + help="Override the HTTP request form sent upstream by the proxy" + ) + group.add_argument( + "--upstream-server", dest="manual_upstream_server", default=None, + action="store", type=parse_server_spec, + help="Override the destination server all requests are sent to." + ) + + parser.add_argument( "-q", action="store_true", dest="quiet", help="Quiet." @@ -217,11 +273,6 @@ def common_options(parser): help="Set sticky cookie filter. Matched against requests." ) parser.add_argument( - "-T", - action="store_true", dest="transparent_proxy", default=False, - help="Set transparent proxy mode." - ) - parser.add_argument( "-u", action="store", dest="stickyauth_filt", default=None, metavar="FILTER", help="Set sticky auth filter. Matched against requests." diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 4a58e771..846abba7 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -144,10 +144,6 @@ class StatusBar(common.WWrap): r.append("[") r.append(("heading_key", "u")) r.append(":%s]"%self.master.stickyauth_txt) - if self.master.server.config.reverse_proxy: - r.append("[") - r.append(("heading_key", "P")) - r.append(":%s]"%utils.unparse_url(*self.master.server.config.reverse_proxy)) if self.master.state.default_body_view.name != "Auto": r.append("[") r.append(("heading_key", "M")) @@ -172,6 +168,8 @@ class StatusBar(common.WWrap): if opts: r.append("[%s]"%(":".join(opts))) + if self.master.server.config.upstream_server: + r.append("[dest:%s]"%utils.unparse_url(*self.master.server.config.upstream_server)) if self.master.scripts: r.append("[scripts:%s]"%len(self.master.scripts)) if self.master.debug: @@ -763,15 +761,6 @@ class ConsoleMaster(flow.FlowMaster): self.state.default_body_view = v self.refresh_focus() - def set_reverse_proxy(self, txt): - if not txt: - self.server.config.reverse_proxy = None - else: - s = utils.parse_proxy_spec(txt) - if not s: - return "Invalid reverse proxy specification" - self.server.config.reverse_proxy = s - def drawscreen(self): size = self.ui.get_cols_rows() canvas = self.view.render(size, focus=1) @@ -866,16 +855,6 @@ class ConsoleMaster(flow.FlowMaster): contentview.view_prompts, self.change_default_display_mode ) - elif k == "P": - if self.server.config.reverse_proxy: - p = utils.unparse_url(*self.server.config.reverse_proxy) - else: - p = "" - self.prompt( - "Reverse proxy: ", - p, - self.set_reverse_proxy - ) elif k == "R": self.view_grideditor( grideditor.ReplaceEditor( diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index f5b5f83f..9b636840 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -2,7 +2,7 @@ import os, sys, copy import urwid import common, grideditor, contentview from .. import utils, flow, controller -from ..protocol.http import CONTENT_MISSING +from ..protocol.http import HTTPResponse, CONTENT_MISSING class SearchError(Exception): pass @@ -571,7 +571,7 @@ class FlowView(common.WWrap): conn = self.flow.request else: if not self.flow.response: - self.flow.response = flow.Response( + self.flow.response = HTTPResponse( self.flow.request, self.flow.request.httpversion, 200, "OK", flow.ODictCaseless(), "", None diff --git a/libmproxy/console/help.py b/libmproxy/console/help.py index 0d01ac6f..b0b7c48c 100644 --- a/libmproxy/console/help.py +++ b/libmproxy/console/help.py @@ -109,7 +109,6 @@ class HelpView(urwid.ListBox): ("q", "quit / return to flow list"), ("Q", "quit without confirm prompt"), - ("P", "set reverse proxy mode"), ("R", "edit replacement patterns"), ("s", "set/unset script"), ("S", "server replay"), diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index 6200757f..b253fbd5 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -1,4 +1,4 @@ -from libmproxy.proxy.primitives import AddressPriority +from ..proxy.primitives import AddressPriority KILL = 0 # const for killed requests diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 77a09e61..aff4a817 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -843,6 +843,12 @@ class HttpAuthenticationError(Exception): class HTTPHandler(ProtocolHandler, TemporaryServerChangeMixin): + def __init__(self, c): + super(HTTPHandler, self).__init__(c) + self.expected_form_in = c.config.http_form_in + self.expected_form_out = c.config.http_form_out + self.skip_authentication = False + def handle_messages(self): while self.handle_flow(): pass @@ -877,13 +883,15 @@ class HTTPHandler(ProtocolHandler, TemporaryServerChangeMixin): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.change_server) try: req = HTTPRequest.from_stream(self.c.client_conn.rfile, - body_size_limit=self.c.config.body_size_limit) + body_size_limit=self.c.config.body_size_limit) self.c.log("request", [req._assemble_first_line(req.form_in)]) - self.process_request(flow, req) + send_upstream = self.process_request(flow, req) + if not send_upstream: + return True # Be careful NOT to assign the request to the flow before # process_request completes. This is because the call can raise an - # exception. If the requets object is already attached, this results + # exception. If the request object is already attached, this results # in an Error object that has an attached request that has not been # sent through to the Master. flow.request = req @@ -1004,44 +1012,48 @@ class HTTPHandler(ProtocolHandler, TemporaryServerChangeMixin): Upgrade the connection to SSL after an authority (CONNECT) request has been made. """ self.c.log("Received CONNECT request. Upgrading to SSL...") - self.c.mode = "transparent" - self.c.determine_conntype() + self.expected_form_in = "relative" + self.expected_form_out = "relative" self.c.establish_ssl(server=True, client=True) self.c.log("Upgrade to SSL completed.") - raise ConnectionTypeChange def process_request(self, flow, request): - if self.c.mode == "regular": + + if not self.skip_authentication: self.authenticate(request) - if request.form_in == "authority" and self.c.client_conn.ssl_established: - raise http.HttpError(502, "Must not CONNECT on already encrypted connection") - # If we have a CONNECT request, we might need to intercept if request.form_in == "authority": - directly_addressed_at_mitmproxy = (self.c.mode == "regular" and not self.c.config.forward_proxy) - if directly_addressed_at_mitmproxy: - self.c.set_server_address((request.host, request.port), AddressPriority.FROM_PROTOCOL) - flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow - self.c.client_conn.wfile.write( - 'HTTP/1.1 200 Connection established\r\n' + - ('Proxy-agent: %s\r\n' % self.c.server_version) + - '\r\n' - ) - self.c.client_conn.wfile.flush() - self.ssl_upgrade() # raises ConnectionTypeChange exception - - if self.c.mode == "regular": - if request.form_in == "authority": # forward mode - self.hook_reconnect(request) - elif request.form_in == "absolute": - if request.scheme != "http": - raise http.HttpError(400, "Invalid Request") - if not self.c.config.forward_proxy: - request.form_out = "relative" + if self.c.client_conn.ssl_established: + raise http.HttpError(400, "Must not CONNECT on already encrypted connection") + + if self.expected_form_in == "absolute": + if not self.c.config.get_upstream_server: self.c.set_server_address((request.host, request.port), AddressPriority.FROM_PROTOCOL) flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow - else: - raise http.HttpError(400, "Invalid request form (absolute-form or authority-form required)") + self.c.client_conn.send( + 'HTTP/1.1 200 Connection established\r\n' + + ('Proxy-agent: %s\r\n' % self.c.server_version) + + '\r\n' + ) + self.ssl_upgrade() + self.skip_authentication = True + return False + else: + self.hook_reconnect(request) + return True + elif request.form_in == self.expected_form_in: + if request.form_in == "absolute": + if request.scheme != "http": + raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme) + + self.c.set_server_address((request.host, request.port), AddressPriority.FROM_PROTOCOL) + flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow + + request.form_out = self.expected_form_out + return True + + raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" % (self.expected_form_in, + request.form_in)) def authenticate(self, request): if self.c.config.authenticator: diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 38c6ce89..ae24d4c9 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,7 +1,7 @@ import os from .. import utils, platform from netlib import http_auth, certutils - +from .primitives import ConstUpstreamServerResolver, TransparentUpstreamServerResolver TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" @@ -10,17 +10,17 @@ CONF_DIR = "~/.mitmproxy" class ProxyConfig: def __init__(self, confdir=CONF_DIR, clientcerts=None, - no_upstream_cert=False, body_size_limit=None, reverse_proxy=None, - forward_proxy=None, transparent_proxy=None, authenticator=None, + no_upstream_cert=False, body_size_limit=None, get_upstream_server=None, + http_form_in="absolute", http_form_out="relative", authenticator=None, ciphers=None, certs=None ): self.ciphers = ciphers self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit - self.reverse_proxy = reverse_proxy - self.forward_proxy = forward_proxy - self.transparent_proxy = transparent_proxy + self.get_upstream_server = get_upstream_server + self.http_form_in = http_form_in + self.http_form_out = http_form_out self.authenticator = authenticator self.confdir = os.path.expanduser(confdir) self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) @@ -28,32 +28,34 @@ class ProxyConfig: def process_proxy_options(parser, options): body_size_limit = utils.parse_size(options.body_size_limit) - if options.reverse_proxy and options.transparent_proxy: - return parser.error("Can't set both reverse proxy and transparent proxy.") + c = 0 + http_form_in, http_form_out = "absolute", "relative" + get_upstream_server = None if options.transparent_proxy: + c += 1 if not platform.resolver: return parser.error("Transparent mode not supported on this platform.") - trans = dict( - resolver=platform.resolver(), - sslports=TRANSPARENT_SSL_PORTS - ) - else: - trans = None - + get_upstream_server = TransparentUpstreamServerResolver(platform.resolver(), TRANSPARENT_SSL_PORTS) + http_form_in, http_form_out = "relative", "relative" if options.reverse_proxy: - rp = utils.parse_proxy_spec(options.reverse_proxy) - if not rp: - return parser.error("Invalid reverse proxy specification: %s" % options.reverse_proxy) - else: - rp = None - + c += 1 + get_upstream_server = ConstUpstreamServerResolver(options.reverse_proxy) + http_form_in, http_form_out = "relative", "relative" if options.forward_proxy: - fp = utils.parse_proxy_spec(options.forward_proxy) - if not fp: - return parser.error("Invalid forward proxy specification: %s" % options.forward_proxy) - else: - fp = None + c += 1 + get_upstream_server = ConstUpstreamServerResolver(options.forward_proxy) + http_form_in, http_form_out = "absolute", "absolute" + if options.manual_upstream_server: + c += 1 + get_upstream_server = ConstUpstreamServerResolver(options.manual_upstream_server) + if c > 1: + return parser.error("Transparent mode, reverse mode, forward mode and " + "specification of an upstream server are mutually exclusive.") + if options.http_form_in: + http_form_in = options.http_form_in + if options.http_form_out: + http_form_out = options.http_form_out if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) @@ -93,9 +95,9 @@ def process_proxy_options(parser, options): clientcerts=options.clientcerts, body_size_limit=body_size_limit, no_upstream_cert=options.no_upstream_cert, - reverse_proxy=rp, - forward_proxy=fp, - transparent_proxy=trans, + get_upstream_server=get_upstream_server, + http_form_in=http_form_in, + http_form_out=http_form_out, authenticator=authenticator, ciphers=options.ciphers, certs = certs, diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index 8dd0e16a..e49d9bb4 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -18,19 +18,48 @@ class ProxyServerError(Exception): pass +class UpstreamServerResolver(object): + def __call__(self, conn): + """ + Returns the address of the server to connect to. + """ + raise NotImplementedError + + +class ConstUpstreamServerResolver(UpstreamServerResolver): + def __init__(self, dst): + self.dst = dst + + def __call__(self, conn): + return self.dst + + +class TransparentUpstreamServerResolver(UpstreamServerResolver): + def __init__(self, resolver, sslports): + self.resolver = resolver + self.sslports = sslports + + def __call__(self, conn): + dst = self.resolver.original_addr(conn) + if not dst: + raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") + + if dst[1] in self.sslports: + ssl = True + else: + ssl = False + return [ssl, ssl] + list(dst) + + class AddressPriority(object): """ Enum that signifies the priority of the given address when choosing the destination host. Higher is better (None < i) """ - FORCE = 5 - """forward mode""" - MANUALLY_CHANGED = 4 + MANUALLY_CHANGED = 3 """user changed the target address in the ui""" - FROM_SETTINGS = 3 - """reverse proxy mode""" - FROM_CONNECTION = 2 - """derived from transparent resolver""" + FROM_SETTINGS = 2 + """upstream server from arguments (reverse proxy, forward proxy or from transparent resolver)""" FROM_PROTOCOL = 1 """derived from protocol (e.g. absolute-form http requests)""" diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 37ec7758..c77ab2a8 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -1,10 +1,9 @@ import socket -from .. import version, protocol -from libmproxy.proxy.primitives import Log -from .primitives import ProxyServerError -from .connection import ClientConnection, ServerConnection -from .primitives import ProxyError, ConnectionTypeChange, AddressPriority +from OpenSSL import SSL from netlib import tcp +from .primitives import ProxyServerError, Log, ProxyError, ConnectionTypeChange, AddressPriority +from .connection import ClientConnection, ServerConnection +from .. import version, protocol class DummyServer: @@ -23,6 +22,7 @@ class DummyServer: class ProxyServer(tcp.TCPServer): allow_reuse_address = True bound = True + def __init__(self, config, port, host='', server_version=version.NAMEVERSION): """ Raises ProxyServerError if there's a startup problem. @@ -51,20 +51,17 @@ class ProxyServer(tcp.TCPServer): class ConnectionHandler: def __init__(self, config, client_connection, client_address, server, channel, server_version): self.config = config + """@type: libmproxy.proxy.config.ProxyConfig""" self.client_conn = ClientConnection(client_connection, client_address, server) + """@type: libmproxy.proxy.connection.ClientConnection""" self.server_conn = None + """@type: libmproxy.proxy.connection.ServerConnection""" self.channel, self.server_version = channel, server_version self.close = False self.conntype = None self.sni = None - self.mode = "regular" - if self.config.reverse_proxy: - self.mode = "reverse" - if self.config.transparent_proxy: - self.mode = "transparent" - def handle(self): self.log("clientconnect") self.channel.ask("clientconnect", self) @@ -74,25 +71,13 @@ class ConnectionHandler: try: try: # Can we already identify the target server and connect to it? - server_address = None - address_priority = None - if self.config.forward_proxy: - server_address = self.config.forward_proxy[1:] - address_priority = AddressPriority.FORCE - elif self.config.reverse_proxy: - server_address = self.config.reverse_proxy[1:] - address_priority = AddressPriority.FROM_SETTINGS - elif self.config.transparent_proxy: - server_address = self.config.transparent_proxy["resolver"].original_addr( - self.client_conn.connection) - if not server_address: - raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") - address_priority = AddressPriority.FROM_CONNECTION - self.log("transparent to %s:%s" % server_address) - - if server_address: - self.set_server_address(server_address, address_priority) - self._handle_ssl() + if self.config.get_upstream_server: + upstream_info = self.config.get_upstream_server(self.client_conn.connection) + self.set_server_address(upstream_info[2:], AddressPriority.FROM_SETTINGS) + client_ssl, server_ssl = upstream_info[:2] + if client_ssl or server_ssl: + self.establish_server_connection() + self.establish_ssl(client=client_ssl, server=server_ssl) while not self.close: try: @@ -114,28 +99,9 @@ class ConnectionHandler: self.log("clientdisconnect") self.channel.tell("clientdisconnect", self) - def _handle_ssl(self): - """ - Helper function of .handle() - Check if we can already identify SSL connections. - If so, connect to the server and establish an SSL connection - """ - client_ssl = False - server_ssl = False - - if self.config.transparent_proxy: - client_ssl = server_ssl = (self.server_conn.address.port in self.config.transparent_proxy["sslports"]) - elif self.config.reverse_proxy: - client_ssl = server_ssl = (self.config.reverse_proxy[0] == "https") - # TODO: Make protocol generic (as with transparent proxies) - # TODO: Add SSL-terminating capatbility (SSL -> mitmproxy -> plain and vice versa) - if client_ssl or server_ssl: - self.establish_server_connection() - self.establish_ssl(client=client_ssl, server=server_ssl) - def del_server_connection(self): """ - Deletes an existing server connection. + Deletes (and closes) an existing server connection. """ if self.server_conn and self.server_conn.connection: self.server_conn.finish() @@ -152,7 +118,6 @@ class ConnectionHandler: """ Sets a new server address with the given priority. Does not re-establish either connection or SSL handshake. - @type priority: libmproxy.proxy.primitives.AddressPriority """ address = tcp.Address.wrap(address) @@ -188,8 +153,7 @@ class ConnectionHandler: """ Establishes SSL on the existing connection(s) to the server or the client, as specified by the parameters. If the target server is on the pass-through list, - the conntype attribute will be changed and the SSL connection won't be wrapped. - A protocol handler must raise a ConnTypeChanged exception if it detects that this is happening + the conntype attribute will be changed and a ConnTypeChanged exception will be raised. """ # TODO: Implement SSL pass-through handling and change conntype passthrough = [ @@ -198,7 +162,7 @@ class ConnectionHandler: ] if self.server_conn.address.host in passthrough or self.sni in passthrough: self.conntype = "tcp" - return + raise ConnectionTypeChange # Logging if client or server: diff --git a/libmproxy/utils.py b/libmproxy/utils.py index 1ec4685d..17ad6f09 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -1,7 +1,6 @@ import os, datetime, urllib, re import time, functools, cgi import json -from netlib import http def timestamp(): """ @@ -143,14 +142,6 @@ class LRUCache: return ret return wrap - -def parse_proxy_spec(url): - p = http.parse_url(url) - if not p or not p[1]: - return None - return p[:3] - - def parse_content_type(c): """ A simple parser for content-type values. Returns a (type, subtype, diff --git a/test/test_cmdline.py b/test/test_cmdline.py index dbc61bfc..12e8aa89 100644 --- a/test/test_cmdline.py +++ b/test/test_cmdline.py @@ -36,6 +36,17 @@ def test_parse_replace_hook(): ) +def test_parse_server_spec(): + tutils.raises("Invalid server specification", cmdline.parse_server_spec, "") + assert cmdline.parse_server_spec("http://foo.com:88") == [False, False, "foo.com", 88] + assert cmdline.parse_server_spec("http://foo.com") == [False, False, "foo.com", 80] + assert cmdline.parse_server_spec("https://foo.com") == [True, True, "foo.com", 443] + assert cmdline.parse_server_spec("https2http://foo.com") == [True, False, "foo.com", 80] + assert cmdline.parse_server_spec("http2https://foo.com") == [False, True, "foo.com", 443] + tutils.raises("Invalid server specification", cmdline.parse_server_spec, "foo.com") + tutils.raises("Invalid server specification", cmdline.parse_server_spec, "http://") + + def test_parse_setheaders(): x = cmdline.parse_setheader("/foo/bar/voing") assert x == ("foo", "bar", "voing") diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index 6ff0cb65..290d82a1 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -90,7 +90,7 @@ class TestInvalidRequests(tservers.HTTPProxTest): def test_double_connect(self): p = self.pathoc() r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port)) - assert r.status_code == 502 + assert r.status_code == 400 assert "Must not CONNECT on already encrypted connection" in r.content def test_relative_request(self): @@ -98,7 +98,7 @@ class TestInvalidRequests(tservers.HTTPProxTest): p.connect() r = p.request("get:/p/200") assert r.status_code == 400 - assert "Invalid request form" in r.content + assert "Invalid HTTP request form" in r.content class TestProxyChaining(tservers.HTTPChainProxyTest): diff --git a/test/test_proxy.py b/test/test_proxy.py index f53aa762..0f438482 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -43,28 +43,15 @@ class TestServerConnection: sc.finish() -class MockParser: - def __init__(self): - self.err = None - - def error(self, e): - self.err = e - - def __repr__(self): - return "ParseError(%s)"%self.err - - class TestProcessProxyOptions: def p(self, *args): - parser = argparse.ArgumentParser() + parser = tutils.MockParser() cmdline.common_options(parser) opts = parser.parse_args(args=args) - m = MockParser() - return m, process_proxy_options(m, opts) + return parser, process_proxy_options(parser, opts) def assert_err(self, err, *args): - m, p = self.p(*args) - assert err.lower() in m.err.lower() + tutils.raises(err, self.p, *args) def assert_noerr(self, *args): m, p = self.p(*args) @@ -84,11 +71,10 @@ class TestProcessProxyOptions: @mock.patch("libmproxy.platform.resolver") def test_transparent_reverse(self, o): - self.assert_err("can't set both", "-P", "reverse", "-T") + self.assert_err("mutually exclusive", "-R", "http://localhost", "-T") self.assert_noerr("-T") - assert o.call_count == 1 - self.assert_err("invalid reverse proxy", "-P", "reverse") - self.assert_noerr("-P", "http://localhost") + self.assert_err("Invalid server specification", "-R", "reverse") + self.assert_noerr("-R", "http://localhost") def test_client_certs(self): with tutils.tmpdir() as confdir: diff --git a/test/test_utils.py b/test/test_utils.py index f40e0a31..11c26dba 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -87,14 +87,6 @@ def test_LRUCache(): assert len(f._cachelist_one) == 2 -def test_parse_proxy_spec(): - assert not utils.parse_proxy_spec("") - assert utils.parse_proxy_spec("http://foo.com:88") == ("http", "foo.com", 88) - assert utils.parse_proxy_spec("http://foo.com") == ("http", "foo.com", 80) - assert not utils.parse_proxy_spec("foo.com") - assert not utils.parse_proxy_spec("http://") - - def test_unparse_url(): assert utils.unparse_url("http", "foo.com", 99, "") == "http://foo.com:99" assert utils.unparse_url("http", "foo.com", 80, "") == "http://foo.com" diff --git a/test/tservers.py b/test/tservers.py index bfafc8cd..9ad3b735 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -4,6 +4,7 @@ import shutil, tempfile import flask from libmproxy.proxy.config import ProxyConfig from libmproxy.proxy.server import ProxyServer +from libmproxy.proxy.primitives import TransparentUpstreamServerResolver import libpathod.test, libpathod.pathoc from libmproxy import flow, controller from libmproxy.cmdline import APP_HOST, APP_PORT @@ -193,10 +194,9 @@ class TransparentProxTest(ProxTestBase): ports = [cls.server.port, cls.server2.port] else: ports = [] - d["transparent_proxy"] = dict( - resolver = cls.resolver(cls.server.port), - sslports = ports - ) + d["get_upstream_server"] = TransparentUpstreamServerResolver(cls.resolver(cls.server.port), ports) + d["http_form_in"] = "relative" + d["http_form_out"] = "relative" return d def pathod(self, spec, sni=None): @@ -225,11 +225,14 @@ class ReverseProxTest(ProxTestBase): @classmethod def get_proxy_config(cls): d = ProxTestBase.get_proxy_config() - d["reverse_proxy"] = ( - "https" if cls.ssl else "http", - "127.0.0.1", - cls.server.port - ) + d["get_upstream_server"] = lambda c: ( + True if cls.ssl else False, + True if cls.ssl else False, + "127.0.0.1", + cls.server.port + ) + d["http_form_in"] = "relative" + d["http_form_out"] = "relative" return d def pathoc(self, sni=None): @@ -258,18 +261,17 @@ class ChainProxTest(ProxTestBase): Chain n instances of mitmproxy in a row - because we can. """ n = 2 - chain_config = [lambda: ProxyConfig( + chain_config = [lambda port: ProxyConfig( + get_upstream_server = lambda c: (False, False, "127.0.0.1", port), + http_form_in = "absolute", + http_form_out = "absolute" )] * n @classmethod def setupAll(cls): super(ChainProxTest, cls).setupAll() cls.chain = [] for i in range(cls.n): - config = cls.chain_config[i]() - config.forward_proxy = ("http", "127.0.0.1", - cls.proxy.port if i == 0 else - cls.chain[-1].port - ) + config = cls.chain_config[i](cls.proxy.port if i == 0 else cls.chain[-1].port) tmaster = cls.masterclass(config) tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp) cls.chain.append(ProxyThread(tmaster)) diff --git a/test/tutils.py b/test/tutils.py index 3f6592b0..c527a64a 100644 --- a/test/tutils.py +++ b/test/tutils.py @@ -1,4 +1,4 @@ -import os, shutil, tempfile +import os, shutil, tempfile, argparse from contextlib import contextmanager from libmproxy import flow, utils, controller from libmproxy.protocol import http @@ -136,6 +136,15 @@ def tmpdir(*args, **kwargs): shutil.rmtree(temp_workdir) +class MockParser(argparse.ArgumentParser): + """ + argparse.ArgumentParser sys.exits() by default. + Make it more testable by throwing an exception instead. + """ + def error(self, message): + raise Exception(message) + + def raises(exc, obj, *args, **kwargs): """ Assert that a callable raises a specified exception. |