add advanced proxying options, add SSL-terminating capability to mitmproxy

8 files changed, 140 insertions, 117 deletions

diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py
index 72c13769..6c0cae9f 100644
--- a/libmproxy/cmdline.py
+++ b/libmproxy/cmdline.py
@@ -1,12 +1,16 @@
 from . import proxy
 import re, filt
 import argparse
+from argparse import ArgumentTypeError
+from netlib import http
 
 APP_HOST = "mitm.it"
 APP_PORT = 80
 
-class ParseException(Exception): pass
-class OptionException(Exception): pass
+
+class ParseException(Exception):
+    pass
+
 
 def _parse_hook(s):
     sep, rem = s[0], s[1:]
@@ -91,6 +95,26 @@ def parse_setheader(s):
     return _parse_hook(s)
 
 
+def parse_server_spec(url):
+
+    normalized_url = re.sub("^https?2", "", url)
+
+    p = http.parse_url(normalized_url)
+    if not p or not p[1]:
+        raise ArgumentTypeError("Invalid server specification: %s" % url)
+
+    if url.lower().startswith("https2http"):
+        ssl = [True, False]
+    elif url.lower().startswith("http2https"):
+        ssl = [False, True]
+    elif url.lower().startswith("https"):
+        ssl = [True, True]
+    else:
+        ssl = [False, False]
+
+    return ssl + list(p[1:3])
+
+
 def get_common_options(options):
     stickycookie, stickyauth = None, None
     if options.stickycookie_filt:
@@ -104,17 +128,17 @@ def get_common_options(options):
         try:
             p = parse_replace_hook(i)
         except ParseException, e:
-            raise OptionException(e.message)
+            raise ArgumentTypeError(e.message)
         reps.append(p)
     for i in options.replace_file:
         try:
             patt, rex, path = parse_replace_hook(i)
         except ParseException, e:
-            raise OptionException(e.message)
+            raise ArgumentTypeError(e.message)
         try:
             v = open(path, "rb").read()
         except IOError, e:
-            raise OptionException("Could not read replace file: %s"%path)
+            raise ArgumentTypeError("Could not read replace file: %s"%path)
         reps.append((patt, rex, v))
 
 
@@ -123,7 +147,7 @@ def get_common_options(options):
         try:
             p = parse_setheader(i)
         except ParseException, e:
-            raise OptionException(e.message)
+            raise ArgumentTypeError(e.message)
         setheaders.append(p)
 
     return dict(
@@ -185,17 +209,49 @@ def common_options(parser):
         action="store", type = int, dest="port", default=8080,
         help = "Proxy service port."
     )
+    # We could make a mutually exclusive group out of -R, -F, -T, but we don't do because
+    #  - --upstream-server should be in that group as well, but it's already in a different group.
+    #  - our own error messages are more helpful
     parser.add_argument(
-        "-P",
-        action="store", dest="reverse_proxy", default=None,
+        "-R",
+        action="store", type=parse_server_spec, dest="reverse_proxy", default=None,
         help="Reverse proxy to upstream server: http[s]://host[:port]"
     )
     parser.add_argument(
         "-F",
-        action="store", dest="forward_proxy", default=None,
+        action="store", type=parse_server_spec, dest="forward_proxy", default=None,
         help="Proxy to unconditionally forward to: http[s]://host[:port]"
     )
     parser.add_argument(
+        "-T",
+        action="store_true", dest="transparent_proxy", default=False,
+        help="Set transparent proxy mode."
+    )
+
+    group = parser.add_argument_group(
+        "Advanced Proxy Options",
+        """
+            The following options allow a custom adjustment of the proxy behavior.
+            Normally, you don't want to use these options directly and use the provided wrappers instead (-R, -F, -T).
+        """.strip()
+    )
+    group.add_argument(
+        "--http-form-in", dest="http_form_in", default=None,
+        action="store", choices=("relative", "absolute"),
+        help="Override the HTTP request form accepted by the proxy"
+    )
+    group.add_argument(
+        "--http-form-out", dest="http_form_out", default=None,
+        action="store", choices=("relative", "absolute"),
+        help="Override the HTTP request form sent upstream by the proxy"
+    )
+    group.add_argument(
+        "--upstream-server", dest="manual_upstream_server", default=None,
+        action="store", type=parse_server_spec,
+        help="Override the destination server all requests are sent to."
+    )
+
+    parser.add_argument(
         "-q",
         action="store_true", dest="quiet",
         help="Quiet."
@@ -217,11 +273,6 @@ def common_options(parser):
         help="Set sticky cookie filter. Matched against requests."
     )
     parser.add_argument(
-        "-T",
-        action="store_true", dest="transparent_proxy", default=False,
-        help="Set transparent proxy mode."
-    )
-    parser.add_argument(
         "-u",
         action="store", dest="stickyauth_filt", default=None, metavar="FILTER",
         help="Set sticky auth filter. Matched against requests."
diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py
index 4a58e771..846abba7 100644
--- a/libmproxy/console/__init__.py
+++ b/libmproxy/console/__init__.py
@@ -144,10 +144,6 @@ class StatusBar(common.WWrap):
             r.append("[")
             r.append(("heading_key", "u"))
             r.append(":%s]"%self.master.stickyauth_txt)
-        if self.master.server.config.reverse_proxy:
-            r.append("[")
-            r.append(("heading_key", "P"))
-            r.append(":%s]"%utils.unparse_url(*self.master.server.config.reverse_proxy))
         if self.master.state.default_body_view.name != "Auto":
             r.append("[")
             r.append(("heading_key", "M"))
@@ -172,6 +168,8 @@ class StatusBar(common.WWrap):
         if opts:
             r.append("[%s]"%(":".join(opts)))
 
+        if self.master.server.config.upstream_server:
+            r.append("[dest:%s]"%utils.unparse_url(*self.master.server.config.upstream_server))
         if self.master.scripts:
             r.append("[scripts:%s]"%len(self.master.scripts))
         if self.master.debug:
@@ -763,15 +761,6 @@ class ConsoleMaster(flow.FlowMaster):
         self.state.default_body_view = v
         self.refresh_focus()
 
-    def set_reverse_proxy(self, txt):
-        if not txt:
-            self.server.config.reverse_proxy = None
-        else:
-            s = utils.parse_proxy_spec(txt)
-            if not s:
-                return "Invalid reverse proxy specification"
-            self.server.config.reverse_proxy = s
-
     def drawscreen(self):
         size = self.ui.get_cols_rows()
         canvas = self.view.render(size, focus=1)
@@ -866,16 +855,6 @@ class ConsoleMaster(flow.FlowMaster):
                                     contentview.view_prompts,
                                     self.change_default_display_mode
                                 )
-                            elif k == "P":
-                                if self.server.config.reverse_proxy:
-                                    p = utils.unparse_url(*self.server.config.reverse_proxy)
-                                else:
-                                    p = ""
-                                self.prompt(
-                                    "Reverse proxy: ",
-                                    p,
-                                    self.set_reverse_proxy
-                                )
                             elif k == "R":
                                 self.view_grideditor(
                                     grideditor.ReplaceEditor(
diff --git a/libmproxy/console/help.py b/libmproxy/console/help.py
index 0d01ac6f..b0b7c48c 100644
--- a/libmproxy/console/help.py
+++ b/libmproxy/console/help.py
@@ -109,7 +109,6 @@ class HelpView(urwid.ListBox):
 
             ("q", "quit / return to flow list"),
             ("Q", "quit without confirm prompt"),
-            ("P", "set reverse proxy mode"),
             ("R", "edit replacement patterns"),
             ("s", "set/unset script"),
             ("S", "server replay"),
diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py
index c8147a3c..aff4a817 100644
--- a/libmproxy/protocol/http.py
+++ b/libmproxy/protocol/http.py
@@ -1027,7 +1027,7 @@ class HTTPHandler(ProtocolHandler, TemporaryServerChangeMixin):
                 raise http.HttpError(400, "Must not CONNECT on already encrypted connection")
 
             if self.expected_form_in == "absolute":
-                if not self.c.config.upstream_server:
+                if not self.c.config.get_upstream_server:
                     self.c.set_server_address((request.host, request.port), AddressPriority.FROM_PROTOCOL)
                     flow.server_conn = self.c.server_conn  # Update server_conn attribute on the flow
                     self.c.client_conn.send(
diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py
index 248808e9..ae24d4c9 100644
--- a/libmproxy/proxy/config.py
+++ b/libmproxy/proxy/config.py
@@ -1,7 +1,7 @@
 import os
 from .. import utils, platform
 from netlib import http_auth, certutils
-
+from .primitives import ConstUpstreamServerResolver, TransparentUpstreamServerResolver
 
 TRANSPARENT_SSL_PORTS = [443, 8443]
 CONF_BASENAME = "mitmproxy"
@@ -10,18 +10,17 @@ CONF_DIR = "~/.mitmproxy"
 
 class ProxyConfig:
     def __init__(self, confdir=CONF_DIR, clientcerts=None,
-                       no_upstream_cert=False, body_size_limit=None, upstream_server=None,
-                       http_form_in="absolute", http_form_out="relative", transparent_proxy=None, authenticator=None,
+                       no_upstream_cert=False, body_size_limit=None, get_upstream_server=None,
+                       http_form_in="absolute", http_form_out="relative", authenticator=None,
                        ciphers=None, certs=None
                 ):
         self.ciphers = ciphers
         self.clientcerts = clientcerts
         self.no_upstream_cert = no_upstream_cert
         self.body_size_limit = body_size_limit
-        self.upstream_server = upstream_server
+        self.get_upstream_server = get_upstream_server
         self.http_form_in = http_form_in
         self.http_form_out = http_form_out
-        self.transparent_proxy = transparent_proxy
         self.authenticator = authenticator
         self.confdir = os.path.expanduser(confdir)
         self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME)
@@ -29,32 +28,34 @@ class ProxyConfig:
 
 def process_proxy_options(parser, options):
     body_size_limit = utils.parse_size(options.body_size_limit)
-    if options.reverse_proxy and options.transparent_proxy:
-        return parser.error("Can't set both reverse proxy and transparent proxy.")
 
+    c = 0
+    http_form_in, http_form_out = "absolute", "relative"
+    get_upstream_server = None
     if options.transparent_proxy:
+        c += 1
         if not platform.resolver:
             return parser.error("Transparent mode not supported on this platform.")
-        trans = dict(
-            resolver=platform.resolver(),
-            sslports=TRANSPARENT_SSL_PORTS
-        )
-    else:
-        trans = None
-
+        get_upstream_server = TransparentUpstreamServerResolver(platform.resolver(), TRANSPARENT_SSL_PORTS)
+        http_form_in, http_form_out = "relative", "relative"
     if options.reverse_proxy:
-        rp = utils.parse_proxy_spec(options.reverse_proxy)
-        if not rp:
-            return parser.error("Invalid reverse proxy specification: %s" % options.reverse_proxy)
-    else:
-        rp = None
-
+        c += 1
+        get_upstream_server = ConstUpstreamServerResolver(options.reverse_proxy)
+        http_form_in, http_form_out = "relative", "relative"
     if options.forward_proxy:
-        fp = utils.parse_proxy_spec(options.forward_proxy)
-        if not fp:
-            return parser.error("Invalid forward proxy specification: %s" % options.forward_proxy)
-    else:
-        fp = None
+        c += 1
+        get_upstream_server = ConstUpstreamServerResolver(options.forward_proxy)
+        http_form_in, http_form_out = "absolute", "absolute"
+    if options.manual_upstream_server:
+        c += 1
+        get_upstream_server = ConstUpstreamServerResolver(options.manual_upstream_server)
+    if c > 1:
+        return parser.error("Transparent mode, reverse mode, forward mode and "
+                            "specification of an upstream server are mutually exclusive.")
+    if options.http_form_in:
+        http_form_in = options.http_form_in
+    if options.http_form_out:
+        http_form_out = options.http_form_out
 
     if options.clientcerts:
         options.clientcerts = os.path.expanduser(options.clientcerts)
@@ -94,10 +95,9 @@ def process_proxy_options(parser, options):
         clientcerts=options.clientcerts,
         body_size_limit=body_size_limit,
         no_upstream_cert=options.no_upstream_cert,
-        upstream_server=(rp or fp),
-        http_form_in=("relative" if (rp or trans) else "absolute"),
-        http_form_out=("absolute" if fp else "relative"),
-        transparent_proxy=trans,
+        get_upstream_server=get_upstream_server,
+        http_form_in=http_form_in,
+        http_form_out=http_form_out,
         authenticator=authenticator,
         ciphers=options.ciphers,
         certs = certs,
diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py
index 75ad5482..e49d9bb4 100644
--- a/libmproxy/proxy/primitives.py
+++ b/libmproxy/proxy/primitives.py
@@ -18,17 +18,48 @@ class ProxyServerError(Exception):
     pass
 
 
+class UpstreamServerResolver(object):
+    def __call__(self, conn):
+        """
+        Returns the address of the server to connect to.
+        """
+        raise NotImplementedError
+
+
+class ConstUpstreamServerResolver(UpstreamServerResolver):
+    def __init__(self, dst):
+        self.dst = dst
+
+    def __call__(self, conn):
+        return self.dst
+
+
+class TransparentUpstreamServerResolver(UpstreamServerResolver):
+    def __init__(self, resolver, sslports):
+        self.resolver = resolver
+        self.sslports = sslports
+
+    def __call__(self, conn):
+        dst = self.resolver.original_addr(conn)
+        if not dst:
+            raise ProxyError(502, "Transparent mode failure: could not resolve original destination.")
+
+        if dst[1] in self.sslports:
+            ssl = True
+        else:
+            ssl = False
+        return [ssl, ssl] + list(dst)
+
+
 class AddressPriority(object):
     """
     Enum that signifies the priority of the given address when choosing the destination host.
     Higher is better (None < i)
     """
-    MANUALLY_CHANGED = 4
+    MANUALLY_CHANGED = 3
     """user changed the target address in the ui"""
-    FROM_SETTINGS = 3
-    """upstream proxy from arguments (reverse proxy or forward proxy)"""
-    FROM_CONNECTION = 2
-    """derived from transparent resolver"""
+    FROM_SETTINGS = 2
+    """upstream server from arguments (reverse proxy, forward proxy or from transparent resolver)"""
     FROM_PROTOCOL = 1
     """derived from protocol (e.g. absolute-form http requests)"""
 
diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py
index 5aaabf87..554ee551 100644
--- a/libmproxy/proxy/server.py
+++ b/libmproxy/proxy/server.py
@@ -68,22 +68,13 @@ class ConnectionHandler:
         try:
             try:
                 # Can we already identify the target server and connect to it?
-                server_address = None
-                address_priority = None
-                if self.config.upstream_server:
-                    server_address = self.config.upstream_server[1:]
-                    address_priority = AddressPriority.FROM_SETTINGS
-                elif self.config.transparent_proxy:
-                    server_address = self.config.transparent_proxy["resolver"].original_addr(
-                        self.client_conn.connection)
-                    if not server_address:
-                        raise ProxyError(502, "Transparent mode failure: could not resolve original destination.")
-                    address_priority = AddressPriority.FROM_CONNECTION
-                    self.log("transparent to %s:%s" % server_address)
-
-                if server_address:
-                    self.set_server_address(server_address, address_priority)
-                    self._handle_ssl()
+                if self.config.get_upstream_server:
+                    upstream_info = self.config.get_upstream_server(self.client_conn.connection)
+                    self.set_server_address(upstream_info[2:], AddressPriority.FROM_SETTINGS)
+                    client_ssl, server_ssl = upstream_info[:2]
+                    if client_ssl or server_ssl:
+                        self.establish_server_connection()
+                        self.establish_ssl(client=client_ssl, server=server_ssl)
 
                 while not self.close:
                     try:
@@ -105,25 +96,6 @@ class ConnectionHandler:
         self.log("clientdisconnect")
         self.channel.tell("clientdisconnect", self)
 
-    def _handle_ssl(self):
-        """
-        Helper function of .handle()
-        Check if we can already identify SSL connections.
-        If so, connect to the server and establish an SSL connection
-        """
-        client_ssl = False
-        server_ssl = False
-
-        if self.config.transparent_proxy:
-            client_ssl = server_ssl = (self.server_conn.address.port in self.config.transparent_proxy["sslports"])
-        elif self.config.upstream_server:
-            client_ssl = server_ssl = (self.config.upstream_server[0] == "https")
-            # TODO: Make protocol generic (as with transparent proxies)
-            # TODO: Add SSL-terminating capatbility (SSL -> mitmproxy -> plain and vice versa)
-        if client_ssl or server_ssl:
-            self.establish_server_connection()
-            self.establish_ssl(client=client_ssl, server=server_ssl)
-
     def del_server_connection(self):
         """
         Deletes an existing server connection.
diff --git a/libmproxy/utils.py b/libmproxy/utils.py
index 1ec4685d..17ad6f09 100644
--- a/libmproxy/utils.py
+++ b/libmproxy/utils.py
@@ -1,7 +1,6 @@
 import os, datetime, urllib, re
 import time, functools, cgi
 import json
-from netlib import http
 
 def timestamp():
     """
@@ -143,14 +142,6 @@ class LRUCache:
                 return ret
         return wrap
 
-
-def parse_proxy_spec(url):
-    p = http.parse_url(url)
-    if not p or not p[1]:
-        return None
-    return p[:3]
-
-
 def parse_content_type(c):
     """
         A simple parser for content-type values. Returns a (type, subtype,