Merge remote-tracking branch 'upstream/master'

6 files changed, 125 insertions, 193 deletions

diff --git a/libmproxy/authentication.py b/libmproxy/authentication.py
index e5383f5a..500ead6b 100644
--- a/libmproxy/authentication.py
+++ b/libmproxy/authentication.py
@@ -2,35 +2,49 @@ import binascii
 import contrib.md5crypt as md5crypt
 
 class NullProxyAuth():
-    """ No proxy auth at all (returns empty challange headers) """
-    def __init__(self, password_manager=None):
+    """
+        No proxy auth at all (returns empty challange headers)
+    """
+    def __init__(self, password_manager):
         self.password_manager = password_manager
         self.username = ""
 
-    def authenticate(self, auth_value):
-        """ Tests that the specified user is allowed to use the proxy (stub) """
+    def clean(self, headers):
+        """
+            Clean up authentication headers, so they're not passed upstream.
+        """
+        pass
+
+    def authenticate(self, headers):
+        """
+            Tests that the user is allowed to use the proxy
+        """
         return True
 
     def auth_challenge_headers(self):
-        """ Returns a dictionary containing the headers require to challenge the user """
+        """
+            Returns a dictionary containing the headers require to challenge the user
+        """
         return {}
 
-    def get_username(self):
-        return self.username
-
 
 class BasicProxyAuth(NullProxyAuth):
-
-    def __init__(self, password_manager, realm="mitmproxy"):
+    CHALLENGE_HEADER = 'Proxy-Authenticate'
+    AUTH_HEADER = 'Proxy-Authorization'
+    def __init__(self, password_manager, realm):
         NullProxyAuth.__init__(self, password_manager)
-        self.realm = "mitmproxy"
+        self.realm = realm
 
-    def authenticate(self, auth_value):
-        if (not auth_value) or (not auth_value[0]):
-            return False;
+    def clean(self, headers):
+        del headers[self.AUTH_HEADER]
+
+    def authenticate(self, headers):
+        auth_value = headers.get(self.AUTH_HEADER, [])
+        if not auth_value:
+            return False
         try:
-            scheme, username, password = self.parse_authorization_header(auth_value[0])
-        except:
+            scheme, username, password = self.parse_auth_value(auth_value[0])
+        except ValueError:
             return False
         if scheme.lower()!='basic':
             return False
@@ -40,14 +54,26 @@ class BasicProxyAuth(NullProxyAuth):
         return True
 
     def auth_challenge_headers(self):
-        return {'Proxy-Authenticate':'Basic realm="%s"'%self.realm}
+        return {self.CHALLENGE_HEADER:'Basic realm="%s"'%self.realm}
 
-    def parse_authorization_header(self, auth_value):
+    def unparse_auth_value(self, scheme, username, password):
+        v = binascii.b2a_base64(username + ":" + password)
+        return scheme + " " + v
+
+    def parse_auth_value(self, auth_value):
         words = auth_value.split()
+        if len(words) != 2:
+            raise ValueError("Invalid basic auth credential.")
         scheme = words[0]
-        user = binascii.a2b_base64(words[1])
-        username, password = user.split(':')
-        return scheme, username, password
+        try:
+            user = binascii.a2b_base64(words[1])
+        except binascii.Error:
+            raise ValueError("Invalid basic auth credential: user:password pair not valid base64: %s"%words[1])
+        parts = user.split(':')
+        if len(parts) != 2:
+            raise ValueError("Invalid basic auth credential: decoded user:password pair not valid: %s"%user)
+        return scheme, parts[0], parts[1]
+
 
 class PasswordManager():
     def __init__(self):
@@ -56,8 +82,8 @@ class PasswordManager():
     def test(self, username, password_token):
         return False
 
-class PermissivePasswordManager(PasswordManager):
 
+class PermissivePasswordManager(PasswordManager):
     def __init__(self):
         PasswordManager.__init__(self)
 
@@ -66,16 +92,17 @@ class PermissivePasswordManager(PasswordManager):
             return True
         return False
 
-class HtpasswdPasswordManager(PasswordManager):
-    """ Read usernames and passwords from a file created by Apache htpasswd"""
 
+class HtpasswdPasswordManager(PasswordManager):
+    """
+        Read usernames and passwords from a file created by Apache htpasswd
+    """
     def __init__(self, filehandle):
         PasswordManager.__init__(self)
         entries = (line.strip().split(':') for line in filehandle)
         valid_entries = (entry for entry in entries if len(entry)==2)
         self.usernames = {username:token for username,token in valid_entries}
 
-
     def test(self, username, password_token):
         if username not in self.usernames:
             return False
@@ -84,8 +111,8 @@ class HtpasswdPasswordManager(PasswordManager):
         expected = md5crypt.md5crypt(password_token, salt, '$'+magic+'$')
         return expected==full_token
 
-class SingleUserPasswordManager(PasswordManager):
 
+class SingleUserPasswordManager(PasswordManager):
     def __init__(self, username, password):
         PasswordManager.__init__(self)
         self.username = username
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py
index db1ebf0d..de70bea8 100644
--- a/libmproxy/cmdline.py
+++ b/libmproxy/cmdline.py
@@ -248,11 +248,6 @@ def common_options(parser):
         help="Byte size limit of HTTP request and response bodies."\
              " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
     )
-    parser.add_argument(
-        "--cert-wait-time", type=float,
-        action="store", dest="cert_wait_time", default=0,
-        help="Wait for specified number of seconds after a new cert is generated. This can smooth over small discrepancies between the client and server times."
-    )
 
     parser.add_argument(
         "--no-upstream-cert", default=False,
@@ -338,46 +333,29 @@ def common_options(parser):
     group = parser.add_argument_group(
         "Proxy Authentication",
         """
-            Specification of which users are allowed to access the proxy and the method used for authenticating them.
-            If authscheme is specified, one must specify a list of authorized users and their passwords.
-            In case that authscheme is not specified, or set to None, any list of authorized users will be ignored.
-        """.strip()
-    )
-
-    group.add_argument(
-        "--authscheme", type=str,
-        action="store", dest="authscheme", default=None, choices=["none", "basic"],
-        help="""
-            Specify the scheme used by the proxy to identify users.
-            If not none, requires the specification of a list of authorized users.
-            This option is ignored if the proxy is in transparent or reverse mode.
-            """.strip()
-
+            Specify which users are allowed to access the proxy and the method
+            used for authenticating them. These options are ignored if the
+            proxy is in transparent or reverse proxy mode.
+        """
     )
-
     user_specification_group = group.add_mutually_exclusive_group()
-
-
     user_specification_group.add_argument(
         "--nonanonymous",
         action="store_true", dest="auth_nonanonymous",
-        help="Allow access to any user as long as a username is specified. Ignores the provided password."
+        help="Allow access to any user long as a credentials are specified."
     )
 
     user_specification_group.add_argument(
         "--singleuser",
         action="store", dest="auth_singleuser", type=str,
-        help="Allows access to a single user as specified by the option value. Specify a username and password in the form username:password."
+        metavar="USER",
+        help="Allows access to a a single user, specified in the form username:password."
     )
-
     user_specification_group.add_argument(
         "--htpasswd",
         action="store", dest="auth_htpasswd", type=argparse.FileType('r'),
+        metavar="PATH",
         help="Allow access to users specified in an Apache htpasswd file."
     )
 
-
-
-
-
     proxy.certificate_option_group(parser)
diff --git a/libmproxy/flow.py b/libmproxy/flow.py
index 9a6b5527..2c4c5513 100755
--- a/libmproxy/flow.py
+++ b/libmproxy/flow.py
@@ -1387,6 +1387,8 @@ class FlowMaster(controller.Master):
         self.kill_nonreplay = kill
 
     def stop_server_playback(self):
+        if self.server_playback.exit:
+            self.shutdown()
         self.server_playback = None
 
     def do_server_playback(self, flow):
@@ -1420,10 +1422,6 @@ class FlowMaster(controller.Master):
                 self.shutdown()
             self.client_playback.tick(self)
 
-        if self.server_playback:
-            if self.server_playback.exit and self.server_playback.count() == 0:
-                self.shutdown()
-
         return controller.Master.tick(self, q)
 
     def duplicate_flow(self, f):
diff --git a/libmproxy/platform/osx.py b/libmproxy/platform/osx.py
index a66c03ed..dda5d9af 100644
--- a/libmproxy/platform/osx.py
+++ b/libmproxy/platform/osx.py
@@ -1,103 +1,23 @@
-import socket, ctypes
-
-# Python socket module does not have this constant
-DIOCNATLOOK = 23
-PFDEV = "/dev/pf"
-
-
-class PF_STATE_XPORT(ctypes.Union):
-    """
-        union pf_state_xport {
-            u_int16_t	port;
-            u_int16_t	call_id;
-            u_int32_t	spi;
-        };
-    """
-    _fields_ = [
-        ("port", ctypes.c_uint),
-        ("call_id", ctypes.c_uint),
-        ("spi", ctypes.c_ulong),
-    ]
-
-
-class PF_ADDR(ctypes.Union):
-    """
-        struct pf_addr {
-            union {
-                struct in_addr		v4;
-                struct in6_addr		v6;
-                u_int8_t		addr8[16];
-                u_int16_t		addr16[8];
-                u_int32_t		addr32[4];
-            } pfa;
-        }
-    """
-    _fields_ = [
-        ("addr8", ctypes.c_byte * 2),
-        ("addr16", ctypes.c_byte * 4),
-        ("addr32", ctypes.c_byte * 8),
-    ]
-
-
-class PFIOC_NATLOOK(ctypes.Structure):
-    """
-        struct pfioc_natlook {
-            struct pf_addr	 saddr;
-            struct pf_addr	 daddr;
-            struct pf_addr	 rsaddr;
-            struct pf_addr	 rdaddr;
-        #ifndef NO_APPLE_EXTENSIONS
-            union pf_state_xport	sxport;
-            union pf_state_xport	dxport;
-            union pf_state_xport	rsxport;
-            union pf_state_xport	rdxport;
-            sa_family_t	 af;
-            u_int8_t	 proto;
-            u_int8_t	 proto_variant;
-            u_int8_t	 direction;
-        #else
-            u_int16_t	 sport;
-            u_int16_t	 dport;
-            u_int16_t	 rsport;
-            u_int16_t	 rdport;
-            sa_family_t	 af;
-            u_int8_t	 proto;
-            u_int8_t	 direction;
-        #endif
-        };
-    """
-    _fields_ = [
-        ("saddr", PF_ADDR),
-        ("daddr", PF_ADDR),
-        ("rsaddr", PF_ADDR),
-        ("rdaddr", PF_ADDR),
-
-        ("sxport", PF_STATE_XPORT),
-        ("dxport", PF_STATE_XPORT),
-        ("rsxport", PF_STATE_XPORT),
-        ("rdxport", PF_STATE_XPORT),
-        ("af", ctypes.c_uint),
-        ("proto", ctypes.c_ushort),
-        ("proto_variant", ctypes.c_ushort),
-        ("direction", ctypes.c_ushort),
-    ]
+import subprocess
+import pf
 
+"""
+    Doing this the "right" way by using DIOCNATLOOK on the pf device turns out
+    to be a pain. Apple has made a number of modifications to the data
+    structures returned, and compiling userspace tools to test and work with
+    this turns out to be a pain in the ass. Parsing pfctl output is short,
+    simple, and works.
+"""
 
 class Resolver:
+    STATECMD = ("sudo", "-n", "/sbin/pfctl", "-s", "state")
     def __init__(self):
-        self.pfdev = open(PFDEV, "r")
+        pass
 
     def original_addr(self, csock):
-        """
-            The following sttruct defintions are plucked from the current XNU source, found here:
-
-                http://www.opensource.apple.com/source/xnu/xnu-1699.26.8/bsd/net/pfvar.h
-
-
-            union pf_state_xport {
-                u_int16_t	port;
-                u_int16_t	call_id;
-                u_int32_t	spi;
-            };
-        """
-        pass
+        peer = csock.getpeername()
+        try:
+            stxt = subprocess.check_output(self.STATECMD, stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError:
+            return None
+        return pf.lookup(peer[0], peer[1], stxt)
diff --git a/libmproxy/platform/pf.py b/libmproxy/platform/pf.py
new file mode 100644
index 00000000..062d3311
--- /dev/null
+++ b/libmproxy/platform/pf.py
@@ -0,0 +1,16 @@
+
+def lookup(address, port, s):
+    """
+        Parse the pfctl state output s, to look up the destination host
+        matching the client (address, port).
+
+        Returns an (address, port) tuple, or None.
+    """
+    spec = "%s:%s"%(address, port)
+    for i in s.split("\n"):
+        if "ESTABLISHED:ESTABLISHED" in i and spec in i:
+            s = i.split()
+            if len(s) > 4:
+                s = s[4].split(":")
+                if len(s) == 2:
+                    return s[0], int(s[1])
diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py
index 4c57aeb0..cf006f60 100755
--- a/libmproxy/proxy.py
+++ b/libmproxy/proxy.py
@@ -38,19 +38,19 @@ class Log(controller.Msg):
 
 
 class ProxyConfig:
-    def __init__(self, certfile = None, cacert = None, clientcerts = None, cert_wait_time=0, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, transparent_proxy=None, certdir = None, authenticator=None):
+    def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, transparent_proxy=None, certdir = None, authenticator=None):
         assert not (reverse_proxy and transparent_proxy)
         self.certfile = certfile
         self.cacert = cacert
         self.clientcerts = clientcerts
-        self.certdir = certdir
-        self.cert_wait_time = cert_wait_time
         self.no_upstream_cert = no_upstream_cert
         self.body_size_limit = body_size_limit
         self.reverse_proxy = reverse_proxy
         self.transparent_proxy = transparent_proxy
         self.authenticator = authenticator
 
+        self.certstore = certutils.CertStore(certdir)
+
 class RequestReplayThread(threading.Thread):
     def __init__(self, config, flow, masterq):
         self.config, self.flow, self.masterq = config, flow, masterq
@@ -247,8 +247,7 @@ class ProxyHandler(tcp.BaseHandler):
                     raise ProxyError(502, "Unable to get remote cert: %s"%str(v))
                 sans = cert.altnames
                 host = cert.cn.decode("utf8").encode("idna")
-            ret = certutils.dummy_cert(self.config.certdir, self.config.cacert, host, sans)
-            time.sleep(self.config.cert_wait_time)
+            ret = self.config.certstore.get_cert(host, sans, self.config.cacert)
             if not ret:
                 raise ProxyError(502, "mitmproxy: Unable to generate dummy cert.")
             return ret
@@ -270,7 +269,10 @@ class ProxyHandler(tcp.BaseHandler):
     def read_request(self, client_conn):
         self.rfile.reset_timestamps()
         if self.config.transparent_proxy:
-            host, port = self.config.transparent_proxy["resolver"].original_addr(self.connection)
+            orig = self.config.transparent_proxy["resolver"].original_addr(self.connection)
+            if not orig:
+                raise ProxyError(502, "Transparent mode failure: could not resolve original destination.")
+            host, port = orig
             if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]):
                 scheme = "https"
                 certfile = self.find_cert(host, port, None)
@@ -311,7 +313,7 @@ class ProxyHandler(tcp.BaseHandler):
             line = self.get_line(self.rfile)
             if line == "":
                 return None
-            if line.startswith("CONNECT"):
+            if http.parse_init_connect(line):
                 r = http.parse_init_connect(line)
                 if not r:
                     raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
@@ -332,14 +334,15 @@ class ProxyHandler(tcp.BaseHandler):
                     raise ProxyError(400, str(v))
                 self.proxy_connect_state = (host, port, httpversion)
                 line = self.rfile.readline(line)
+
             if self.proxy_connect_state:
-                host, port, httpversion = self.proxy_connect_state
                 r = http.parse_init_http(line)
                 if not r:
                     raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
                 method, path, httpversion = r
                 headers = self.read_headers(authenticate=False)
 
+                host, port, _ = self.proxy_connect_state
                 content = http.read_http_body_request(
                     self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                 )
@@ -348,7 +351,7 @@ class ProxyHandler(tcp.BaseHandler):
                 r = http.parse_init_proxy(line)
                 if not r:
                     raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
-                method, scheme, host, port, path, httpversion = http.parse_init_proxy(line)
+                method, scheme, host, port, path, httpversion = r
                 headers = self.read_headers(authenticate=True)
                 content = http.read_http_body_request(
                     self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
@@ -359,8 +362,15 @@ class ProxyHandler(tcp.BaseHandler):
         headers = http.read_headers(self.rfile)
         if headers is None:
             raise ProxyError(400, "Invalid headers")
-        if authenticate and self.config.authenticator and not self.config.authenticator.authenticate(headers.get('Proxy-Authorization', [])):
-            raise ProxyError(407, "Proxy Authentication Required", self.config.authenticator.auth_challenge_headers())
+        if authenticate and self.config.authenticator:
+            if self.config.authenticator.authenticate(headers):
+                self.config.authenticator.clean(headers)
+            else:
+                raise ProxyError(
+                            407,
+                            "Proxy Authentication Required",
+                            self.config.authenticator.auth_challenge_headers()
+                       )
         return headers
 
     def send_response(self, response):
@@ -405,13 +415,6 @@ class ProxyServer(tcp.TCPServer):
         except socket.error, v:
             raise ProxyServerError('Error starting proxy server: ' + v.strerror)
         self.masterq = None
-        if config.certdir:
-            self.certdir = config.certdir
-            self.remove_certdir = False
-        else:
-            self.certdir = tempfile.mkdtemp(prefix="mitmproxy")
-            config.certdir = self.certdir
-            self.remove_certdir = True
         self.apps = AppRegistry()
 
     def start_slave(self, klass, masterq):
@@ -430,11 +433,7 @@ class ProxyServer(tcp.TCPServer):
             pass
 
     def handle_shutdown(self):
-        try:
-            if self.remove_certdir:
-                shutil.rmtree(self.certdir)
-        except OSError:
-            pass
+        self.config.certstore.cleanup()
 
 
 class AppRegistry:
@@ -537,30 +536,24 @@ def process_proxy_options(parser, options):
         if not os.path.exists(options.certdir) or not os.path.isdir(options.certdir):
             parser.error("Dummy cert directory does not exist or is not a directory: %s"%options.certdir)
 
-    if options.authscheme and (options.authscheme!='none'):
-        if not (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd):
-            parser.error("Proxy authentication scheme is specified, but no allowed user list is given.")
-        if options.auth_singleuser and len(options.auth_singleuser.split(':'))!=2:
-            parser.error("Authorized user is not given in correct format username:password")
-        if options.auth_nonanonymous:
-            password_manager = authentication.PermissivePasswordManager()
-        elif options.auth_singleuser:
+    if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd):
+        if options.auth_singleuser:
+            if len(options.auth_singleuser.split(':')) != 2:
+                parser.error("Please specify user in the format username:password")
             username, password = options.auth_singleuser.split(':')
             password_manager = authentication.SingleUserPasswordManager(username, password)
+        elif options.auth_nonanonymous:
+            password_manager = authentication.PermissivePasswordManager()
         elif options.auth_htpasswd:
             password_manager = authentication.HtpasswdPasswordManager(options.auth_htpasswd)
-        # in the meanwhile, basic auth is the only true authentication scheme we support
-        # so just use it
-        authenticator = authentication.BasicProxyAuth(password_manager)
+        authenticator = authentication.BasicProxyAuth(password_manager, "mitmproxy")
     else:
         authenticator = authentication.NullProxyAuth(None)
 
-
     return ProxyConfig(
         certfile = options.cert,
         cacert = cacert,
         clientcerts = options.clientcerts,
-        cert_wait_time = options.cert_wait_time,
         body_size_limit = body_size_limit,
         no_upstream_cert = options.no_upstream_cert,
         reverse_proxy = rp,