Add upstream certificate lookup.

This initiates a connection to the server to obtain certificate information to
generate interception certificates. At the moment, the information used is the
Common Name, and the list of Subject Alternative Names.

7 files changed, 167 insertions, 17 deletions

diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py
index ee4f3b08..42c02449 100644
--- a/libmproxy/cmdline.py
+++ b/libmproxy/cmdline.py
@@ -141,6 +141,17 @@ def common_options(parser):
         help="Byte size limit of HTTP request and response bodies."\
              " Understands k/m/g suffixes, i.e. 3m for 3 megabytes."
     )
+    parser.add_option(
+        "--cert-wait-time", type="float",
+        action="store", dest="cert_wait_time", default=0,
+        help="Wait for specified number of seconds after a new cert is generated. This can smooth over small discrepancies between the client and server times."
+    )
+    parser.add_option(
+        "--upstream-cn-lookup", default=False,
+        action="store_true", dest="upstream_cn_lookup",
+        help="Connect to upstream server to look up certificate Common Name."
+    )
+
     group = optparse.OptionGroup(parser, "Client Replay")
     group.add_option(
         "-c",
@@ -149,12 +160,6 @@ def common_options(parser):
     )
     parser.add_option_group(group)
 
-    parser.add_option(
-        "--cert-wait-time", type="float",
-        action="store", dest="cert_wait_time", default=0,
-        help="Wait for specified number of seconds after a new cert is generated. This can smooth over small discrepancies between the client and server times."
-    )
-
     group = optparse.OptionGroup(parser, "Server Replay")
     group.add_option(
         "-S",
diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py
index 3a7f807e..ec7c52e4 100644
--- a/libmproxy/proxy.py
+++ b/libmproxy/proxy.py
@@ -35,12 +35,13 @@ class ProxyError(Exception):
 
 
 class ProxyConfig:
-    def __init__(self, certfile = None, ciphers = None, cacert = None, cert_wait_time=0, body_size_limit = None, reverse_proxy=None):
+    def __init__(self, certfile = None, ciphers = None, cacert = None, cert_wait_time=0, upstream_cn_lookup=False, body_size_limit = None, reverse_proxy=None):
         self.certfile = certfile
         self.ciphers = ciphers
         self.cacert = cacert
         self.certdir = None
         self.cert_wait_time = cert_wait_time
+        self.upstream_cn_lookup = upstream_cn_lookup
         self.body_size_limit = body_size_limit
         self.reverse_proxy = reverse_proxy
 
@@ -343,11 +344,14 @@ class ProxyHandler(SocketServer.StreamRequestHandler):
         if server:
             server.terminate()
 
-    def find_cert(self, host):
+    def find_cert(self, host, port):
         if self.config.certfile:
             return self.config.certfile
         else:
-            ret = utils.dummy_cert(self.config.certdir, self.config.cacert, host)
+            sans = []
+            if self.config.upstream_cn_lookup:
+                host, sans = utils.get_remote_cn(host, port)
+            ret = utils.dummy_cert(self.config.certdir, self.config.cacert, host, sans)
             time.sleep(self.config.cert_wait_time)
             if not ret:
                 raise ProxyError(502, "mitmproxy: Unable to generate dummy cert.")
@@ -374,7 +378,7 @@ class ProxyHandler(SocketServer.StreamRequestHandler):
                         )
             self.wfile.flush()
             kwargs = dict(
-                certfile = self.find_cert(host),
+                certfile = self.find_cert(host, port),
                 keyfile = self.config.certfile or self.config.cacert,
                 server_side = True,
                 ssl_version = ssl.PROTOCOL_SSLv23,
@@ -538,5 +542,6 @@ def process_proxy_options(parser, options):
         ciphers = options.ciphers,
         cert_wait_time = options.cert_wait_time,
         body_size_limit = body_size_limit,
+        upstream_cn_lookup = options.upstream_cn_lookup,
         reverse_proxy = rp
     )
diff --git a/libmproxy/resources/cert.cnf b/libmproxy/resources/cert.cnf
index 5f80c2d6..4f2525a9 100644
--- a/libmproxy/resources/cert.cnf
+++ b/libmproxy/resources/cert.cnf
@@ -28,3 +28,5 @@ basicConstraints = CA:false
 keyUsage = nonRepudiation, digitalSignature, keyEncipherment
 nsCertType = server
 
+[ alt_names ]
+%(sans)s
diff --git a/libmproxy/utils.py b/libmproxy/utils.py
index 16540434..3381ad33 100644
--- a/libmproxy/utils.py
+++ b/libmproxy/utils.py
@@ -12,8 +12,8 @@
 #
 # You should have received a copy of the GNU General Public License
 # along with this program.  If not, see <http://www.gnu.org/licenses/>.
-import re, os, subprocess, datetime, urlparse, string, urllib
-import time, functools, cgi, textwrap, hashlib
+import re, os, subprocess, datetime, urlparse, string, urllib, socket
+import time, functools, cgi, textwrap, hashlib, ssl, tempfile
 import json
 
 CERT_SLEEP_TIME = 1
@@ -276,7 +276,7 @@ def dummy_ca(path):
     return True
 
 
-def dummy_cert(certdir, ca, commonname):
+def dummy_cert(certdir, ca, commonname, sans):
     """
         certdir: Certificate directory.
         ca: Path to the certificate authority file, or None.
@@ -293,8 +293,14 @@ def dummy_cert(certdir, ca, commonname):
     reqpath = os.path.join(certdir, namehash + ".req")
 
     template = open(pkg_data.path("resources/cert.cnf")).read()
+
+    ss = []
+    for i, v in enumerate(sans):
+        ss.append("DNS.%s = %s"%(i, v))
+    ss = "\n".join(ss)
+
     f = open(confpath, "w")
-    f.write(template%(dict(commonname=commonname)))
+    f.write(template%(dict(commonname=commonname, sans=ss)))
     f.close()
 
     if ca:
@@ -483,3 +489,62 @@ def parse_size(s):
         return int(s) * mult
     except ValueError:
         raise ValueError("Invalid size specification: %s"%s)
+
+
+def get_remote_cn(host, port):
+    addr = socket.gethostbyname(host)
+    s = ssl.get_server_certificate((addr, port))
+    f = tempfile.NamedTemporaryFile()
+    f.write(s)
+    f.flush()
+    p = subprocess.Popen(
+        [
+            "openssl",
+            "x509",
+            "-in", f.name,
+            "-text",
+            "-noout"
+        ],
+        stdout = subprocess.PIPE
+    )
+    out, _ = p.communicate()
+    return parse_text_cert(out)
+
+
+CNRE = re.compile(
+    r"""
+        Subject:.*CN=(\S*)
+    """,
+    re.VERBOSE|re.MULTILINE
+)
+SANRE = re.compile(
+    r"""
+        X509v3\ Subject\ Alternative\ Name:\s*
+        (.*)$
+    """,
+    re.VERBOSE|re.MULTILINE
+)
+
+
+def parse_text_cert(txt):
+    """
+        Returns a (common name, [subject alternative names]) tuple.
+    """
+    r = re.search(CNRE, txt)
+    if r:
+        cn = r.group(1)
+    else:
+        return None
+
+    r = re.search(SANRE, txt)
+    san = []
+    if r:
+        for i in r.group(1).split(","):
+            i = i.strip()
+            k, v = i.split(":")
+            if k == "DNS":
+                san.append(v)
+    else:
+        san = []
+    return (cn, san)
+
diff --git a/test/data/text_cert b/test/data/text_cert
new file mode 100644
index 00000000..f52ad8ae
--- /dev/null
+++ b/test/data/text_cert
@@ -0,0 +1,48 @@
+Certificate:
+    Data:
+        Version: 3 (0x2)
+        Serial Number:
+            47:4f:4f:50:01:6d
+        Signature Algorithm: sha1WithRSAEncryption
+        Issuer: C=US, O=Google Inc, CN=Google Internet Authority
+        Validity
+            Not Before: Jan 17 12:55:04 2012 GMT
+            Not After : Jan 17 12:55:04 2013 GMT
+        Subject: C=US, ST=California, O=Google Inc, CN=google.com
+        Subject Public Key Info:
+            Public Key Algorithm: rsaEncryption
+            RSA Public Key: (1024 bit)
+                Modulus (1024 bit):
+                    00:ba:1f:73:14:76:7e:e7:79:73:21:48:7a:57:7d:
+                    a6:3d:af:18:1e:46:1f:4c:bd:b6:60:18:6b:9b:77:
+                    df:16:c9:82:e2:8a:5f:f7:a5:de:d5:c0:28:c3:62:
+                    4f:54:99:ce:ef:04:2c:9d:44:cb:41:9c:04:08:07:
+                    03:d8:d5:3d:07:74:2c:67:5b:12:d3:48:c6:36:e5:
+                    02:c5:80:0f:f1:59:95:2b:7c:5f:ce:e1:ec:2e:70:
+                    d8:94:9e:7c:e6:a7:8b:4a:a0:42:a5:3a:b1:37:6c:
+                    68:25:06:67:e1:2e:ec:3d:25:05:f4:f9:fd:59:e5:
+                    da:f5:7e:cc:8d:d8:98:39:69
+                Exponent: 65537 (0x10001)
+        X509v3 extensions:
+            X509v3 Subject Key Identifier: 
+                2F:7F:89:78:2F:E8:2F:D8:D9:85:3A:77:17:9E:74:2D:AB:AD:CD:5F
+            X509v3 Authority Key Identifier: 
+                keyid:BF:C0:30:EB:F5:43:11:3E:67:BA:9E:91:FB:FC:6A:DA:E3:6B:12:24
+
+            X509v3 CRL Distribution Points: 
+                URI:http://www.gstatic.com/GoogleInternetAuthority/GoogleInternetAuthority.crl
+
+            Authority Information Access: 
+                CA Issuers - URI:http://www.gstatic.com/GoogleInternetAuthority/GoogleInternetAuthority.crt
+
+            X509v3 Subject Alternative Name: 
+                DNS:google.com, DNS:*.google.com, DNS:*.google.ac, DNS:*.google.ad, DNS:*.google.ae, DNS:*.google.af, DNS:*.google.ag, DNS:*.google.am, DNS:*.google.as, DNS:*.google.at, DNS:*.google.az, DNS:*.google.ba, DNS:*.google.be, DNS:*.google.bf, DNS:*.google.bg, DNS:*.google.bi, DNS:*.google.bj, DNS:*.google.bs, DNS:*.google.by, DNS:*.google.ca, DNS:*.google.cat, DNS:*.google.cc, DNS:*.google.cd, DNS:*.google.cf, DNS:*.google.cg, DNS:*.google.ch, DNS:*.google.ci, DNS:*.google.cl, DNS:*.google.cm, DNS:*.google.cn, DNS:*.google.co.ao, DNS:*.google.co.bw, DNS:*.google.co.ck, DNS:*.google.co.cr, DNS:*.google.co.hu, DNS:*.google.co.id, DNS:*.google.co.il, DNS:*.google.co.im, DNS:*.google.co.in, DNS:*.google.co.je, DNS:*.google.co.jp, DNS:*.google.co.ke, DNS:*.google.co.kr, DNS:*.google.co.ls, DNS:*.google.co.ma, DNS:*.google.co.mz, DNS:*.google.co.nz, DNS:*.google.co.th, DNS:*.google.co.tz, DNS:*.google.co.ug, DNS:*.google.co.uk, DNS:*.google.co.uz, DNS:*.google.co.ve, DNS:*.google.co.vi, DNS:*.google.co.za, DNS:*.google.co.zm, DNS:*.google.co.zw, DNS:*.google.com.af, DNS:*.google.com.ag, DNS:*.google.com.ai, DNS:*.google.com.ar, DNS:*.google.com.au, DNS:*.google.com.bd, DNS:*.google.com.bh, DNS:*.google.com.bn, DNS:*.google.com.bo, DNS:*.google.com.br, DNS:*.google.com.by, DNS:*.google.com.bz, DNS:*.google.com.cn, DNS:*.google.com.co, DNS:*.google.com.cu, DNS:*.google.com.cy, DNS:*.google.com.do, DNS:*.google.com.ec, DNS:*.google.com.eg, DNS:*.google.com.et, DNS:*.google.com.fj, DNS:*.google.com.ge, DNS:*.google.com.gh, DNS:*.google.com.gi, DNS:*.google.com.gr, DNS:*.google.com.gt, DNS:*.google.com.hk, DNS:*.google.com.iq, DNS:*.google.com.jm, DNS:*.google.com.jo, DNS:*.google.com.kh, DNS:*.google.com.kw, DNS:*.google.com.lb, DNS:*.google.com.ly, DNS:*.google.com.mt, DNS:*.google.com.mx, DNS:*.google.com.my, DNS:*.google.com.na, DNS:*.google.com.nf, DNS:*.google.com.ng, DNS:*.google.com.ni, DNS:*.google.com.np, DNS:*.google.com.nr, DNS:*.google.com.om, DNS:*.google.com.pa, DNS:*.google.com.pe, DNS:*.google.com.ph, DNS:*.google.com.pk, DNS:*.google.com.pl, DNS:*.google.com.pr, DNS:*.google.com.py, DNS:*.google.com.qa, DNS:*.google.com.ru, DNS:*.google.com.sa, DNS:*.google.com.sb, DNS:*.google.com.sg, DNS:*.google.com.sl, DNS:*.google.com.sv, DNS:*.google.com.tj, DNS:*.google.com.tn, DNS:*.google.com.tr, DNS:*.google.com.tw, DNS:*.google.com.ua, DNS:*.google.com.uy, DNS:*.google.com.vc, DNS:*.google.com.ve, DNS:*.google.com.vn, DNS:*.google.cv, DNS:*.google.cz, DNS:*.google.de, DNS:*.google.dj, DNS:*.google.dk, DNS:*.google.dm, DNS:*.google.dz, DNS:*.google.ee, DNS:*.google.es, DNS:*.google.fi, DNS:*.google.fm, DNS:*.google.fr, DNS:*.google.ga, DNS:*.google.ge, DNS:*.google.gg, DNS:*.google.gl, DNS:*.google.gm, DNS:*.google.gp, DNS:*.google.gr, DNS:*.google.gy, DNS:*.google.hk, DNS:*.google.hn, DNS:*.google.hr, DNS:*.google.ht, DNS:*.google.hu, DNS:*.google.ie, DNS:*.google.im, DNS:*.google.info, DNS:*.google.iq, DNS:*.google.is, DNS:*.google.it, DNS:*.google.it.ao, DNS:*.google.je, DNS:*.google.jo, DNS:*.google.jobs, DNS:*.google.jp, DNS:*.google.kg, DNS:*.google.ki, DNS:*.google.kz, DNS:*.google.la, DNS:*.google.li, DNS:*.google.lk, DNS:*.google.lt, DNS:*.google.lu, DNS:*.google.lv, DNS:*.google.md, DNS:*.google.me, DNS:*.google.mg, DNS:*.google.mk, DNS:*.google.ml, DNS:*.google.mn, DNS:*.google.ms, DNS:*.google.mu, DNS:*.google.mv, DNS:*.google.mw, DNS:*.google.ne, DNS:*.google.ne.jp, DNS:*.google.net, DNS:*.google.nl, DNS:*.google.no, DNS:*.google.nr, DNS:*.google.nu, DNS:*.google.off.ai, DNS:*.google.pk, DNS:*.google.pl, DNS:*.google.pn, DNS:*.google.ps, DNS:*.google.pt, DNS:*.google.ro, DNS:*.google.rs, DNS:*.google.ru, DNS:*.google.rw, DNS:*.google.sc, DNS:*.google.se, DNS:*.google.sh, DNS:*.google.si, DNS:*.google.sk, DNS:*.google.sm, DNS:*.google.sn, DNS:*.google.so, DNS:*.google.st, DNS:*.google.td, DNS:*.google.tg, DNS:*.google.tk, DNS:*.google.tl, DNS:*.google.tm, DNS:*.google.tn, DNS:*.google.to, DNS:*.google.tp, DNS:*.google.tt, DNS:*.google.us, DNS:*.google.uz, DNS:*.google.vg, DNS:*.google.vu, DNS:*.google.ws, DNS:google.ac, DNS:google.ad, DNS:google.ae, DNS:google.af, DNS:google.ag, DNS:google.am, DNS:google.as, DNS:google.at, DNS:google.az, DNS:google.ba, DNS:google.be, DNS:google.bf, DNS:google.bg, DNS:google.bi, DNS:google.bj, DNS:google.bs, DNS:google.by, DNS:google.ca, DNS:google.cat, DNS:google.cc, DNS:google.cd, DNS:google.cf, DNS:google.cg, DNS:google.ch, DNS:google.ci, DNS:google.cl, DNS:google.cm, DNS:google.cn, DNS:google.co.ao, DNS:google.co.bw, DNS:google.co.ck, DNS:google.co.cr, DNS:google.co.hu, DNS:google.co.id, DNS:google.co.il, DNS:google.co.im, DNS:google.co.in, DNS:google.co.je, DNS:google.co.jp, DNS:google.co.ke, DNS:google.co.kr, DNS:google.co.ls, DNS:google.co.ma, DNS:google.co.mz, DNS:google.co.nz, DNS:google.co.th, DNS:google.co.tz, DNS:google.co.ug, DNS:google.co.uk, DNS:google.co.uz, DNS:google.co.ve, DNS:google.co.vi, DNS:google.co.za, DNS:google.co.zm, DNS:google.co.zw, DNS:google.com.af, DNS:google.com.ag, DNS:google.com.ai, DNS:google.com.ar, DNS:google.com.au, DNS:google.com.bd, DNS:google.com.bh, DNS:google.com.bn, DNS:google.com.bo, DNS:google.com.br, DNS:google.com.by, DNS:google.com.bz, DNS:google.com.cn, DNS:google.com.co, DNS:google.com.cu, DNS:google.com.cy, DNS:google.com.do, DNS:google.com.ec, DNS:google.com.eg, DNS:google.com.et, DNS:google.com.fj, DNS:google.com.ge, DNS:google.com.gh, DNS:google.com.gi, DNS:google.com.gr, DNS:google.com.gt, DNS:google.com.hk, DNS:google.com.iq, DNS:google.com.jm, DNS:google.com.jo, DNS:google.com.kh, DNS:google.com.kw, DNS:google.com.lb, DNS:google.com.ly, DNS:google.com.mt, DNS:google.com.mx, DNS:google.com.my, DNS:google.com.na, DNS:google.com.nf, DNS:google.com.ng, DNS:google.com.ni, DNS:google.com.np, DNS:google.com.nr, DNS:google.com.om, DNS:google.com.pa, DNS:google.com.pe, DNS:google.com.ph, DNS:google.com.pk, DNS:google.com.pl, DNS:google.com.pr, DNS:google.com.py, DNS:google.com.qa, DNS:google.com.ru, DNS:google.com.sa, DNS:google.com.sb, DNS:google.com.sg, DNS:google.com.sl, DNS:google.com.sv, DNS:google.com.tj, DNS:google.com.tn, DNS:google.com.tr, DNS:google.com.tw, DNS:google.com.ua, DNS:google.com.uy, DNS:google.com.vc, DNS:google.com.ve, DNS:google.com.vn, DNS:google.cv, DNS:google.cz, DNS:google.de, DNS:google.dj, DNS:google.dk, DNS:google.dm, DNS:google.dz, DNS:google.ee, DNS:google.es, DNS:google.fi, DNS:google.fm, DNS:google.fr, DNS:google.ga, DNS:google.ge, DNS:google.gg, DNS:google.gl, DNS:google.gm, DNS:google.gp, DNS:google.gr, DNS:google.gy, DNS:google.hk, DNS:google.hn, DNS:google.hr, DNS:google.ht, DNS:google.hu, DNS:google.ie, DNS:google.im, DNS:google.info, DNS:google.iq, DNS:google.is, DNS:google.it, DNS:google.it.ao, DNS:google.je, DNS:google.jo, DNS:google.jobs, DNS:google.jp, DNS:google.kg, DNS:google.ki, DNS:google.kz, DNS:google.la, DNS:google.li, DNS:google.lk, DNS:google.lt, DNS:google.lu, DNS:google.lv, DNS:google.md, DNS:google.me, DNS:google.mg, DNS:google.mk, DNS:google.ml, DNS:google.mn, DNS:google.ms, DNS:google.mu, DNS:google.mv, DNS:google.mw, DNS:google.ne, DNS:google.ne.jp, DNS:google.net, DNS:google.nl, DNS:google.no, DNS:google.nr, DNS:google.nu, DNS:google.off.ai, DNS:google.pk, DNS:google.pl, DNS:google.pn, DNS:google.ps, DNS:google.pt, DNS:google.ro, DNS:google.rs, DNS:google.ru, DNS:google.rw, DNS:google.sc, DNS:google.se, DNS:google.sh, DNS:google.si, DNS:google.sk, DNS:google.sm, DNS:google.sn, DNS:google.so, DNS:google.st, DNS:google.td, DNS:google.tg, DNS:google.tk, DNS:google.tl, DNS:google.tm, DNS:google.tn, DNS:google.to, DNS:google.tp, DNS:google.tt, DNS:google.us, DNS:google.uz, DNS:google.vg, DNS:google.vu, DNS:google.ws
+    Signature Algorithm: sha1WithRSAEncryption
+        99:99:f5:1c:aa:a5:4c:eb:3f:45:1c:26:71:d7:a0:bb:bf:02:
+        32:04:b0:8b:d9:63:d8:6a:ff:d6:cb:af:2f:08:9d:f9:af:11:
+        16:07:af:88:4d:7a:d9:0e:62:55:b4:15:e0:d9:b4:2a:56:c5:
+        d3:d4:80:98:e6:5a:02:80:2b:a8:f4:82:83:85:6a:f4:d5:1a:
+        9b:64:2f:c8:21:6b:8a:75:41:4b:1d:2e:47:77:eb:f7:2f:05:
+        77:81:61:29:5e:ef:ce:ef:7a:8e:3a:37:59:38:57:a5:2f:da:
+        06:83:56:4a:de:ef:ce:82:36:c1:ef:43:a3:18:67:90:a5:43:
+        e9:12
diff --git a/test/test_utils.py b/test/test_utils.py
index c2f81a7b..06366773 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -149,14 +149,16 @@ class udummy_cert(libpry.AutoTree):
         p = utils.dummy_cert(
             os.path.join(d, "foo"),
             cacert,
-            "foo.com"
+            "foo.com",
+            ["one.com", "two.com", "*.three.com"]
         )
         assert os.path.exists(p)
         # Short-circuit
         assert utils.dummy_cert(
             os.path.join(d, "foo"),
             cacert,
-            "foo.com"
+            "foo.com",
+            []
         )
 
     def test_no_ca(self):
@@ -164,7 +166,8 @@ class udummy_cert(libpry.AutoTree):
         p = utils.dummy_cert(
             d,
             None,
-            "foo.com"
+            "foo.com",
+            []
         )
         assert os.path.exists(p)
 
@@ -255,7 +258,16 @@ class u_parse_size(libpry.AutoTree):
         libpry.raises(ValueError, utils.parse_size, "ak")
 
 
+class uparse_text_cert(libpry.AutoTree):
+    def test_simple(self):
+        c = file("data/text_cert", "r").read()
+        cn, san = utils.parse_text_cert(c)
+        assert cn == "google.com"
+        assert len(san) == 436
+
+
 tests = [
+    uparse_text_cert(),
     uformat_timestamp(),
     uisBin(),
     uisXML(),
diff --git a/test/tools/getcn b/test/tools/getcn
new file mode 100755
index 00000000..d17ec0f6
--- /dev/null
+++ b/test/tools/getcn
@@ -0,0 +1,13 @@
+#!/usr/bin/env python
+
+import sys
+sys.path.insert(0, "../..")
+from libmproxy import utils
+
+cn, san = utils.get_remote_cn(sys.argv[1], 443)
+print cn
+if san:
+    for i in san:
+        print "\t", i
+
+