diff options
Diffstat (limited to 'libmproxy')
| -rw-r--r-- | libmproxy/certutils.py | 234 | ||||
| -rw-r--r-- | libmproxy/cmdline.py | 24 | ||||
| -rw-r--r-- | libmproxy/console/__init__.py | 9 | ||||
| -rw-r--r-- | libmproxy/dump.py | 4 | ||||
| -rw-r--r-- | libmproxy/flow.py | 15 | ||||
| -rw-r--r-- | libmproxy/proxy.py | 19 | ||||
| -rw-r--r-- | libmproxy/resources/cert.cnf | 3 | ||||
| -rw-r--r-- | libmproxy/utils.py | 167 |
8 files changed, 290 insertions, 185 deletions
diff --git a/libmproxy/certutils.py b/libmproxy/certutils.py new file mode 100644 index 00000000..c1e5d93e --- /dev/null +++ b/libmproxy/certutils.py @@ -0,0 +1,234 @@ +import subprocess, os, ssl, hashlib, socket +from pyasn1.type import univ, constraint, char, namedtype, tag +from pyasn1.codec.der.decoder import decode +import OpenSSL +import utils + +CERT_SLEEP_TIME = 1 +CERT_EXPIRY = str(365 * 3) + + +def dummy_ca(path): + """ + Creates a dummy CA, and writes it to path. + + This function also creates the necessary directories if they don't exist. + + Returns True if operation succeeded, False if not. + """ + dirname = os.path.dirname(path) + if not os.path.exists(dirname): + os.makedirs(dirname) + + if path.endswith(".pem"): + basename, _ = os.path.splitext(path) + else: + basename = path + + cmd = [ + "openssl", + "req", + "-new", + "-x509", + "-config", utils.pkg_data.path("resources/ca.cnf"), + "-nodes", + "-days", CERT_EXPIRY, + "-out", path, + "-newkey", "rsa:1024", + "-keyout", path, + ] + ret = subprocess.call( + cmd, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE + ) + # begin nocover + if ret: + return False + # end nocover + + cmd = [ + "openssl", + "pkcs12", + "-export", + "-password", "pass:", + "-nokeys", + "-in", path, + "-out", os.path.join(dirname, basename + "-cert.p12") + ] + ret = subprocess.call( + cmd, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE + ) + # begin nocover + if ret: + return False + # end nocover + cmd = [ + "openssl", + "x509", + "-in", path, + "-out", os.path.join(dirname, basename + "-cert.pem") + ] + ret = subprocess.call( + cmd, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE + ) + # begin nocover + if ret: + return False + # end nocover + + return True + + +def dummy_cert(certdir, ca, commonname, sans): + """ + certdir: Certificate directory. + ca: Path to the certificate authority file, or None. + commonname: Common name for the generated certificate. + + Returns cert path if operation succeeded, None if not. + """ + namehash = hashlib.sha256(commonname).hexdigest() + certpath = os.path.join(certdir, namehash + ".pem") + if os.path.exists(certpath): + return certpath + + confpath = os.path.join(certdir, namehash + ".cnf") + reqpath = os.path.join(certdir, namehash + ".req") + + template = open(utils.pkg_data.path("resources/cert.cnf")).read() + + ss = [] + for i, v in enumerate(sans): + ss.append("DNS.%s = %s"%(i+1, v)) + ss = "\n".join(ss) + + f = open(confpath, "w") + f.write( + template%( + dict( + commonname=commonname, + sans=ss, + altnames="subjectAltName = @alt_names" if ss else "" + ) + ) + ) + f.close() + + if ca: + # Create a dummy signed certificate. Uses same key as the signing CA + cmd = [ + "openssl", + "req", + "-new", + "-config", confpath, + "-out", reqpath, + "-key", ca, + ] + ret = subprocess.call( + cmd, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE + ) + if ret: return None + cmd = [ + "openssl", + "x509", + "-req", + "-in", reqpath, + "-days", CERT_EXPIRY, + "-out", certpath, + "-CA", ca, + "-CAcreateserial", + "-extfile", confpath, + "-extensions", "v3_cert_req", + ] + ret = subprocess.call( + cmd, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE + ) + if ret: return None + else: + # Create a new selfsigned certificate + key + cmd = [ + "openssl", + "req", + "-new", + "-x509", + "-config", confpath, + "-nodes", + "-days", CERT_EXPIRY, + "-out", certpath, + "-newkey", "rsa:1024", + "-keyout", certpath, + ] + ret = subprocess.call( + cmd, + stderr=subprocess.PIPE, + stdout=subprocess.PIPE, + stdin=subprocess.PIPE + ) + if ret: return None + return certpath + + +class GeneralName(univ.Choice): + # We are only interested in dNSNames. We use a default handler to ignore + # other types. + componentType = namedtype.NamedTypes( + namedtype.NamedType('dNSName', char.IA5String().subtype( + implicitTag=tag.Tag(tag.tagClassContext, tag.tagFormatSimple, 2) + ) + ), + ) + + +class GeneralNames(univ.SequenceOf): + componentType = GeneralName() + sizeSpec = univ.SequenceOf.sizeSpec + constraint.ValueSizeConstraint(1, 1024) + + +class SSLCert: + def __init__(self, pemtxt): + """ + Returns a (common name, [subject alternative names]) tuple. + """ + self.cert = OpenSSL.crypto.load_certificate(OpenSSL.crypto.FILETYPE_PEM, pemtxt) + + @property + def cn(self): + cn = None + for i in self.cert.get_subject().get_components(): + if i[0] == "CN": + cn = i[1] + return cn + + @property + def altnames(self): + altnames = [] + for i in range(self.cert.get_extension_count()): + ext = self.cert.get_extension(i) + if ext.get_short_name() == "subjectAltName": + dec = decode(ext.get_data(), asn1Spec=GeneralNames()) + for i in dec[0]: + altnames.append(i[0]) + return altnames + + + +def get_remote_cert(host, port): + addr = socket.gethostbyname(host) + s = ssl.get_server_certificate((addr, port)) + return SSLCert(s) + + diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index ee4f3b08..27819294 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -45,6 +45,7 @@ def get_common_options(options): stickyauth = stickyauth, wfile = options.wfile, verbosity = options.verbose, + nopop = options.nopop, ) @@ -141,6 +142,17 @@ def common_options(parser): help="Byte size limit of HTTP request and response bodies."\ " Understands k/m/g suffixes, i.e. 3m for 3 megabytes." ) + parser.add_option( + "--cert-wait-time", type="float", + action="store", dest="cert_wait_time", default=0, + help="Wait for specified number of seconds after a new cert is generated. This can smooth over small discrepancies between the client and server times." + ) + parser.add_option( + "--upstream-cert", default=False, + action="store_true", dest="upstream_cert", + help="Connect to upstream server to look up certificate details." + ) + group = optparse.OptionGroup(parser, "Client Replay") group.add_option( "-c", @@ -149,12 +161,6 @@ def common_options(parser): ) parser.add_option_group(group) - parser.add_option( - "--cert-wait-time", type="float", - action="store", dest="cert_wait_time", default=0, - help="Wait for specified number of seconds after a new cert is generated. This can smooth over small discrepancies between the client and server times." - ) - group = optparse.OptionGroup(parser, "Server Replay") group.add_option( "-S", @@ -178,6 +184,12 @@ def common_options(parser): help= "Disable response refresh, " "which updates times in cookies and headers for replayed responses." ) + group.add_option( + "--no-pop", + action="store_true", dest="nopop", default=False, + help="Disable response pop from response flow. " + "This makes it possible to replay same response multiple times." + ) parser.add_option_group(group) proxy.certificate_option_group(parser) diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 7d936892..5d9c5da2 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -126,7 +126,10 @@ class StatusBar(common.WWrap): if self.master.server_playback: r.append("[") r.append(("heading_key", "splayback")) - r.append(":%s to go]"%self.master.server_playback.count()) + if self.master.nopop: + r.append(":%s in file]"%self.master.server_playback.count()) + else: + r.append(":%s to go]"%self.master.server_playback.count()) if self.master.state.intercept_txt: r.append("[") r.append(("heading_key", "i")) @@ -297,6 +300,7 @@ class Options(object): "stickyauth", "verbosity", "wfile", + "nopop", ] def __init__(self, **kwargs): for k, v in kwargs.items(): @@ -350,6 +354,7 @@ class ConsoleMaster(flow.FlowMaster): self.anticomp = options.anticomp self.killextra = options.kill self.rheaders = options.rheaders + self.nopop = options.nopop self.eventlog = options.eventlog self.eventlist = urwid.SimpleListWalker([]) @@ -422,7 +427,7 @@ class ConsoleMaster(flow.FlowMaster): self.start_server_playback( ret, self.killextra, self.rheaders, - False + False, self.nopop ) def spawn_editor(self, data): diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 4520ad82..81d7dc4d 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -28,6 +28,7 @@ class Options(object): "keepserving", "kill", "no_server", + "nopop", "refresh_server_playback", "rfile", "rheaders", @@ -97,7 +98,8 @@ class DumpMaster(flow.FlowMaster): self.start_server_playback( self._readflow(options.server_replay), options.kill, options.rheaders, - not options.keepserving + not options.keepserving, + options.nopop ) if options.client_replay: diff --git a/libmproxy/flow.py b/libmproxy/flow.py index c4bf35a5..e7af996c 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -768,12 +768,12 @@ class ClientPlaybackState: class ServerPlaybackState: - def __init__(self, headers, flows, exit): + def __init__(self, headers, flows, exit, nopop): """ headers: Case-insensitive list of request headers that should be included in request-response matching. """ - self.headers, self.exit = headers, exit + self.headers, self.exit, self.nopop = headers, exit, nopop self.fmap = {} for i in flows: if i.response: @@ -815,7 +815,12 @@ class ServerPlaybackState: l = self.fmap.get(self._hash(request)) if not l: return None - return l.pop(0) + + if self.nopop: + return l[0] + else: + return l.pop(0) + class StickyCookieState: @@ -1251,12 +1256,12 @@ class FlowMaster(controller.Master): def stop_client_playback(self): self.client_playback = None - def start_server_playback(self, flows, kill, headers, exit): + def start_server_playback(self, flows, kill, headers, exit, nopop): """ flows: List of flows. kill: Boolean, should we kill requests not part of the replay? """ - self.server_playback = ServerPlaybackState(headers, flows, exit) + self.server_playback = ServerPlaybackState(headers, flows, exit, nopop) self.kill_nonreplay = kill def stop_server_playback(self): diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 72a7a5a3..33e50890 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -21,7 +21,7 @@ import sys, os, string, socket, time import shutil, tempfile, threading import optparse, SocketServer, ssl -import utils, flow +import utils, flow, certutils NAME = "mitmproxy" @@ -35,12 +35,13 @@ class ProxyError(Exception): class ProxyConfig: - def __init__(self, certfile = None, ciphers = None, cacert = None, cert_wait_time=0, body_size_limit = None, reverse_proxy=None): + def __init__(self, certfile = None, ciphers = None, cacert = None, cert_wait_time=0, upstream_cert=False, body_size_limit = None, reverse_proxy=None): self.certfile = certfile self.ciphers = ciphers self.cacert = cacert self.certdir = None self.cert_wait_time = cert_wait_time + self.upstream_cert = upstream_cert self.body_size_limit = body_size_limit self.reverse_proxy = reverse_proxy @@ -347,11 +348,16 @@ class ProxyHandler(SocketServer.StreamRequestHandler): if server: server.terminate() - def find_cert(self, host): + def find_cert(self, host, port): if self.config.certfile: return self.config.certfile else: - ret = utils.dummy_cert(self.config.certdir, self.config.cacert, host) + sans = [] + if self.config.upstream_cert: + cert = certutils.get_remote_cert(host, port) + sans = cert.altnames + host = cert.cn + ret = certutils.dummy_cert(self.config.certdir, self.config.cacert, host, sans) time.sleep(self.config.cert_wait_time) if not ret: raise ProxyError(502, "mitmproxy: Unable to generate dummy cert.") @@ -378,7 +384,7 @@ class ProxyHandler(SocketServer.StreamRequestHandler): ) self.wfile.flush() kwargs = dict( - certfile = self.find_cert(host), + certfile = self.find_cert(host, port), keyfile = self.config.certfile or self.config.cacert, server_side = True, ssl_version = ssl.PROTOCOL_SSLv23, @@ -524,7 +530,7 @@ def process_proxy_options(parser, options): cacert = os.path.join(options.confdir, "mitmproxy-ca.pem") cacert = os.path.expanduser(cacert) if not os.path.exists(cacert): - utils.dummy_ca(cacert) + certutils.dummy_ca(cacert) if getattr(options, "cache", None) is not None: options.cache = os.path.expanduser(options.cache) body_size_limit = utils.parse_size(options.body_size_limit) @@ -542,5 +548,6 @@ def process_proxy_options(parser, options): ciphers = options.ciphers, cert_wait_time = options.cert_wait_time, body_size_limit = body_size_limit, + upstream_cert = options.upstream_cert, reverse_proxy = rp ) diff --git a/libmproxy/resources/cert.cnf b/libmproxy/resources/cert.cnf index 5f80c2d6..4d95f646 100644 --- a/libmproxy/resources/cert.cnf +++ b/libmproxy/resources/cert.cnf @@ -27,4 +27,7 @@ nsCertType = server basicConstraints = CA:false keyUsage = nonRepudiation, digitalSignature, keyEncipherment nsCertType = server +%(altnames)s +[ alt_names ] +%(sans)s diff --git a/libmproxy/utils.py b/libmproxy/utils.py index 16540434..f7cf5f32 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -12,13 +12,10 @@ # # You should have received a copy of the GNU General Public License # along with this program. If not, see <http://www.gnu.org/licenses/>. -import re, os, subprocess, datetime, urlparse, string, urllib -import time, functools, cgi, textwrap, hashlib +import re, os, datetime, urlparse, string, urllib +import time, functools, cgi, textwrap import json -CERT_SLEEP_TIME = 1 -CERT_EXPIRY = str(365 * 3) - def timestamp(): """ Returns a serializable UTC timestamp. @@ -197,166 +194,6 @@ class Data: pkg_data = Data(__name__) -def dummy_ca(path): - """ - Creates a dummy CA, and writes it to path. - - This function also creates the necessary directories if they don't exist. - - Returns True if operation succeeded, False if not. - """ - dirname = os.path.dirname(path) - if not os.path.exists(dirname): - os.makedirs(dirname) - - if path.endswith(".pem"): - basename, _ = os.path.splitext(path) - else: - basename = path - - cmd = [ - "openssl", - "req", - "-new", - "-x509", - "-config", pkg_data.path("resources/ca.cnf"), - "-nodes", - "-days", CERT_EXPIRY, - "-out", path, - "-newkey", "rsa:1024", - "-keyout", path, - ] - ret = subprocess.call( - cmd, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE - ) - # begin nocover - if ret: - return False - # end nocover - - cmd = [ - "openssl", - "pkcs12", - "-export", - "-password", "pass:", - "-nokeys", - "-in", path, - "-out", os.path.join(dirname, basename + "-cert.p12") - ] - ret = subprocess.call( - cmd, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE - ) - # begin nocover - if ret: - return False - # end nocover - cmd = [ - "openssl", - "x509", - "-in", path, - "-out", os.path.join(dirname, basename + "-cert.pem") - ] - ret = subprocess.call( - cmd, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE - ) - # begin nocover - if ret: - return False - # end nocover - - return True - - -def dummy_cert(certdir, ca, commonname): - """ - certdir: Certificate directory. - ca: Path to the certificate authority file, or None. - commonname: Common name for the generated certificate. - - Returns cert path if operation succeeded, None if not. - """ - namehash = hashlib.sha256(commonname).hexdigest() - certpath = os.path.join(certdir, namehash + ".pem") - if os.path.exists(certpath): - return certpath - - confpath = os.path.join(certdir, namehash + ".cnf") - reqpath = os.path.join(certdir, namehash + ".req") - - template = open(pkg_data.path("resources/cert.cnf")).read() - f = open(confpath, "w") - f.write(template%(dict(commonname=commonname))) - f.close() - - if ca: - # Create a dummy signed certificate. Uses same key as the signing CA - cmd = [ - "openssl", - "req", - "-new", - "-config", confpath, - "-out", reqpath, - "-key", ca, - ] - ret = subprocess.call( - cmd, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE - ) - if ret: return None - cmd = [ - "openssl", - "x509", - "-req", - "-in", reqpath, - "-days", CERT_EXPIRY, - "-out", certpath, - "-CA", ca, - "-CAcreateserial", - "-extfile", confpath, - "-extensions", "v3_cert", - ] - ret = subprocess.call( - cmd, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE - ) - if ret: return None - else: - # Create a new selfsigned certificate + key - cmd = [ - "openssl", - "req", - "-new", - "-x509", - "-config", confpath, - "-nodes", - "-days", CERT_EXPIRY, - "-out", certpath, - "-newkey", "rsa:1024", - "-keyout", certpath, - ] - ret = subprocess.call( - cmd, - stderr=subprocess.PIPE, - stdout=subprocess.PIPE, - stdin=subprocess.PIPE - ) - if ret: return None - return certpath - - class LRUCache: """ A decorator that implements a self-expiring LRU cache for class |