From ba674ad5514c5f30315fc688a07fdac634d94dfc Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 1 Mar 2013 09:05:39 +1300 Subject: New SNI handling mechanism. --- libmproxy/proxy.py | 57 +++++++++++++++++++++++++++++++++++++----------------- 1 file changed, 39 insertions(+), 18 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 7c229064..c9ceb8de 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -80,8 +80,7 @@ class ServerConnection(tcp.TCPClient): def terminate(self): try: - if not self.wfile.closed: - self.wfile.flush() + self.wfile.flush() self.connection.close() except IOError: pass @@ -110,6 +109,27 @@ class RequestReplayThread(threading.Thread): self.channel.ask(err) +class HandleSNI: + def __init__(self, handler, client_conn, host, port, cert, key): + self.handler, self.client_conn, self.host, self.port = handler, client_conn, host, port + self.cert, self.key = cert, key + + def __call__(self, connection): + try: + sn = connection.get_servername() + if sn: + self.handler.get_server_connection(self.client_conn, "https", self.host, self.port, sn) + new_context = SSL.Context(SSL.TLSv1_METHOD) + new_context.use_privatekey_file(self.key) + new_context.use_certificate_file(self.cert) + connection.set_context(new_context) + self.handler.sni = sn.decode("utf8").encode("idna") + # An unhandled exception in this method will core dump PyOpenSSL, so + # make dang sure it doesn't happen. + except Exception, e: + pass + + class ProxyHandler(tcp.BaseHandler): def __init__(self, config, connection, client_address, server, channel, server_version): self.channel, self.server_version = channel, server_version @@ -266,18 +286,15 @@ class ProxyHandler(tcp.BaseHandler): l = Log(msg) self.channel.tell(l) - def find_cert(self, host, port, sni): + def find_cert(self, cc, host, port, sni): if self.config.certfile: return self.config.certfile else: sans = [] if not self.config.no_upstream_cert: - try: - cert = certutils.get_remote_cert(host, port, sni) - except tcp.NetLibError, v: - raise ProxyError(502, "Unable to get remote cert: %s"%str(v)) - sans = cert.altnames - host = cert.cn.decode("utf8").encode("idna") + conn = self.get_server_connection(cc, "https", host, port, sni) + sans = conn.cert.altnames + host = conn.cert.cn.decode("utf8").encode("idna") ret = self.config.certstore.get_cert(host, sans, self.config.cacert) if not ret: raise ProxyError(502, "mitmproxy: Unable to generate dummy cert.") @@ -292,11 +309,6 @@ class ProxyHandler(tcp.BaseHandler): line = fp.readline() return line - def handle_sni(self, conn): - sn = conn.get_servername() - if sn: - self.sni = sn.decode("utf8").encode("idna") - def read_request_transparent(self, client_conn): orig = self.config.transparent_proxy["resolver"].original_addr(self.connection) if not orig: @@ -304,9 +316,13 @@ class ProxyHandler(tcp.BaseHandler): host, port = orig if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]): scheme = "https" - certfile = self.find_cert(host, port, None) + dummycert = self.find_cert(client_conn, host, port, host) try: - self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert) + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) + self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) except tcp.NetLibError, v: raise ProxyError(400, str(v)) else: @@ -346,9 +362,14 @@ class ProxyHandler(tcp.BaseHandler): '\r\n' ) self.wfile.flush() - certfile = self.find_cert(host, port, None) + certfile = self.find_cert(client_conn, host, port, host) + + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) try: - self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert) + self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert, handle_sni=sni) except tcp.NetLibError, v: raise ProxyError(400, str(v)) self.proxy_connect_state = (host, port, httpversion) -- cgit v1.2.3 From 10db82e9a030235ab884e70d1809ad6d673c2d13 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 2 Mar 2013 14:52:05 +1300 Subject: Test SNI for ordinary proxy connections. --- libmproxy/proxy.py | 19 ++++++++++--------- 1 file changed, 10 insertions(+), 9 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index c9ceb8de..54cb6f8e 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -126,7 +126,7 @@ class HandleSNI: self.handler.sni = sn.decode("utf8").encode("idna") # An unhandled exception in this method will core dump PyOpenSSL, so # make dang sure it doesn't happen. - except Exception, e: + except Exception, e: # pragma: no cover pass @@ -141,6 +141,8 @@ class ProxyHandler(tcp.BaseHandler): def get_server_connection(self, cc, scheme, host, port, sni): sc = self.server_conn + if not sni: + sni = host if sc and (scheme, host, port, sni) != (sc.scheme, sc.host, sc.port, sc.sni): sc.terminate() self.server_conn = None @@ -214,7 +216,7 @@ class ProxyHandler(tcp.BaseHandler): # the case, we want to reconnect without sending an error # to the client. while 1: - sc = self.get_server_connection(cc, scheme, host, port, host) + sc = self.get_server_connection(cc, scheme, host, port, self.sni) sc.send(request) sc.rfile.reset_timestamps() try: @@ -362,14 +364,13 @@ class ProxyHandler(tcp.BaseHandler): '\r\n' ) self.wfile.flush() - certfile = self.find_cert(client_conn, host, port, host) - - sni = HandleSNI( - self, client_conn, host, port, - dummycert, self.config.certfile or self.config.cacert - ) + dummycert = self.find_cert(client_conn, host, port, host) try: - self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert, handle_sni=sni) + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) + self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) except tcp.NetLibError, v: raise ProxyError(400, str(v)) self.proxy_connect_state = (host, port, httpversion) -- cgit v1.2.3 From a95d78438c7197b0b6643a61899914083de70da9 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 2 Mar 2013 15:06:49 +1300 Subject: Test SNI for transparent mode. --- libmproxy/proxy.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 54cb6f8e..964c15a9 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -140,6 +140,13 @@ class ProxyHandler(tcp.BaseHandler): tcp.BaseHandler.__init__(self, connection, client_address, server) def get_server_connection(self, cc, scheme, host, port, sni): + """ + When SNI is in play, this means we have an SSL-encrypted + connection, which means that the entire handler is dedicated to a + single server connection - no multiplexing. If this assumption ever + breaks, we'll have to do something different with the SNI host + variable on the handler object. + """ sc = self.server_conn if not sni: sni = host @@ -329,7 +336,6 @@ class ProxyHandler(tcp.BaseHandler): raise ProxyError(400, str(v)) else: scheme = "http" - host = self.sni or host line = self.get_line(self.rfile) if line == "": return None -- cgit v1.2.3 From 415844511c19b17743b42a5833590d1d683427d2 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 2 Mar 2013 16:59:16 +1300 Subject: Test cert generation errors. --- libmproxy/proxy.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 964c15a9..a6a72d55 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -306,7 +306,7 @@ class ProxyHandler(tcp.BaseHandler): host = conn.cert.cn.decode("utf8").encode("idna") ret = self.config.certstore.get_cert(host, sans, self.config.cacert) if not ret: - raise ProxyError(502, "mitmproxy: Unable to generate dummy cert.") + raise ProxyError(502, "Unable to generate dummy cert.") return ret def get_line(self, fp): -- cgit v1.2.3 From c20d1d7d32ea2ab1d1c4dd9a34724a8732c23338 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 2 Mar 2013 22:42:36 +1300 Subject: Extend unit tests for proxy.py to some tricky cases. --- libmproxy/proxy.py | 46 ++++++++++++++++++++++------------------------ 1 file changed, 22 insertions(+), 24 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index a6a72d55..458ea2b5 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -326,11 +326,11 @@ class ProxyHandler(tcp.BaseHandler): if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]): scheme = "https" dummycert = self.find_cert(client_conn, host, port, host) + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) try: - sni = HandleSNI( - self, client_conn, host, port, - dummycert, self.config.certfile or self.config.cacert - ) self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) except tcp.NetLibError, v: raise ProxyError(400, str(v)) @@ -356,31 +356,29 @@ class ProxyHandler(tcp.BaseHandler): line = self.get_line(self.rfile) if line == "": return None - if http.parse_init_connect(line): - r = http.parse_init_connect(line) - if not r: - raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) - host, port, httpversion = r - - headers = self.read_headers(authenticate=True) - self.wfile.write( - 'HTTP/1.1 200 Connection established\r\n' + - ('Proxy-agent: %s\r\n'%self.server_version) + - '\r\n' - ) - self.wfile.flush() - dummycert = self.find_cert(client_conn, host, port, host) - try: + if not self.proxy_connect_state: + connparts = http.parse_init_connect(line) + if connparts: + host, port, httpversion = connparts + headers = self.read_headers(authenticate=True) + self.wfile.write( + 'HTTP/1.1 200 Connection established\r\n' + + ('Proxy-agent: %s\r\n'%self.server_version) + + '\r\n' + ) + self.wfile.flush() + dummycert = self.find_cert(client_conn, host, port, host) sni = HandleSNI( self, client_conn, host, port, dummycert, self.config.certfile or self.config.cacert ) - self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) - except tcp.NetLibError, v: - raise ProxyError(400, str(v)) - self.proxy_connect_state = (host, port, httpversion) - line = self.rfile.readline(line) + try: + self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) + except tcp.NetLibError, v: + raise ProxyError(400, str(v)) + self.proxy_connect_state = (host, port, httpversion) + line = self.rfile.readline(line) if self.proxy_connect_state: r = http.parse_init_http(line) -- cgit v1.2.3 From 5c6587d4a80cc45b23154237ca94858da60c7da5 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 3 Mar 2013 10:37:06 +1300 Subject: Move HTTP auth module to netlib. --- libmproxy/authentication.py | 122 ------------------------------------------ libmproxy/contrib/md5crypt.py | 94 -------------------------------- libmproxy/proxy.py | 13 +++-- 3 files changed, 6 insertions(+), 223 deletions(-) delete mode 100644 libmproxy/authentication.py delete mode 100644 libmproxy/contrib/md5crypt.py (limited to 'libmproxy') diff --git a/libmproxy/authentication.py b/libmproxy/authentication.py deleted file mode 100644 index 500ead6b..00000000 --- a/libmproxy/authentication.py +++ /dev/null @@ -1,122 +0,0 @@ -import binascii -import contrib.md5crypt as md5crypt - -class NullProxyAuth(): - """ - No proxy auth at all (returns empty challange headers) - """ - def __init__(self, password_manager): - self.password_manager = password_manager - self.username = "" - - def clean(self, headers): - """ - Clean up authentication headers, so they're not passed upstream. - """ - pass - - def authenticate(self, headers): - """ - Tests that the user is allowed to use the proxy - """ - return True - - def auth_challenge_headers(self): - """ - Returns a dictionary containing the headers require to challenge the user - """ - return {} - - -class BasicProxyAuth(NullProxyAuth): - CHALLENGE_HEADER = 'Proxy-Authenticate' - AUTH_HEADER = 'Proxy-Authorization' - def __init__(self, password_manager, realm): - NullProxyAuth.__init__(self, password_manager) - self.realm = realm - - def clean(self, headers): - del headers[self.AUTH_HEADER] - - def authenticate(self, headers): - auth_value = headers.get(self.AUTH_HEADER, []) - if not auth_value: - return False - try: - scheme, username, password = self.parse_auth_value(auth_value[0]) - except ValueError: - return False - if scheme.lower()!='basic': - return False - if not self.password_manager.test(username, password): - return False - self.username = username - return True - - def auth_challenge_headers(self): - return {self.CHALLENGE_HEADER:'Basic realm="%s"'%self.realm} - - def unparse_auth_value(self, scheme, username, password): - v = binascii.b2a_base64(username + ":" + password) - return scheme + " " + v - - def parse_auth_value(self, auth_value): - words = auth_value.split() - if len(words) != 2: - raise ValueError("Invalid basic auth credential.") - scheme = words[0] - try: - user = binascii.a2b_base64(words[1]) - except binascii.Error: - raise ValueError("Invalid basic auth credential: user:password pair not valid base64: %s"%words[1]) - parts = user.split(':') - if len(parts) != 2: - raise ValueError("Invalid basic auth credential: decoded user:password pair not valid: %s"%user) - return scheme, parts[0], parts[1] - - -class PasswordManager(): - def __init__(self): - pass - - def test(self, username, password_token): - return False - - -class PermissivePasswordManager(PasswordManager): - def __init__(self): - PasswordManager.__init__(self) - - def test(self, username, password_token): - if username: - return True - return False - - -class HtpasswdPasswordManager(PasswordManager): - """ - Read usernames and passwords from a file created by Apache htpasswd - """ - def __init__(self, filehandle): - PasswordManager.__init__(self) - entries = (line.strip().split(':') for line in filehandle) - valid_entries = (entry for entry in entries if len(entry)==2) - self.usernames = {username:token for username,token in valid_entries} - - def test(self, username, password_token): - if username not in self.usernames: - return False - full_token = self.usernames[username] - dummy, magic, salt, hashed_password = full_token.split('$') - expected = md5crypt.md5crypt(password_token, salt, '$'+magic+'$') - return expected==full_token - - -class SingleUserPasswordManager(PasswordManager): - def __init__(self, username, password): - PasswordManager.__init__(self) - self.username = username - self.password = password - - def test(self, username, password_token): - return self.username==username and self.password==password_token diff --git a/libmproxy/contrib/md5crypt.py b/libmproxy/contrib/md5crypt.py deleted file mode 100644 index d64ea8ac..00000000 --- a/libmproxy/contrib/md5crypt.py +++ /dev/null @@ -1,94 +0,0 @@ -# Based on FreeBSD src/lib/libcrypt/crypt.c 1.2 -# http://www.freebsd.org/cgi/cvsweb.cgi/~checkout~/src/lib/libcrypt/crypt.c?rev=1.2&content-type=text/plain - -# Original license: -# * "THE BEER-WARE LICENSE" (Revision 42): -# * wrote this file. As long as you retain this notice you -# * can do whatever you want with this stuff. If we meet some day, and you think -# * this stuff is worth it, you can buy me a beer in return. Poul-Henning Kamp - -# This port adds no further stipulations. I forfeit any copyright interest. - -import md5 - -def md5crypt(password, salt, magic='$1$'): - # /* The password first, since that is what is most unknown */ /* Then our magic string */ /* Then the raw salt */ - m = md5.new() - m.update(password + magic + salt) - - # /* Then just as many characters of the MD5(pw,salt,pw) */ - mixin = md5.md5(password + salt + password).digest() - for i in range(0, len(password)): - m.update(mixin[i % 16]) - - # /* Then something really weird... */ - # Also really broken, as far as I can tell. -m - i = len(password) - while i: - if i & 1: - m.update('\x00') - else: - m.update(password[0]) - i >>= 1 - - final = m.digest() - - # /* and now, just to make sure things don't run too fast */ - for i in range(1000): - m2 = md5.md5() - if i & 1: - m2.update(password) - else: - m2.update(final) - - if i % 3: - m2.update(salt) - - if i % 7: - m2.update(password) - - if i & 1: - m2.update(final) - else: - m2.update(password) - - final = m2.digest() - - # This is the bit that uses to64() in the original code. - - itoa64 = './0123456789ABCDEFGHIJKLMNOPQRSTUVWXYZabcdefghijklmnopqrstuvwxyz' - - rearranged = '' - for a, b, c in ((0, 6, 12), (1, 7, 13), (2, 8, 14), (3, 9, 15), (4, 10, 5)): - v = ord(final[a]) << 16 | ord(final[b]) << 8 | ord(final[c]) - for i in range(4): - rearranged += itoa64[v & 0x3f]; v >>= 6 - - v = ord(final[11]) - for i in range(2): - rearranged += itoa64[v & 0x3f]; v >>= 6 - - return magic + salt + '$' + rearranged - -if __name__ == '__main__': - - def test(clear_password, the_hash): - magic, salt = the_hash[1:].split('$')[:2] - magic = '$' + magic + '$' - return md5crypt(clear_password, salt, magic) == the_hash - - test_cases = ( - (' ', '$1$yiiZbNIH$YiCsHZjcTkYd31wkgW8JF.'), - ('pass', '$1$YeNsbWdH$wvOF8JdqsoiLix754LTW90'), - ('____fifteen____', '$1$s9lUWACI$Kk1jtIVVdmT01p0z3b/hw1'), - ('____sixteen_____', '$1$dL3xbVZI$kkgqhCanLdxODGq14g/tW1'), - ('____seventeen____', '$1$NaH5na7J$j7y8Iss0hcRbu3kzoJs5V.'), - ('__________thirty-three___________', '$1$HO7Q6vzJ$yGwp2wbL5D7eOVzOmxpsy.'), - ('apache', '$apr1$J.w5a/..$IW9y6DR0oO/ADuhlMF5/X1') - ) - - for clearpw, hashpw in test_cases: - if test(clearpw, hashpw): - print '%s: pass' % clearpw - else: - print '%s: FAIL' % clearpw diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 458ea2b5..1bf57b8a 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -16,9 +16,8 @@ import sys, os, string, socket, time import shutil, tempfile, threading import SocketServer from OpenSSL import SSL -from netlib import odict, tcp, http, wsgi, certutils, http_status +from netlib import odict, tcp, http, wsgi, certutils, http_status, http_auth import utils, flow, version, platform, controller -import authentication KILL = 0 @@ -619,14 +618,14 @@ def process_proxy_options(parser, options): if len(options.auth_singleuser.split(':')) != 2: parser.error("Please specify user in the format username:password") username, password = options.auth_singleuser.split(':') - password_manager = authentication.SingleUserPasswordManager(username, password) + password_manager = http_auth.PassManSingleUser(username, password) elif options.auth_nonanonymous: - password_manager = authentication.PermissivePasswordManager() + password_manager = http_auth.PassManNonAnon() elif options.auth_htpasswd: - password_manager = authentication.HtpasswdPasswordManager(options.auth_htpasswd) - authenticator = authentication.BasicProxyAuth(password_manager, "mitmproxy") + password_manager = http_auth.PassManHtpasswd(options.auth_htpasswd) + authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") else: - authenticator = authentication.NullProxyAuth(None) + authenticator = http_auth.NullProxyAuth(None) return ProxyConfig( certfile = options.cert, -- cgit v1.2.3 From d5876a12ed09940df77da823bf857437f7496524 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 3 Mar 2013 11:58:57 +1300 Subject: Unit test proxy option parsing. --- libmproxy/proxy.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 1bf57b8a..9fe878a9 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -573,22 +573,19 @@ def process_proxy_options(parser, options): if options.cert: options.cert = os.path.expanduser(options.cert) if not os.path.exists(options.cert): - parser.error("Manually created certificate does not exist: %s"%options.cert) + return parser.error("Manually created certificate does not exist: %s"%options.cert) cacert = os.path.join(options.confdir, "mitmproxy-ca.pem") cacert = os.path.expanduser(cacert) if not os.path.exists(cacert): certutils.dummy_ca(cacert) - if getattr(options, "cache", None) is not None: - options.cache = os.path.expanduser(options.cache) body_size_limit = utils.parse_size(options.body_size_limit) - if options.reverse_proxy and options.transparent_proxy: - parser.errror("Can't set both reverse proxy and transparent proxy.") + return parser.error("Can't set both reverse proxy and transparent proxy.") if options.transparent_proxy: if not platform.resolver: - parser.error("Transparent mode not supported on this platform.") + return parser.error("Transparent mode not supported on this platform.") trans = dict( resolver = platform.resolver(), sslports = TRANSPARENT_SSL_PORTS @@ -599,30 +596,33 @@ def process_proxy_options(parser, options): if options.reverse_proxy: rp = utils.parse_proxy_spec(options.reverse_proxy) if not rp: - parser.error("Invalid reverse proxy specification: %s"%options.reverse_proxy) + return parser.error("Invalid reverse proxy specification: %s"%options.reverse_proxy) else: rp = None if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): - parser.error("Client certificate directory does not exist or is not a directory: %s"%options.clientcerts) + return parser.error("Client certificate directory does not exist or is not a directory: %s"%options.clientcerts) if options.certdir: options.certdir = os.path.expanduser(options.certdir) if not os.path.exists(options.certdir) or not os.path.isdir(options.certdir): - parser.error("Dummy cert directory does not exist or is not a directory: %s"%options.certdir) + return parser.error("Dummy cert directory does not exist or is not a directory: %s"%options.certdir) if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd): if options.auth_singleuser: if len(options.auth_singleuser.split(':')) != 2: - parser.error("Please specify user in the format username:password") + return parser.error("Invalid single-user specification. Please use the format username:password") username, password = options.auth_singleuser.split(':') password_manager = http_auth.PassManSingleUser(username, password) elif options.auth_nonanonymous: password_manager = http_auth.PassManNonAnon() elif options.auth_htpasswd: - password_manager = http_auth.PassManHtpasswd(options.auth_htpasswd) + try: + password_manager = http_auth.PassManHtpasswd(options.auth_htpasswd) + except ValueError, v: + return parser.error(v.message) authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") else: authenticator = http_auth.NullProxyAuth(None) -- cgit v1.2.3 From 2465b8a376a7eb04eef6a1cce46dd11a8f1830d8 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 3 Mar 2013 12:13:33 +1300 Subject: 100% unit test coverage on proxy.py. Hallelujah! --- libmproxy/proxy.py | 7 ++----- 1 file changed, 2 insertions(+), 5 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 9fe878a9..75e195ea 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -504,10 +504,7 @@ class ProxyServer(tcp.TCPServer): def handle_connection(self, request, client_address): h = ProxyHandler(self.config, request, client_address, self, self.channel, self.server_version) h.handle() - try: - h.finish() - except tcp.NetLibDisconnect, e: - pass + h.finish() def handle_shutdown(self): self.config.certstore.cleanup() @@ -540,7 +537,7 @@ class DummyServer: def __init__(self, config): self.config = config - def start_slave(self, klass, channel): + def start_slave(self, *args): pass def shutdown(self): -- cgit v1.2.3 From 75b5c970950f11249353ee10931305cfc1c21400 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 3 Mar 2013 12:18:19 +1300 Subject: Revert "show current filepath in status bar" This reverts commit bf8367d6cf3e6b5b1e916453c9cf114194174a1a. This just doesn't work. We need a better solution, probably in the next release. --- libmproxy/console/__init__.py | 3 --- 1 file changed, 3 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index a16cc4dc..4e408012 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -195,9 +195,6 @@ class StatusBar(common.WWrap): if self.master.stream: r.append("[W:%s]"%self.master.stream_path) - if self.master.state.last_saveload: - r.append("[%s]"%self.master.state.last_saveload) - return r def redraw(self): -- cgit v1.2.3 From e608d10f455550b8afd09217f0ecf344a0bdc814 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 3 Mar 2013 12:26:20 +1300 Subject: Remove __slots__ to make it possible to inherit from Options classes. --- libmproxy/console/__init__.py | 4 ++-- libmproxy/dump.py | 4 ++-- 2 files changed, 4 insertions(+), 4 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 4e408012..4b5d1274 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -325,7 +325,7 @@ class ConsoleState(flow.State): class Options(object): - __slots__ = [ + attributes = [ "anticache", "anticomp", "client_replay", @@ -352,7 +352,7 @@ class Options(object): def __init__(self, **kwargs): for k, v in kwargs.items(): setattr(self, k, v) - for i in self.__slots__: + for i in self.attributes: if not hasattr(self, i): setattr(self, i, None) diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 3c7eee71..3a315409 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -21,7 +21,7 @@ class DumpError(Exception): pass class Options(object): - __slots__ = [ + attributes = [ "anticache", "anticomp", "client_replay", @@ -45,7 +45,7 @@ class Options(object): def __init__(self, **kwargs): for k, v in kwargs.items(): setattr(self, k, v) - for i in self.__slots__: + for i in self.attributes: if not hasattr(self, i): setattr(self, i, None) -- cgit v1.2.3 From 7835e0c2c7be0e45262a06b5e2ec2399ae019977 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 3 Mar 2013 14:56:56 +1300 Subject: Begin some simple fuzzing with pathod. Finally doing what I started writing pathod for in the first place... --- libmproxy/flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 883c7235..4fefee9f 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -463,7 +463,7 @@ class Request(HTTPMsg): """ Returns a URL string, constructed from the Request's URL compnents. """ - return utils.unparse_url(self.scheme, self.host.decode("idna"), self.port, self.path).encode('ascii') + return utils.unparse_url(self.scheme, self.host.encode("idna"), self.port, self.path).encode('ascii') def set_url(self, url): """ -- cgit v1.2.3 From cde66cd58470cd68a76a9d8b1022a45e99a5cd8d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 3 Mar 2013 22:03:27 +1300 Subject: Fuzzing, and fixes for errors found with fuzzing. --- libmproxy/proxy.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 75e195ea..7459fadf 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -237,6 +237,8 @@ class ProxyHandler(tcp.BaseHandler): continue else: raise + except http.HttpError, v: + raise ProxyError(502, "Invalid server response.") else: break @@ -278,7 +280,6 @@ class ProxyHandler(tcp.BaseHandler): ) else: self.log(cc, cc.error) - if isinstance(e, ProxyError): self.send_error(e.code, e.msg, e.headers) else: -- cgit v1.2.3 From cfb5ba89ce96594b2f8d51f27c9b2ee41ecf18e5 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 14 Mar 2013 09:19:43 +1300 Subject: Introduce a filtered flow writer, and use it in dump.py Fixes #104 --- libmproxy/console/__init__.py | 2 +- libmproxy/dump.py | 4 ++-- libmproxy/flow.py | 17 +++++++++++++++-- 3 files changed, 18 insertions(+), 5 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 4b5d1274..98904c84 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -426,7 +426,7 @@ class ConsoleMaster(flow.FlowMaster): path = os.path.expanduser(path) try: f = file(path, "wb") - flow.FlowMaster.start_stream(self, f) + flow.FlowMaster.start_stream(self, f, None) except IOError, v: return str(v) self.stream_path = path diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 3a315409..d716e433 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -93,7 +93,7 @@ class DumpMaster(flow.FlowMaster): path = os.path.expanduser(options.wfile) try: f = file(path, "wb") - self.start_stream(f) + self.start_stream(f, self.filt) except IOError, v: raise DumpError(v.strerror) @@ -155,6 +155,7 @@ class DumpMaster(flow.FlowMaster): return "\n".join(" "*n + i for i in l) def _process_flow(self, f): + self.state.delete_flow(f) if self.filt and not f.match(self.filt): return @@ -198,7 +199,6 @@ class DumpMaster(flow.FlowMaster): print >> self.outfile, "\n" if self.o.verbosity: self.outfile.flush() - self.state.delete_flow(f) def handle_log(self, l): self.add_event(l.msg) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 4fefee9f..13b32011 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -1588,8 +1588,8 @@ class FlowMaster(controller.Master): self.stream.add(i) self.stop_stream() - def start_stream(self, fp): - self.stream = FlowWriter(fp) + def start_stream(self, fp, filt): + self.stream = FilteredFlowWriter(fp, filt) def stop_stream(self): self.stream.fo.close() @@ -1635,3 +1635,16 @@ class FlowReader: return raise FlowReadError("Invalid data format.") + +class FilteredFlowWriter: + def __init__(self, fo, filt): + self.fo = fo + self.filt = filt + + def add(self, f): + if self.filt and not f.match(self.filt): + return + d = f._get_state() + tnetstring.dump(d, self.fo) + + -- cgit v1.2.3 From 790ad468e4352419ef519401680f99ee3beb148d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 17 Mar 2013 14:35:36 +1300 Subject: Fix bug that caused mis-identification of some HTTPS connections in transparent mode. --- libmproxy/proxy.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 7459fadf..3d55190d 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -323,17 +323,18 @@ class ProxyHandler(tcp.BaseHandler): if not orig: raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") host, port = orig - if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]): + if port in self.config.transparent_proxy["sslports"]: scheme = "https" - dummycert = self.find_cert(client_conn, host, port, host) - sni = HandleSNI( - self, client_conn, host, port, - dummycert, self.config.certfile or self.config.cacert - ) - try: - self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) - except tcp.NetLibError, v: - raise ProxyError(400, str(v)) + if not self.ssl_established: + dummycert = self.find_cert(client_conn, host, port, host) + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) + try: + self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) + except tcp.NetLibError, v: + raise ProxyError(400, str(v)) else: scheme = "http" line = self.get_line(self.rfile) -- cgit v1.2.3 From 0e993bec6f7fa77e73a08053f4558ff1fc36d022 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 17 Mar 2013 17:31:35 +1300 Subject: Add the --host option, which uses the value in the Host header for dispaly URLs. - Can be toggled with "o" then "h" in mitmproxy - Useful for transparent mode --- libmproxy/cmdline.py | 6 ++++++ libmproxy/console/__init__.py | 9 +++++++++ libmproxy/console/common.py | 4 ++-- libmproxy/console/flowlist.py | 2 +- libmproxy/console/flowview.py | 4 ++-- libmproxy/console/help.py | 4 ++++ libmproxy/flow.py | 12 ++++++++++-- 7 files changed, 34 insertions(+), 7 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index de70bea8..1d5902a9 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -154,6 +154,7 @@ def get_common_options(options): script = options.script, stickycookie = stickycookie, stickyauth = stickyauth, + showhost = options.showhost, wfile = options.wfile, verbosity = options.verbose, nopop = options.nopop, @@ -248,6 +249,11 @@ def common_options(parser): help="Byte size limit of HTTP request and response bodies."\ " Understands k/m/g suffixes, i.e. 3m for 3 megabytes." ) + parser.add_argument( + "--host", + action="store_true", dest="showhost", default=False, + help="Use the Host header to construct URLs for display." + ) parser.add_argument( "--no-upstream-cert", default=False, diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 98904c84..fe75a047 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -174,6 +174,8 @@ class StatusBar(common.WWrap): opts.append("anticache") if self.master.anticomp: opts.append("anticomp") + if self.master.showhost: + opts.append("showhost") if not self.master.refresh_server_playback: opts.append("norefresh") if self.master.killextra: @@ -338,6 +340,7 @@ class Options(object): "refresh_server_playback", "rfile", "script", + "showhost", "replacements", "rheaders", "setheaders", @@ -398,6 +401,7 @@ class ConsoleMaster(flow.FlowMaster): self.killextra = options.kill self.rheaders = options.rheaders self.nopop = options.nopop + self.showhost = options.showhost self.eventlog = options.eventlog self.eventlist = urwid.SimpleListWalker([]) @@ -918,6 +922,7 @@ class ConsoleMaster(flow.FlowMaster): ( ("anticache", "a"), ("anticomp", "c"), + ("showhost", "h"), ("killextra", "k"), ("norefresh", "n"), ("no-upstream-certs", "u"), @@ -957,6 +962,10 @@ class ConsoleMaster(flow.FlowMaster): self.anticache = not self.anticache if a == "c": self.anticomp = not self.anticomp + if a == "h": + self.showhost = not self.showhost + self.sync_list_view() + self.refresh_flow(self.currentflow) elif a == "k": self.killextra = not self.killextra elif a == "n": diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 1cc0b5b9..d68aba3d 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -177,7 +177,7 @@ class FlowCache: flowcache = FlowCache() -def format_flow(f, focus, extended=False, padding=2): +def format_flow(f, focus, extended=False, hostheader=False, padding=2): d = dict( intercepting = f.intercepting, @@ -185,7 +185,7 @@ def format_flow(f, focus, extended=False, padding=2): req_is_replay = f.request.is_replay(), req_method = f.request.method, req_acked = f.request.reply.acked, - req_url = f.request.get_url(), + req_url = f.request.get_url(hostheader=hostheader), err_msg = f.error.msg if f.error else None, resp_code = f.response.code if f.response else None, diff --git a/libmproxy/console/flowlist.py b/libmproxy/console/flowlist.py index c70393a1..8fd4efce 100644 --- a/libmproxy/console/flowlist.py +++ b/libmproxy/console/flowlist.py @@ -105,7 +105,7 @@ class ConnectionItem(common.WWrap): common.WWrap.__init__(self, w) def get_text(self): - return common.format_flow(self.flow, self.f) + return common.format_flow(self.flow, self.f, hostheader=self.master.showhost) def selectable(self): return True diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 4215f170..9bec7bc6 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -88,11 +88,11 @@ footer = [ class FlowViewHeader(common.WWrap): def __init__(self, master, f): self.master, self.flow = master, f - self.w = common.format_flow(f, False, extended=True, padding=0) + self.w = common.format_flow(f, False, extended=True, padding=0, hostheader=self.master.showhost) def refresh_flow(self, f): if f == self.flow: - self.w = common.format_flow(f, False, extended=True, padding=0) + self.w = common.format_flow(f, False, extended=True, padding=0, hostheader=self.master.showhost) class CallbackCache: diff --git a/libmproxy/console/help.py b/libmproxy/console/help.py index 178b36f7..40f81955 100644 --- a/libmproxy/console/help.py +++ b/libmproxy/console/help.py @@ -97,6 +97,10 @@ class HelpView(urwid.ListBox): common.highlight_key("anticomp", "c") + [("text", ": prevent compressed responses")] ), + (None, + common.highlight_key("showhost", "h") + + [("text", ": use Host header for URL display")] + ), (None, common.highlight_key("killextra", "k") + [("text", ": kill requests not part of server replay")] diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 13b32011..380eb026 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -459,11 +459,19 @@ class Request(HTTPMsg): query = utils.urlencode(odict.lst) self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment])) - def get_url(self): + def get_url(self, hostheader=False): """ Returns a URL string, constructed from the Request's URL compnents. + + If hostheader is True, we use the value specified in the request + Host header to construct the URL. """ - return utils.unparse_url(self.scheme, self.host.encode("idna"), self.port, self.path).encode('ascii') + if hostheader: + host = self.headers.get_first("host") or self.host + else: + host = self.host + host = host.encode("idna") + return utils.unparse_url(self.scheme, host, self.port, self.path).encode('ascii') def set_url(self, url): """ -- cgit v1.2.3 From e50da8164fb4bd3b561846181b3a36df06e8d577 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 17 Mar 2013 17:43:31 +1300 Subject: Enable --host option for mitmdump --- libmproxy/dump.py | 11 ++++++----- 1 file changed, 6 insertions(+), 5 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/dump.py b/libmproxy/dump.py index d716e433..2e0e62d9 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -57,12 +57,12 @@ def str_response(resp): return r -def str_request(req): +def str_request(req, showhost): if req.client_conn: c = req.client_conn.address[0] else: c = "[replay]" - r = "%s %s %s"%(c, req.method, req.get_url()) + r = "%s %s %s"%(c, req.method, req.get_url(showhost)) if req.stickycookie: r = "[stickycookie] " + r return r @@ -76,6 +76,7 @@ class DumpMaster(flow.FlowMaster): self.anticache = options.anticache self.anticomp = options.anticomp self.eventlog = options.eventlog + self.showhost = options.showhost self.refresh_server_playback = options.refresh_server_playback if filtstr: @@ -179,16 +180,16 @@ class DumpMaster(flow.FlowMaster): result = " << %s"%f.error.msg if self.o.verbosity == 1: - print >> self.outfile, str_request(f.request) + print >> self.outfile, str_request(f.request, self.showhost) print >> self.outfile, result elif self.o.verbosity == 2: - print >> self.outfile, str_request(f.request) + print >> self.outfile, str_request(f.request, self.showhost) print >> self.outfile, self.indent(4, f.request.headers) print >> self.outfile print >> self.outfile, result print >> self.outfile, "\n" elif self.o.verbosity >= 3: - print >> self.outfile, str_request(f.request) + print >> self.outfile, str_request(f.request, self.showhost) print >> self.outfile, self.indent(4, f.request.headers) if utils.isBin(f.request.content): print >> self.outfile, self.indent(4, netlib.utils.hexdump(f.request.content)) -- cgit v1.2.3 From d2d3eb6490a6b342f8d205e26d04c913b8e2a5f7 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 17 Mar 2013 17:53:39 +1300 Subject: Un-break unit tests. Tsk tsk. --- libmproxy/dump.py | 1 + 1 file changed, 1 insertion(+) (limited to 'libmproxy') diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 2e0e62d9..b1022ef5 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -37,6 +37,7 @@ class Options(object): "setheaders", "server_replay", "script", + "showhost", "stickycookie", "stickyauth", "verbosity", -- cgit v1.2.3 From 6614498744a45138adc770ef6e5882366b96b25e Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 18 Mar 2013 08:36:56 +1300 Subject: Update styling, GameCenter highscore tutorial. --- libmproxy/flow.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy') diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 380eb026..d33a3108 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -1309,7 +1309,7 @@ class State(object): if f.request in self._flow_map: del self._flow_map[f.request] self._flow_list.remove(f) - if f.match(self._limit): + if f in self.view: self.view.remove(f) return True -- cgit v1.2.3