diff options
Diffstat (limited to 'libmproxy/proxy.py')
| -rw-r--r-- | libmproxy/proxy.py | 417 |
1 files changed, 247 insertions, 170 deletions
diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index f14e4e3e..3fac17b8 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -16,9 +16,13 @@ import sys, os, string, socket, time import shutil, tempfile, threading import SocketServer from OpenSSL import SSL -from netlib import odict, tcp, http, wsgi, certutils, http_status -import utils, flow, version, platform, controller -import authentication +from netlib import odict, tcp, http, wsgi, certutils, http_status, http_auth +import utils, flow, version, platform, controller, app + + +APP_DOMAIN = "mitm" +APP_IP = "1.1.1.1" +KILL = 0 class ProxyError(Exception): @@ -29,15 +33,14 @@ class ProxyError(Exception): return "ProxyError(%s, %s)"%(self.code, self.msg) -class Log(controller.Msg): +class Log: def __init__(self, msg): - controller.Msg.__init__(self) self.msg = msg class ProxyConfig: - def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, transparent_proxy=None, certdir = None, authenticator=None): - assert not (reverse_proxy and transparent_proxy) + def __init__(self, app=False, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, transparent_proxy=None, certdir = None, authenticator=None): + self.app = app self.certfile = certfile self.cacert = cacert self.clientcerts = clientcerts @@ -49,45 +52,23 @@ class ProxyConfig: self.certstore = certutils.CertStore(certdir) -class RequestReplayThread(threading.Thread): - def __init__(self, config, flow, masterq): - self.config, self.flow, self.masterq = config, flow, masterq - threading.Thread.__init__(self) - - def run(self): - try: - r = self.flow.request - server = ServerConnection(self.config, r.host, r.port) - server.connect(r.scheme) - server.send(r) - httpversion, code, msg, headers, content = http.read_response( - server.rfile, r.method, self.config.body_size_limit - ) - response = flow.Response( - self.flow.request, httpversion, code, msg, headers, content, server.cert - ) - response._send(self.masterq) - except (ProxyError, http.HttpError, tcp.NetLibError), v: - err = flow.Error(self.flow.request, str(v)) - err._send(self.masterq) - - class ServerConnection(tcp.TCPClient): - def __init__(self, config, host, port): + def __init__(self, config, scheme, host, port, sni): tcp.TCPClient.__init__(self, host, port) self.config = config + self.scheme, self.sni = scheme, sni self.requestcount = 0 - def connect(self, scheme): + def connect(self): tcp.TCPClient.connect(self) - if scheme == "https": + if self.scheme == "https": clientcert = None if self.config.clientcerts: path = os.path.join(self.config.clientcerts, self.host.encode("idna")) + ".pem" if os.path.exists(path): clientcert = path try: - self.convert_to_ssl(clientcert=clientcert, sni=self.host) + self.convert_to_ssl(cert=clientcert, sni=self.sni) except tcp.NetLibError, v: raise ProxyError(400, str(v)) @@ -101,49 +82,115 @@ class ServerConnection(tcp.TCPClient): def terminate(self): try: - if not self.wfile.closed: - self.wfile.flush() + self.wfile.flush() self.connection.close() except IOError: pass + +class RequestReplayThread(threading.Thread): + def __init__(self, config, flow, masterq): + self.config, self.flow, self.channel = config, flow, controller.Channel(masterq) + threading.Thread.__init__(self) + + def run(self): + try: + r = self.flow.request + server = ServerConnection(self.config, r.scheme, r.host, r.port, r.host) + server.connect() + server.send(r) + httpversion, code, msg, headers, content = http.read_response( + server.rfile, r.method, self.config.body_size_limit + ) + response = flow.Response( + self.flow.request, httpversion, code, msg, headers, content, server.cert + ) + self.channel.ask(response) + except (ProxyError, http.HttpError, tcp.NetLibError), v: + err = flow.Error(self.flow.request, str(v)) + self.channel.ask(err) + + +class HandleSNI: + def __init__(self, handler, client_conn, host, port, cert, key): + self.handler, self.client_conn, self.host, self.port = handler, client_conn, host, port + self.cert, self.key = cert, key + + def __call__(self, connection): + try: + sn = connection.get_servername() + if sn: + self.handler.get_server_connection(self.client_conn, "https", self.host, self.port, sn) + new_context = SSL.Context(SSL.TLSv1_METHOD) + new_context.use_privatekey_file(self.key) + new_context.use_certificate_file(self.cert) + connection.set_context(new_context) + self.handler.sni = sn.decode("utf8").encode("idna") + # An unhandled exception in this method will core dump PyOpenSSL, so + # make dang sure it doesn't happen. + except Exception, e: # pragma: no cover + pass + + class ProxyHandler(tcp.BaseHandler): - def __init__(self, config, connection, client_address, server, mqueue, server_version): - self.mqueue, self.server_version = mqueue, server_version + def __init__(self, config, connection, client_address, server, channel, server_version): + self.channel, self.server_version = channel, server_version self.config = config - self.server_conn = None self.proxy_connect_state = None self.sni = None + self.server_conn = None tcp.BaseHandler.__init__(self, connection, client_address, server) + def get_server_connection(self, cc, scheme, host, port, sni): + """ + When SNI is in play, this means we have an SSL-encrypted + connection, which means that the entire handler is dedicated to a + single server connection - no multiplexing. If this assumption ever + breaks, we'll have to do something different with the SNI host + variable on the handler object. + """ + sc = self.server_conn + if not sni: + sni = host + if sc and (scheme, host, port, sni) != (sc.scheme, sc.host, sc.port, sc.sni): + sc.terminate() + self.server_conn = None + self.log( + cc, + "switching connection", [ + "%s://%s:%s (sni=%s) -> %s://%s:%s (sni=%s)"%( + scheme, host, port, sni, + sc.scheme, sc.host, sc.port, sc.sni + ) + ] + ) + if not self.server_conn: + try: + self.server_conn = ServerConnection(self.config, scheme, host, port, sni) + self.server_conn.connect() + except tcp.NetLibError, v: + raise ProxyError(502, v) + return self.server_conn + + def del_server_connection(self): + self.server_conn = None + def handle(self): cc = flow.ClientConnect(self.client_address) self.log(cc, "connect") - cc._send(self.mqueue) + self.channel.ask(cc) while self.handle_request(cc) and not cc.close: pass cc.close = True - cd = flow.ClientDisconnect(cc) + cd = flow.ClientDisconnect(cc) self.log( cc, "disconnect", [ "handled %s requests"%cc.requestcount] ) - cd._send(self.mqueue) - - def server_connect(self, scheme, host, port): - sc = self.server_conn - if sc and (host, port) != (sc.host, sc.port): - sc.terminate() - self.server_conn = None - if not self.server_conn: - try: - self.server_conn = ServerConnection(self.config, host, port) - self.server_conn.connect(scheme) - except tcp.NetLibError, v: - raise ProxyError(502, v) + self.channel.tell(cd) def handle_request(self, cc): try: @@ -160,45 +207,68 @@ class ProxyHandler(tcp.BaseHandler): self.log(cc, "Error in wsgi app.", err.split("\n")) return else: - request = request._send(self.mqueue) - if request is None: + request_reply = self.channel.ask(request) + if request_reply == KILL: return - - if isinstance(request, flow.Response): - response = request + elif isinstance(request_reply, flow.Response): request = False - response = response._send(self.mqueue) + response = request_reply + response_reply = self.channel.ask(response) else: + request = request_reply if self.config.reverse_proxy: scheme, host, port = self.config.reverse_proxy else: scheme, host, port = request.scheme, request.host, request.port - self.server_connect(scheme, host, port) - self.server_conn.send(request) - self.server_conn.rfile.reset_timestamps() - httpversion, code, msg, headers, content = http.read_response( - self.server_conn.rfile, - request.method, - self.config.body_size_limit - ) + + # If we've already pumped a request over this connection, + # it's possible that the server has timed out. If this is + # the case, we want to reconnect without sending an error + # to the client. + while 1: + sc = self.get_server_connection(cc, scheme, host, port, self.sni) + sc.send(request) + sc.rfile.reset_timestamps() + try: + httpversion, code, msg, headers, content = http.read_response( + sc.rfile, + request.method, + self.config.body_size_limit + ) + except http.HttpErrorConnClosed, v: + self.del_server_connection() + if sc.requestcount > 1: + continue + else: + raise + except http.HttpError, v: + raise ProxyError(502, "Invalid server response.") + else: + break + response = flow.Response( - request, httpversion, code, msg, headers, content, self.server_conn.cert, self.server_conn.rfile.first_byte_timestamp, utils.timestamp() + request, httpversion, code, msg, headers, content, sc.cert, + sc.rfile.first_byte_timestamp, utils.timestamp() ) + response_reply = self.channel.ask(response) + # Not replying to the server invalidates the server + # connection, so we terminate. + if response_reply == KILL: + sc.terminate() - response = response._send(self.mqueue) - if response is None: - self.server_conn.terminate() - if response is None: - return - self.send_response(response) - if request and http.request_connection_close(request.httpversion, request.headers): - return - # We could keep the client connection when the server - # connection needs to go away. However, we want to mimic - # behaviour as closely as possible to the client, so we - # disconnect. - if http.response_connection_close(response.httpversion, response.headers): + if response_reply == KILL: return + else: + response = response_reply + self.send_response(response) + if request and http.request_connection_close(request.httpversion, request.headers): + return + # We could keep the client connection when the server + # connection needs to go away. However, we want to mimic + # behaviour as closely as possible to the client, so we + # disconnect. + if http.response_connection_close(response.httpversion, response.headers): + return except (IOError, ProxyError, http.HttpError, tcp.NetLibDisconnect), e: if hasattr(e, "code"): cc.error = "%s: %s"%(e.code, e.msg) @@ -207,14 +277,13 @@ class ProxyHandler(tcp.BaseHandler): if request: err = flow.Error(request, cc.error) - err._send(self.mqueue) + self.channel.ask(err) self.log( cc, cc.error, ["url: %s"%request.get_url()] ) else: self.log(cc, cc.error) - if isinstance(e, ProxyError): self.send_error(e.code, e.msg, e.headers) else: @@ -228,23 +297,20 @@ class ProxyHandler(tcp.BaseHandler): msg.append(" -> "+i) msg = "\n".join(msg) l = Log(msg) - l._send(self.mqueue) + self.channel.tell(l) - def find_cert(self, host, port, sni): + def find_cert(self, cc, host, port, sni): if self.config.certfile: return self.config.certfile else: sans = [] if not self.config.no_upstream_cert: - try: - cert = certutils.get_remote_cert(host, port, sni) - except tcp.NetLibError, v: - raise ProxyError(502, "Unable to get remote cert: %s"%str(v)) - sans = cert.altnames - host = cert.cn.decode("utf8").encode("idna") + conn = self.get_server_connection(cc, "https", host, port, sni) + sans = conn.cert.altnames + host = conn.cert.cn.decode("utf8").encode("idna") ret = self.config.certstore.get_cert(host, sans, self.config.cacert) if not ret: - raise ProxyError(502, "mitmproxy: Unable to generate dummy cert.") + raise ProxyError(502, "Unable to generate dummy cert.") return ret def get_line(self, fp): @@ -256,26 +322,27 @@ class ProxyHandler(tcp.BaseHandler): line = fp.readline() return line - def handle_sni(self, conn): - sn = conn.get_servername() - if sn: - self.sni = sn.decode("utf8").encode("idna") - def read_request_transparent(self, client_conn): orig = self.config.transparent_proxy["resolver"].original_addr(self.connection) if not orig: raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") + self.log(client_conn, "transparent to %s:%s"%orig) + host, port = orig - if not self.ssl_established and (port in self.config.transparent_proxy["sslports"]): + if port in self.config.transparent_proxy["sslports"]: scheme = "https" - certfile = self.find_cert(host, port, None) - try: - self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert) - except tcp.NetLibError, v: - raise ProxyError(400, str(v)) + if not self.ssl_established: + dummycert = self.find_cert(client_conn, host, port, host) + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) + try: + self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) + except tcp.NetLibError, v: + raise ProxyError(400, str(v)) else: scheme = "http" - host = self.sni or host line = self.get_line(self.rfile) if line == "": return None @@ -292,50 +359,33 @@ class ProxyHandler(tcp.BaseHandler): self.rfile.first_byte_timestamp, utils.timestamp() ) - def read_request_reverse(self, client_conn): - line = self.get_line(self.rfile) - if line == "": - return None - scheme, host, port = self.config.reverse_proxy - r = http.parse_init_http(line) - if not r: - raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) - method, path, httpversion = r - headers = self.read_headers(authenticate=False) - content = http.read_http_body_request( - self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit - ) - return flow.Request( - client_conn, httpversion, host, port, "http", method, path, headers, content, - self.rfile.first_byte_timestamp, utils.timestamp() - ) - - def read_request_proxy(self, client_conn): line = self.get_line(self.rfile) if line == "": return None - if http.parse_init_connect(line): - r = http.parse_init_connect(line) - if not r: - raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) - host, port, httpversion = r - headers = self.read_headers(authenticate=True) - - self.wfile.write( - 'HTTP/1.1 200 Connection established\r\n' + - ('Proxy-agent: %s\r\n'%self.server_version) + - '\r\n' - ) - self.wfile.flush() - certfile = self.find_cert(host, port, None) - try: - self.convert_to_ssl(certfile, self.config.certfile or self.config.cacert) - except tcp.NetLibError, v: - raise ProxyError(400, str(v)) - self.proxy_connect_state = (host, port, httpversion) - line = self.rfile.readline(line) + if not self.proxy_connect_state: + connparts = http.parse_init_connect(line) + if connparts: + host, port, httpversion = connparts + headers = self.read_headers(authenticate=True) + self.wfile.write( + 'HTTP/1.1 200 Connection established\r\n' + + ('Proxy-agent: %s\r\n'%self.server_version) + + '\r\n' + ) + self.wfile.flush() + dummycert = self.find_cert(client_conn, host, port, host) + sni = HandleSNI( + self, client_conn, host, port, + dummycert, self.config.certfile or self.config.cacert + ) + try: + self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) + except tcp.NetLibError, v: + raise ProxyError(400, str(v)) + self.proxy_connect_state = (host, port, httpversion) + line = self.rfile.readline(line) if self.proxy_connect_state: r = http.parse_init_http(line) @@ -366,6 +416,24 @@ class ProxyHandler(tcp.BaseHandler): self.rfile.first_byte_timestamp, utils.timestamp() ) + def read_request_reverse(self, client_conn): + line = self.get_line(self.rfile) + if line == "": + return None + scheme, host, port = self.config.reverse_proxy + r = http.parse_init_http(line) + if not r: + raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) + method, path, httpversion = r + headers = self.read_headers(authenticate=False) + content = http.read_http_body_request( + self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit + ) + return flow.Request( + client_conn, httpversion, host, port, "http", method, path, headers, content, + self.rfile.first_byte_timestamp, utils.timestamp() + ) + def read_request(self, client_conn): self.rfile.reset_timestamps() if self.config.transparent_proxy: @@ -431,23 +499,31 @@ class ProxyServer(tcp.TCPServer): tcp.TCPServer.__init__(self, (address, port)) except socket.error, v: raise ProxyServerError('Error starting proxy server: ' + v.strerror) - self.masterq = None + self.channel = None self.apps = AppRegistry() + if config.app: + self.apps.add( + app.mapp, + APP_DOMAIN, + 80 + ) + self.apps.add( + app.mapp, + APP_IP, + 80 + ) - def start_slave(self, klass, masterq): - slave = klass(masterq, self) + def start_slave(self, klass, channel): + slave = klass(channel, self) slave.start() - def set_mqueue(self, q): - self.masterq = q + def set_channel(self, channel): + self.channel = channel def handle_connection(self, request, client_address): - h = ProxyHandler(self.config, request, client_address, self, self.masterq, self.server_version) + h = ProxyHandler(self.config, request, client_address, self, self.channel, self.server_version) h.handle() - try: - h.finish() - except tcp.NetLibDisconnect, e: - pass + h.finish() def handle_shutdown(self): self.config.certstore.cleanup() @@ -480,7 +556,7 @@ class DummyServer: def __init__(self, config): self.config = config - def start_slave(self, klass, masterq): + def start_slave(self, *args): pass def shutdown(self): @@ -513,22 +589,19 @@ def process_proxy_options(parser, options): if options.cert: options.cert = os.path.expanduser(options.cert) if not os.path.exists(options.cert): - parser.error("Manually created certificate does not exist: %s"%options.cert) + return parser.error("Manually created certificate does not exist: %s"%options.cert) cacert = os.path.join(options.confdir, "mitmproxy-ca.pem") cacert = os.path.expanduser(cacert) if not os.path.exists(cacert): certutils.dummy_ca(cacert) - if getattr(options, "cache", None) is not None: - options.cache = os.path.expanduser(options.cache) body_size_limit = utils.parse_size(options.body_size_limit) - if options.reverse_proxy and options.transparent_proxy: - parser.errror("Can't set both reverse proxy and transparent proxy.") + return parser.error("Can't set both reverse proxy and transparent proxy.") if options.transparent_proxy: if not platform.resolver: - parser.error("Transparent mode not supported on this platform.") + return parser.error("Transparent mode not supported on this platform.") trans = dict( resolver = platform.resolver(), sslports = TRANSPARENT_SSL_PORTS @@ -539,35 +612,39 @@ def process_proxy_options(parser, options): if options.reverse_proxy: rp = utils.parse_proxy_spec(options.reverse_proxy) if not rp: - parser.error("Invalid reverse proxy specification: %s"%options.reverse_proxy) + return parser.error("Invalid reverse proxy specification: %s"%options.reverse_proxy) else: rp = None if options.clientcerts: options.clientcerts = os.path.expanduser(options.clientcerts) if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts): - parser.error("Client certificate directory does not exist or is not a directory: %s"%options.clientcerts) + return parser.error("Client certificate directory does not exist or is not a directory: %s"%options.clientcerts) if options.certdir: options.certdir = os.path.expanduser(options.certdir) if not os.path.exists(options.certdir) or not os.path.isdir(options.certdir): - parser.error("Dummy cert directory does not exist or is not a directory: %s"%options.certdir) + return parser.error("Dummy cert directory does not exist or is not a directory: %s"%options.certdir) if (options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd): if options.auth_singleuser: if len(options.auth_singleuser.split(':')) != 2: - parser.error("Please specify user in the format username:password") + return parser.error("Invalid single-user specification. Please use the format username:password") username, password = options.auth_singleuser.split(':') - password_manager = authentication.SingleUserPasswordManager(username, password) + password_manager = http_auth.PassManSingleUser(username, password) elif options.auth_nonanonymous: - password_manager = authentication.PermissivePasswordManager() + password_manager = http_auth.PassManNonAnon() elif options.auth_htpasswd: - password_manager = authentication.HtpasswdPasswordManager(options.auth_htpasswd) - authenticator = authentication.BasicProxyAuth(password_manager, "mitmproxy") + try: + password_manager = http_auth.PassManHtpasswd(options.auth_htpasswd) + except ValueError, v: + return parser.error(v.message) + authenticator = http_auth.BasicProxyAuth(password_manager, "mitmproxy") else: - authenticator = authentication.NullProxyAuth(None) + authenticator = http_auth.NullProxyAuth(None) return ProxyConfig( + app = options.app, certfile = options.cert, cacert = cacert, clientcerts = options.clientcerts, |