diff options
Diffstat (limited to 'libmproxy')
| -rw-r--r-- | libmproxy/flow.py | 1 | ||||
| -rw-r--r-- | libmproxy/protocol.py | 95 | ||||
| -rw-r--r-- | libmproxy/proxy.py | 520 |
3 files changed, 214 insertions, 402 deletions
diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 32306513..5e3ad8e9 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -854,7 +854,6 @@ class ClientConnect(StateObject): """ self.address = address self.close = False - self.requestcount = 0 self.error = None def __str__(self): diff --git a/libmproxy/protocol.py b/libmproxy/protocol.py new file mode 100644 index 00000000..cd9b4ce5 --- /dev/null +++ b/libmproxy/protocol.py @@ -0,0 +1,95 @@ +from libmproxy.proxy import ProxyError, ConnectionHandler +from netlib import http + + +def handle_messages(conntype, connection_handler): + handler = None + if conntype == "http": + handler = HTTPHandler(connection_handler) + else: + raise NotImplementedError + + return handler.handle_messages() + + +class ConnectionTypeChange(Exception): + pass + + +class ProtocolHandler(object): + def __init__(self, c): + self.c = c + + +class HTTPHandler(ProtocolHandler): + + def handle_messages(self): + while self.handle_request(): + pass + self.c.close = True + + def handle_request(self): + request = self.read_request() + if request is None: + return + raise NotImplementedError + + def read_request(self): + self.c.client_conn.rfile.reset_timestamps() + + request_line = self.get_line(self.c.client_conn.rfile) + method, path, httpversion = http.parse_init(request_line) + headers = self.read_headers(authenticate=True) + + if self.c.mode == "regular": + if method == "CONNECT": + r = http.parse_init_connect(request_line) + if not r: + raise ProxyError(400, "Bad HTTP request line: %s"%repr(request_line)) + host, port, _ = r + if self.c.config.forward_proxy: + #FIXME: Treat as request, no custom handling + self.c.server_conn.wfile.write(request_line) + for key, value in headers.items(): + self.c.server_conn.wfile.write("%s: %s\r\n"%(key, value)) + self.c.server_conn.wfile.write("\r\n") + else: + self.c.server_address = (host, port) + self.c.establish_server_connection() + + self.c.handle_ssl() + self.c.determine_conntype("transparent", host, port) + raise ConnectionTypeChange + else: + r = http.parse_init_proxy(request_line) + if not r: + raise ProxyError(400, "Bad HTTP request line: %s"%repr(request_line)) + method, scheme, host, port, path, httpversion = r + if not self.c.config.forward_proxy: + if (not self.c.server_conn) or (self.c.server_address != (host, port)): + self.c.server_address = (host, port) + self.c.establish_server_connection() + + def get_line(self, fp): + """ + Get a line, possibly preceded by a blank. + """ + line = fp.readline() + if line == "\r\n" or line == "\n": # Possible leftover from previous message + line = fp.readline() + return line + + def read_headers(self, authenticate=False): + headers = http.read_headers(self.c.client_conn.rfile) + if headers is None: + raise ProxyError(400, "Invalid headers") + if authenticate and self.c.config.authenticator: + if self.c.config.authenticator.authenticate(headers): + self.c.config.authenticator.clean(headers) + else: + raise ProxyError( + 407, + "Proxy Authentication Required", + self.c.config.authenticator.auth_challenge_headers() + ) + return headers
\ No newline at end of file diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py index 15f75b8d..5ac40e92 100644 --- a/libmproxy/proxy.py +++ b/libmproxy/proxy.py @@ -3,7 +3,7 @@ import shutil, tempfile, threading import SocketServer from OpenSSL import SSL from netlib import odict, tcp, http, certutils, http_status, http_auth -import utils, flow, version, platform, controller +import utils, flow, version, platform, controller, protocol TRANSPARENT_SSL_PORTS = [443, 8443] @@ -19,11 +19,6 @@ class ProxyError(Exception): return "ProxyError(%s, %s)"%(self.code, self.msg) -class Log: - def __init__(self, msg): - self.msg = msg - - class ProxyConfig: def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, forward_proxy=None, transparent_proxy=None, authenticator=None): self.certfile = certfile @@ -39,31 +34,31 @@ class ProxyConfig: class ServerConnection(tcp.TCPClient): - def __init__(self, config, scheme, host, port, sni): + def __init__(self, config, host, port, sni): tcp.TCPClient.__init__(self, host, port) self.config = config - self.scheme, self.sni = scheme, sni - self.requestcount = 0 + self.sni = sni self.tcp_setup_timestamp = None self.ssl_setup_timestamp = None def connect(self): tcp.TCPClient.connect(self) self.tcp_setup_timestamp = time.time() - if self.scheme == "https": - clientcert = None - if self.config.clientcerts: - path = os.path.join(self.config.clientcerts, self.host.encode("idna")) + ".pem" - if os.path.exists(path): - clientcert = path - try: - self.convert_to_ssl(cert=clientcert, sni=self.sni) - self.ssl_setup_timestamp = time.time() - except tcp.NetLibError, v: - raise ProxyError(400, str(v)) + + def establish_ssl(self): + clientcert = None + if self.config.clientcerts: + path = os.path.join(self.config.clientcerts, self.host.encode("idna")) + ".pem" + if os.path.exists(path): + clientcert = path + try: + self.convert_to_ssl(cert=clientcert, sni=self.sni) + self.ssl_setup_timestamp = time.time() + except tcp.NetLibError, v: + raise ProxyError(400, str(v)) def send(self, request): - self.requestcount += 1 + print "deprecated" d = request._assemble() if not d: raise ProxyError(502, "Cannot transmit an incomplete request.") @@ -104,420 +99,143 @@ class RequestReplayThread(threading.Thread): err = flow.Error(self.flow.request, str(v)) self.channel.ask("error", err) - -class HandleSNI: - def __init__(self, handler, client_conn, host, port, cert, key): - self.handler, self.client_conn, self.host, self.port = handler, client_conn, host, port - self.cert, self.key = cert, key - - def __call__(self, connection): - try: - sn = connection.get_servername() - if sn: - self.handler.get_server_connection(self.client_conn, "https", self.host, self.port, sn) - new_context = SSL.Context(SSL.TLSv1_METHOD) - new_context.use_privatekey_file(self.key) - new_context.use_certificate(self.cert.x509) - connection.set_context(new_context) - self.handler.sni = sn.decode("utf8").encode("idna") - # An unhandled exception in this method will core dump PyOpenSSL, so - # make dang sure it doesn't happen. - except Exception, e: # pragma: no cover - pass - - -class ProxyHandler(tcp.BaseHandler): - def __init__(self, config, connection, client_address, server, channel, server_version): - self.channel, self.server_version = channel, server_version +class ConnectionHandler: + def __init__(self, config, client_connection, client_address, server, channel, server_version): self.config = config - self.proxy_connect_state = None - self.sni = None - self.server_conn = None - tcp.BaseHandler.__init__(self, connection, client_address, server) - - def get_server_connection(self, cc, scheme, host, port, sni, request=None): - """ - When SNI is in play, this means we have an SSL-encrypted - connection, which means that the entire handler is dedicated to a - single server connection - no multiplexing. If this assumption ever - breaks, we'll have to do something different with the SNI host - variable on the handler object. - - `conn_info` holds the initial connection's parameters, as the - hook might change them. Also, the hook might require an initial - request to figure out connection settings; in this case it can - set require_request, which will cause the connection to be - re-opened after the client's request arrives. - """ - sc = self.server_conn - if not sni: - sni = host - conn_info = (scheme, host, port, sni) - if sc and (conn_info != sc.conn_info or (request and sc.require_request)): - sc.terminate() - self.server_conn = None - self.log( - cc, - "switching connection", [ - "%s://%s:%s (sni=%s) -> %s://%s:%s (sni=%s)"%( - scheme, host, port, sni, - sc.scheme, sc.host, sc.port, sc.sni - ) - ] - ) - if not self.server_conn: - try: - self.server_conn = ServerConnection(self.config, scheme, host, port, sni) + self.client_address, self.client_conn = client_address, tcp.BaseHandler(client_connection) + self.server_address, self.server_conn = None, None + self.channel, self.server_version = channel, server_version - # Additional attributes, used if the server_connect hook - # needs to change parameters - self.server_conn.request = request - self.server_conn.require_request = False + self.conntype = None + self.sni = None - self.server_conn.conn_info = conn_info - self.channel.ask("serverconnect", self.server_conn) - self.server_conn.connect() - except tcp.NetLibError, v: - raise ProxyError(502, v) - return self.server_conn + self.mode = "regular" + if self.config.reverse_proxy: + self.mode = "reverse" + if self.config.transparent_proxy: + self.mode = "transparent" def del_server_connection(self): if self.server_conn: self.server_conn.terminate() + self.channel.tell("serverdisconnect", self) self.server_conn = None + self.sni = None def handle(self): - cc = flow.ClientConnect(self.client_address) - self.log(cc, "connect") - self.channel.ask("clientconnect", cc) - while self.handle_request(cc) and not cc.close: - pass - cc.close = True + self.log("connect") + self.channel.ask("clientconnect", self) + + # Can we already identify the target server and connect to it? + if self.config.forward_proxy: + self.server_address = self.config.forward_proxy[1:] + else: + if self.config.reverse_proxy: + self.server_address = self.config.reverse_proxy[1:] + elif self.config.transparent_proxy: + self.server_address = self.config.transparent_proxy["resolver"].original_addr(self.connection) + if not self.server_address: + raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") + self.log("transparent to %s:%s"%self.server_address) + + if self.server_address: + self.establish_server_connection() + self.handle_ssl() + + self.determine_conntype(self.mode, *self.server_address) + + while not self.close: + try: + protocol.handle_messages(self.conntype, self) + except protocol.ConnectionTypeChange: + continue + self.del_server_connection() - cd = flow.ClientDisconnect(cc) - self.log( - cc, "disconnect", - [ - "handled %s requests"%cc.requestcount] - ) - self.channel.tell("clientdisconnect", cd) + self.log("disconnect") + self.channel.tell("clientdisconnect", self) - def handle_request(self, cc): - try: - request, err = None, None - request = self.read_request(cc) - if request is None: - return - cc.requestcount += 1 - - request_reply = self.channel.ask("request", request) - if request_reply is None or request_reply == KILL: - return - elif isinstance(request_reply, flow.Response): - request = False - response = request_reply - response_reply = self.channel.ask("response", response) - else: - request = request_reply - if self.config.reverse_proxy: - scheme, host, port = self.config.reverse_proxy - elif self.config.forward_proxy: - scheme, host, port = self.config.forward_proxy - else: - scheme, host, port = request.scheme, request.host, request.port - - # If we've already pumped a request over this connection, - # it's possible that the server has timed out. If this is - # the case, we want to reconnect without sending an error - # to the client. - while 1: - sc = self.get_server_connection(cc, scheme, host, port, self.sni, request=request) - sc.send(request) - if sc.requestcount == 1: # add timestamps only for first request (others are not directly affected) - request.tcp_setup_timestamp = sc.tcp_setup_timestamp - request.ssl_setup_timestamp = sc.ssl_setup_timestamp - sc.rfile.reset_timestamps() - try: - tsstart = utils.timestamp() - peername = sc.connection.getpeername() - if peername: - request.ip = peername[0] - httpversion, code, msg, headers, content = http.read_response( - sc.rfile, - request.method, - self.config.body_size_limit - ) - except http.HttpErrorConnClosed, v: - self.del_server_connection() - if sc.requestcount > 1: - continue - else: - raise - except http.HttpError, v: - raise ProxyError(502, "Invalid server response.") - else: - break - - response = flow.Response( - request, httpversion, code, msg, headers, content, sc.cert, - sc.rfile.first_byte_timestamp - ) - response_reply = self.channel.ask("response", response) - # Not replying to the server invalidates the server - # connection, so we terminate. - if response_reply == KILL: - sc.terminate() - - if response_reply == KILL: - return - else: - response = response_reply - self.send_response(response) - if request and http.connection_close(request.httpversion, request.headers): - return - # We could keep the client connection when the server - # connection needs to go away. However, we want to mimic - # behaviour as closely as possible to the client, so we - # disconnect. - if http.connection_close(response.httpversion, response.headers): - return - except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e: - if hasattr(e, "code"): - cc.error = "%s: %s"%(e.code, e.msg) - else: - cc.error = str(e) - - if request: - err = flow.Error(request, cc.error) - self.channel.ask("error", err) - self.log( - cc, cc.error, - ["url: %s"%request.get_url()] - ) - else: - self.log(cc, cc.error) - if isinstance(e, ProxyError): - self.send_error(e.code, e.msg, e.headers) + def determine_conntype(self, mode, host, port): + #TODO: Add ruleset to select correct protocol depending on mode/target port etc. + self.conntype = "http" + + def establish_server_connection(self): + """ + Establishes a new server connection to self.server_address. + If there is already an existing server connection, it will be killed. + """ + self.del_server_connection() + self.server_conn = ServerConnection(self.config, *self.server_address, self.sni) + self.channel.tell("serverconnect", self) + + def handle_ssl(self): + if self.config.transparent_proxy: + client_ssl, server_ssl = (self.server_address[1] in self.config.transparent_proxy["sslports"]) + elif self.config.reverse_proxy: + client_ssl, server_ssl = (self.config.reverse_proxy[0] == "https") + # TODO: Make protocol generic (as with transparent proxies) + # TODO: Add SSL-terminating capatbility (SSL -> mitmproxy -> plain and vice versa) else: - return True + client_ssl, server_ssl = True # In regular mode, this function will only be called on HTTP CONNECT + + # TODO: Implement SSL pass-through handling and change conntype + + if server_ssl and not self.server_conn.ssl_established: + self.server_conn.establish_ssl() + if client_ssl and not self.client_conn.ssl_established: + dummycert = self.find_cert() + self.client_conn.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=self.handle_sni) - def log(self, cc, msg, subs=()): + def log(self, msg, subs=()): msg = [ - "%s:%s: "%cc.address + msg + "%s:%s: "%(self.client_address, msg) ] for i in subs: msg.append(" -> "+i) msg = "\n".join(msg) - l = Log(msg) - self.channel.tell("log", l) + self.channel.tell("log", msg) - def find_cert(self, cc, host, port, sni): + def find_cert(self): if self.config.certfile: with open(self.config.certfile, "rb") as f: return certutils.SSLCert.from_pem(f.read()) else: + host = self.server_address[0] sans = [] - if not self.config.no_upstream_cert: - conn = self.get_server_connection(cc, "https", host, port, sni) - sans = conn.cert.altnames - if conn.cert.cn: - host = conn.cert.cn.decode("utf8").encode("idna") + if not self.config.no_upstream_cert or not self.server_conn.ssl_established: + upstream_cert = self.server_conn.cert + if upstream_cert.cn: + host = upstream_cert.cn.decode("utf8").encode("idna") + sans = upstream_cert.altnames + ret = self.config.certstore.get_cert(host, sans, self.config.cacert) if not ret: raise ProxyError(502, "Unable to generate dummy cert.") return ret - def establish_ssl(self, client_conn, host, port): - dummycert = self.find_cert(client_conn, host, port, host) - sni = HandleSNI( - self, client_conn, host, port, - dummycert, self.config.certfile or self.config.cacert - ) - try: - self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni) - except tcp.NetLibError, v: - raise ProxyError(400, str(v)) - - def get_line(self, fp): - """ - Get a line, possibly preceded by a blank. - """ - line = fp.readline() - if line == "\r\n" or line == "\n": # Possible leftover from previous message - line = fp.readline() - return line - - def read_request(self, client_conn): - self.rfile.reset_timestamps() - if self.config.transparent_proxy: - return self.read_request_transparent(client_conn) - elif self.config.reverse_proxy: - return self.read_request_reverse(client_conn) - else: - return self.read_request_proxy(client_conn) - - def read_request_transparent(self, client_conn): - orig = self.config.transparent_proxy["resolver"].original_addr(self.connection) - if not orig: - raise ProxyError(502, "Transparent mode failure: could not resolve original destination.") - self.log(client_conn, "transparent to %s:%s"%orig) - - host, port = orig - if port in self.config.transparent_proxy["sslports"]: - scheme = "https" - else: - scheme = "http" - - return self._read_request_origin_form(client_conn, scheme, host, port) - - def read_request_reverse(self, client_conn): - scheme, host, port = self.config.reverse_proxy - return self._read_request_origin_form(client_conn, scheme, host, port) - - def read_request_proxy(self, client_conn): - # Check for a CONNECT command. - if not self.proxy_connect_state: - line = self.get_line(self.rfile) - if line == "": - return None - self.proxy_connect_state = self._read_request_authority_form(line) - - # Check for an actual request - if self.proxy_connect_state: - host, port, _ = self.proxy_connect_state - return self._read_request_origin_form(client_conn, "https", host, port) - else: - # noinspection PyUnboundLocalVariable - return self._read_request_absolute_form(client_conn, line) - - def _read_request_authority_form(self, line): + def handle_sni(self, connection): """ - The authority-form of request-target is only used for CONNECT requests. - The CONNECT method is used to request a tunnel to the destination server. - This function sends a "200 Connection established" response to the client - and returns the host information that can be used to process further requests in origin-form. - An example authority-form request line would be: - CONNECT www.example.com:80 HTTP/1.1 + This callback gets called during the SSL handshake with the client. + The client has just sent the Sever Name Indication (SNI). We now connect upstream to + figure out which certificate needs to be served. """ - connparts = http.parse_init_connect(line) - if connparts: - self.read_headers(authenticate=True) - # respond according to http://tools.ietf.org/html/draft-luotonen-web-proxy-tunneling-01 section 3.2 - self.wfile.write( - 'HTTP/1.1 200 Connection established\r\n' + - ('Proxy-agent: %s\r\n'%self.server_version) + - '\r\n' - ) - self.wfile.flush() - return connparts - - def _read_request_absolute_form(self, client_conn, line): - """ - When making a request to a proxy (other than CONNECT or OPTIONS), - a client must send the target uri in absolute-form. - An example absolute-form request line would be: - GET http://www.example.com/foo.html HTTP/1.1 - """ - r = http.parse_init_proxy(line) - if not r: - raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) - method, scheme, host, port, path, httpversion = r - headers = self.read_headers(authenticate=True) - self.handle_expect_header(headers, httpversion) - content = http.read_http_body( - self.rfile, headers, self.config.body_size_limit, True - ) - r = flow.Request( - client_conn, httpversion, host, port, scheme, method, path, headers, content, - self.rfile.first_byte_timestamp, utils.timestamp() - ) - r.set_live(self.rfile, self.wfile) - return r - - def _read_request_origin_form(self, client_conn, scheme, host, port): - """ - Read a HTTP request with regular (origin-form) request line. - An example origin-form request line would be: - GET /foo.html HTTP/1.1 - - The request destination is already known from one of the following sources: - 1) transparent proxy: destination provided by platform resolver - 2) reverse proxy: fixed destination - 3) regular proxy: known from CONNECT command. - """ - if scheme.lower() == "https" and not self.ssl_established: - self.establish_ssl(client_conn, host, port) - - line = self.get_line(self.rfile) - if line == "": - return None - - r = http.parse_init_http(line) - if not r: - raise ProxyError(400, "Bad HTTP request line: %s"%repr(line)) - method, path, httpversion = r - headers = self.read_headers(authenticate=False) - self.handle_expect_header(headers, httpversion) - content = http.read_http_body( - self.rfile, headers, self.config.body_size_limit, True - ) - r = flow.Request( - client_conn, httpversion, host, port, scheme, method, path, headers, content, - self.rfile.first_byte_timestamp, utils.timestamp() - ) - r.set_live(self.rfile, self.wfile) - return r - - def handle_expect_header(self, headers, httpversion): - if "expect" in headers: - if "100-continue" in headers['expect'] and httpversion >= (1, 1): - #FIXME: Check if content-length is over limit - self.wfile.write('HTTP/1.1 100 Continue\r\n' - '\r\n') - del headers['expect'] - - def read_headers(self, authenticate=False): - headers = http.read_headers(self.rfile) - if headers is None: - raise ProxyError(400, "Invalid headers") - if authenticate and self.config.authenticator: - if self.config.authenticator.authenticate(headers): - self.config.authenticator.clean(headers) - else: - raise ProxyError( - 407, - "Proxy Authentication Required", - self.config.authenticator.auth_challenge_headers() - ) - return headers - - def send_response(self, response): - d = response._assemble() - if not d: - raise ProxyError(502, "Cannot transmit an incomplete response.") - self.wfile.write(d) - self.wfile.flush() - - def send_error(self, code, body, headers): try: - response = http_status.RESPONSES.get(code, "Unknown") - html_content = '<html><head>\n<title>%d %s</title>\n</head>\n<body>\n%s\n</body>\n</html>'%(code, response, body) - self.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response)) - self.wfile.write("Server: %s\r\n"%self.server_version) - self.wfile.write("Content-type: text/html\r\n") - self.wfile.write("Content-Length: %d\r\n"%len(html_content)) - if headers: - for key, value in headers.items(): - self.wfile.write("%s: %s\r\n"%(key, value)) - self.wfile.write("Connection: close\r\n") - self.wfile.write("\r\n") - self.wfile.write(html_content) - self.wfile.flush() - except: + sn = connection.get_servername() + if sn and sn != self.sni: + self.sni = sn.decode("utf8").encode("idna") + self.establish_server_connection() # reconnect to upstream server with SNI + self.handle_ssl() # establish SSL with upstream + # Now, change client context to reflect changed certificate: + new_context = SSL.Context(SSL.TLSv1_METHOD) + new_context.use_privatekey_file(self.config.certfile or self.config.cacert) + dummycert = self.find_cert() + new_context.use_certificate(dummycert.x509) + connection.set_context(new_context) + # An unhandled exception in this method will core dump PyOpenSSL, so + # make dang sure it doesn't happen. + except Exception, e: # pragma: no cover pass - class ProxyServerError(Exception): pass @@ -543,8 +261,8 @@ class ProxyServer(tcp.TCPServer): def set_channel(self, channel): self.channel = channel - def handle_connection(self, request, client_address): - h = ProxyHandler(self.config, request, client_address, self, self.channel, self.server_version) + def handle_client_connection(self, conn, client_address): + h = ConnectionHandler(self.config, conn, client_address, self, self.channel, self.server_version) h.handle() h.finish() |