diff options
Diffstat (limited to 'libmproxy/protocolold.py')
| -rw-r--r-- | libmproxy/protocolold.py | 391 |
1 files changed, 391 insertions, 0 deletions
diff --git a/libmproxy/protocolold.py b/libmproxy/protocolold.py new file mode 100644 index 00000000..9f98e37e --- /dev/null +++ b/libmproxy/protocolold.py @@ -0,0 +1,391 @@ +import libmproxy.utils, libmproxy.flow +from netlib import http, http_status, tcp +import netlib.utils +from netlib.odict import ODictCaseless +import select +from proxy import ProxyError, KILL + +LEGACY = True + + +""" +Minimalistic cleanroom reimplemementation of a couple of flow.* classes. Most functionality is missing, +but they demonstrate what needs to be added/changed to/within the existing classes. +""" + + +class Flow(object): + def __init__(self, conntype, client_conn, server_conn, error): + self.conntype = conntype + self.client_conn, self.server_conn = client_conn, server_conn + self.error = error + + +class HTTPFlow(Flow): + def __init__(self, client_conn, server_conn, error, request, response): + Flow.__init__(self, "http", client_conn, server_conn, error) + self.request, self.response = request, response + + +class HttpAuthenticationError(Exception): + def __init__(self, auth_headers=None): + self.auth_headers = auth_headers + + def __str__(self): + return "HttpAuthenticationError" + + +class HTTPMessage(object): + def _assemble_headers(self): + headers = self.headers.copy() + libmproxy.utils.del_all(headers, + ["proxy-connection", + "transfer-encoding"]) + if self.content: + headers["Content-Length"] = [str(len(self.content))] + elif 'Transfer-Encoding' in self.headers: # content-length for e.g. chuncked transfer-encoding with no content + headers["Content-Length"] = ["0"] + + return str(headers) + + +class HTTPResponse(HTTPMessage): + def __init__(self, httpversion, code, msg, headers, content, timestamp_start, timestamp_end): + self.httpversion = httpversion + self.code = code + self.msg = msg + self.headers = headers + self.content = content + self.timestamp_start = timestamp_start + self.timestamp_end = timestamp_end + + assert isinstance(headers, ODictCaseless) + + #FIXME: Compatibility Fix + @property + def request(self): + return False + + def _assemble_response_line(self): + return 'HTTP/%s.%s %s %s' % (self.httpversion[0], self.httpversion[1], self.code, self.msg) + + def _assemble(self): + response_line = self._assemble_response_line() + return '%s\r\n%s\r\n%s' % (response_line, self._assemble_headers(), self.content) + + @classmethod + def from_stream(cls, rfile, request_method, include_content=True, body_size_limit=None): + """ + Parse an HTTP response from a file stream + """ + if not include_content: + raise NotImplementedError + + httpversion, code, msg, headers, content = http.read_response( + rfile, + request_method, + body_size_limit) + timestamp_start = rfile.first_byte_timestamp + timestamp_end = libmproxy.utils.timestamp() + return HTTPResponse(httpversion, code, msg, headers, content, timestamp_start, timestamp_end) + + +class HTTPRequest(HTTPMessage): + def __init__(self, form_in, method, scheme, host, port, path, httpversion, headers, content, + timestamp_start, timestamp_end, form_out=None): + self.form_in = form_in + self.method = method + self.scheme = scheme + self.host = host + self.port = port + self.path = path + self.httpversion = httpversion + self.headers = headers + self.content = content + self.timestamp_start = timestamp_start + self.timestamp_end = timestamp_end + + self.form_out = form_out or self.form_in + assert isinstance(headers, ODictCaseless) + + #FIXME: Compatibility Fixes + def is_live(self): + return True + @property + def wfile(self): + import mock + return mock.Mock(side_effect=tcp.NetLibDisconnect) + + def _assemble_request_line(self, form=None): + form = form or self.form_out + request_line = None + if form == "asterisk" or form == "origin": + request_line = '%s %s HTTP/%s.%s' % (self.method, self.path, self.httpversion[0], self.httpversion[1]) + elif form == "authority": + request_line = '%s %s:%s HTTP/%s.%s' % (self.method, self.host, self.port, + self.httpversion[0], self.httpversion[1]) + elif form == "absolute": + request_line = '%s %s://%s:%s%s HTTP/%s.%s' % \ + (self.method, self.scheme, self.host, self.port, self.path, + self.httpversion[0], self.httpversion[1]) + else: + raise http.HttpError(400, "Invalid request form") + return request_line + + def _assemble(self): + request_line = self._assemble_request_line() + return '%s\r\n%s\r\n%s' % (request_line, self._assemble_headers(), self.content) + + @classmethod + def from_stream(cls, rfile, include_content=True, body_size_limit=None): + """ + Parse an HTTP request from a file stream + """ + httpversion, host, port, scheme, method, path, headers, content, timestamp_start, timestamp_end \ + = None, None, None, None, None, None, None, None, None, None + + request_line = HTTPHandler.get_line(rfile) + timestamp_start = rfile.first_byte_timestamp + + request_line_parts = http.parse_init(request_line) + if not request_line_parts: + raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + method, path, httpversion = request_line_parts + + if path == '*': + form_in = "asterisk" + elif path.startswith("/"): + form_in = "origin" + if not netlib.utils.isascii(path): + raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + elif method.upper() == 'CONNECT': + form_in = "authority" + r = http.parse_init_connect(request_line) + if not r: + raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + host, port, _ = r + path = None + else: + form_in = "absolute" + r = http.parse_init_proxy(request_line) + if not r: + raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + _, scheme, host, port, path, _ = r + + headers = http.read_headers(rfile) + if headers is None: + raise http.HttpError(400, "Invalid headers") + + if include_content: + content = http.read_http_body(rfile, headers, body_size_limit, True) + timestamp_end = libmproxy.utils.timestamp() + + return HTTPRequest(form_in, method, scheme, host, port, path, httpversion, headers, content, + timestamp_start, timestamp_end) + + +class HTTPHandler(ProtocolHandler): + def handle_messages(self): + while self.handle_flow(): + pass + self.c.close = True + + def get_response_from_server(self, request): + request_raw = request._assemble() + + for i in range(2): + try: + self.c.server_conn.wfile.write(request_raw) + self.c.server_conn.wfile.flush() + return HTTPResponse.from_stream(self.c.server_conn.rfile, request.method, + body_size_limit=self.c.config.body_size_limit) + except (tcp.NetLibDisconnect, http.HttpErrorConnClosed), v: + self.c.log("error in server communication: %s" % str(v)) + if i < 1: + # In any case, we try to reconnect at least once. + # This is necessary because it might be possible that we already initiated an upstream connection + # after clientconnect that has already been expired, e.g consider the following event log: + # > clientconnect (transparent mode destination known) + # > serverconnect + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + self.c.server_reconnect() + else: + raise v + + def handle_flow(self): + flow = HTTPFlow(self.c.client_conn, self.c.server_conn, None, None, None) + try: + flow.request = HTTPRequest.from_stream(self.c.client_conn.rfile, + body_size_limit=self.c.config.body_size_limit) + self.c.log("request", [flow.request._assemble_request_line(flow.request.form_in)]) + + request_reply = self.c.channel.ask("request" if LEGACY else "httprequest", + flow.request if LEGACY else flow) + if request_reply is None or request_reply == KILL: + return False + + if isinstance(request_reply, HTTPResponse) or (LEGACY and isinstance(request_reply, libmproxy.flow.Response)): + flow.response = request_reply + else: + self.process_request(flow.request) + flow.response = self.get_response_from_server(flow.request) + + self.c.log("response", [flow.response._assemble_response_line() if not LEGACY else flow.response._assemble().splitlines()[0]]) + response_reply = self.c.channel.ask("response" if LEGACY else "httpresponse", + flow.response if LEGACY else flow) + if response_reply is None or response_reply == KILL: + return False + + raw = flow.response._assemble() + self.c.client_conn.wfile.write(raw) + self.c.client_conn.wfile.flush() + flow.timestamp_end = libmproxy.utils.timestamp() + + if (http.connection_close(flow.request.httpversion, flow.request.headers) or + http.connection_close(flow.response.httpversion, flow.response.headers)): + return False + + if flow.request.form_in == "authority": + self.ssl_upgrade(flow.request) + return True + except (HttpAuthenticationError, http.HttpError, ProxyError, tcp.NetLibError), e: + self.handle_error(e, flow) + return False + + def handle_error(self, error, flow=None): + code, message, headers = None, None, None + if isinstance(error, HttpAuthenticationError): + code, message, headers = 407, "Proxy Authentication Required", error.auth_headers + elif isinstance(error, (http.HttpError, ProxyError)): + code, message = error.code, error.msg + elif isinstance(error, tcp.NetLibError): + code = 502 + message = error.message or error.__class__ + + if code: + err = "%s: %s" % (code, message) + else: + err = message + + self.c.log("error: %s" %err) + + if flow: + flow.error = libmproxy.flow.Error(False, err) + self.c.channel.ask("error" if LEGACY else "httperror", + flow.error if LEGACY else flow) + else: + pass # FIXME: Is there any use case for persisting errors that occur outside of flows? + + if code: + try: + self.send_error(code, message, headers) + except: + pass + + def send_error(self, code, message, headers): + response = http_status.RESPONSES.get(code, "Unknown") + html_content = '<html><head>\n<title>%d %s</title>\n</head>\n<body>\n%s\n</body>\n</html>' % \ + (code, response, message) + self.c.client_conn.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response)) + self.c.client_conn.wfile.write("Server: %s\r\n" % self.c.server_version) + self.c.client_conn.wfile.write("Content-type: text/html\r\n") + self.c.client_conn.wfile.write("Content-Length: %d\r\n" % len(html_content)) + if headers: + for key, value in headers.items(): + self.c.client_conn.wfile.write("%s: %s\r\n" % (key, value)) + self.c.client_conn.wfile.write("Connection: close\r\n") + self.c.client_conn.wfile.write("\r\n") + self.c.client_conn.wfile.write(html_content) + self.c.client_conn.wfile.flush() + + def ssl_upgrade(self, upstream_request=None): + """ + Upgrade the connection to SSL after an authority (CONNECT) request has been made. + If the authority request has been forwarded upstream (because we have another proxy server there), + money-patch the ConnectionHandler.server_reconnect function to resend the request on reconnect. + + This isn't particular beautiful code, but it isolates this rare edge-case from the + protocol-agnostic ConnectionHandler + """ + self.c.mode = "transparent" + self.c.determine_conntype() + self.c.establish_ssl(server=True, client=True) + + if upstream_request: + self.c.log("Hook reconnect function") + original_reconnect_func = self.c.server_reconnect + + def reconnect_http_proxy(): + self.c.log("Hooked reconnect function") + self.c.log("Hook: Run original redirect") + original_reconnect_func(no_ssl=True) + self.c.log("Hook: Write CONNECT request to upstream proxy", [upstream_request._assemble_request_line()]) + self.c.server_conn.wfile.write(upstream_request._assemble()) + self.c.server_conn.wfile.flush() + self.c.log("Hook: Read answer to CONNECT request from proxy") + resp = HTTPResponse.from_stream(self.c.server_conn.rfile, upstream_request.method) + if resp.code != 200: + raise ProxyError(resp.code, + "Cannot reestablish SSL connection with upstream proxy: \r\n" + str(resp.headers)) + self.c.log("Hook: Establish SSL with upstream proxy") + self.c.establish_ssl(server=True) + + self.c.server_reconnect = reconnect_http_proxy + + raise ConnectionTypeChange + + def process_request(self, request): + if self.c.mode == "regular": + self.authenticate(request) + if request.form_in == "authority" and self.c.client_conn.ssl_established: + raise http.HttpError(502, "Must not CONNECT on already encrypted connection") + + # If we have a CONNECT request, we might need to intercept + if request.form_in == "authority": + directly_addressed_at_mitmproxy = (self.c.mode == "regular") and not self.c.config.forward_proxy + if directly_addressed_at_mitmproxy: + self.c.establish_server_connection((request.host, request.port)) + self.c.client_conn.wfile.write( + 'HTTP/1.1 200 Connection established\r\n' + + ('Proxy-agent: %s\r\n' % self.c.server_version) + + '\r\n' + ) + self.c.client_conn.wfile.flush() + self.ssl_upgrade() # raises ConnectionTypeChange exception + + if self.c.mode == "regular": + if request.form_in == "authority": + pass + elif request.form_in == "absolute": + if request.scheme != "http": + raise http.HttpError(400, "Invalid Request") + if not self.c.config.forward_proxy: + request.form_out = "origin" + if ((not self.c.server_conn) or + (self.c.server_conn.address != (request.host, request.port))): + self.c.establish_server_connection((request.host, request.port)) + else: + raise http.HttpError(400, "Invalid Request") + + def authenticate(self, request): + if self.c.config.authenticator: + if self.c.config.authenticator.authenticate(request.headers): + self.c.config.authenticator.clean(request.headers) + else: + raise HttpAuthenticationError(self.c.config.authenticator.auth_challenge_headers()) + return request.headers + + @staticmethod + def get_line(fp): + """ + Get a line, possibly preceded by a blank. + """ + line = fp.readline() + if line == "\r\n" or line == "\n": # Possible leftover from previous message + line = fp.readline() + if line == "": + raise tcp.NetLibDisconnect + return line
\ No newline at end of file |