diff options
Diffstat (limited to 'libmproxy')
| -rw-r--r-- | libmproxy/protocol/http.py | 172 |
1 files changed, 124 insertions, 48 deletions
diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index e81c7640..3560f0bd 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -27,7 +27,8 @@ def get_line(fp): Get a line, possibly preceded by a blank. """ line = fp.readline() - if line == "\r\n" or line == "\n": # Possible leftover from previous message + if line == "\r\n" or line == "\n": + # Possible leftover from previous message line = fp.readline() if line == "": raise tcp.NetLibDisconnect() @@ -241,25 +242,47 @@ class HTTPRequest(HTTPMessage): is content associated, but not present. CONTENT_MISSING evaluates to False to make checking for the presence of content natural. - form_in: The request form which mitmproxy has received. The following values are possible: - - relative (GET /index.html, OPTIONS *) (covers origin form and asterisk form) - - absolute (GET http://example.com:80/index.html) - - authority-form (CONNECT example.com:443) - Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 + form_in: The request form which mitmproxy has received. The following + values are possible: - form_out: The request form which mitmproxy has send out to the destination + - relative (GET /index.html, OPTIONS *) (covers origin form and + asterisk form) + - absolute (GET http://example.com:80/index.html) + - authority-form (CONNECT example.com:443) + Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 + + form_out: The request form which mitmproxy has send out to the + destination timestamp_start: Timestamp indicating when request transmission started timestamp_end: Timestamp indicating when request transmission ended """ - def __init__(self, form_in, method, scheme, host, port, path, httpversion, headers, - content, timestamp_start=None, timestamp_end=None, form_out=None): + def __init__( + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + content, + timestamp_start=None, + timestamp_end=None, + form_out=None + ): assert isinstance(headers, ODictCaseless) or not headers - HTTPMessage.__init__(self, httpversion, headers, content, timestamp_start, - timestamp_end) - + HTTPMessage.__init__( + self, + httpversion, + headers, + content, + timestamp_start, + timestamp_end + ) self.form_in = form_in self.method = method self.scheme = scheme @@ -312,7 +335,8 @@ class HTTPRequest(HTTPMessage): request_line = get_line(rfile) - if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start + if hasattr(rfile, "first_byte_timestamp"): + # more accurate timestamp_start timestamp_start = rfile.first_byte_timestamp request_line_parts = http.parse_init(request_line) @@ -683,7 +707,9 @@ class HTTPResponse(HTTPMessage): return "<HTTPResponse: {code} {msg} ({contenttype}, {size})>".format( code=self.code, msg=self.msg, - contenttype=self.headers.get_first("content-type", "unknown content type"), + contenttype=self.headers.get_first( + "content-type", "unknown content type" + ), size=size ) @@ -704,7 +730,8 @@ class HTTPResponse(HTTPMessage): body_size_limit, include_body=include_body) - if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start + if hasattr(rfile, "first_byte_timestamp"): + # more accurate timestamp_start timestamp_start = rfile.first_byte_timestamp if include_body: @@ -745,7 +772,11 @@ class HTTPResponse(HTTPMessage): def _assemble_head(self, preserve_transfer_encoding=False): return '%s\r\n%s\r\n' % ( - self._assemble_first_line(), self._assemble_headers(preserve_transfer_encoding=preserve_transfer_encoding)) + self._assemble_first_line(), + self._assemble_headers( + preserve_transfer_encoding=preserve_transfer_encoding + ) + ) def assemble(self): """ @@ -755,7 +786,10 @@ class HTTPResponse(HTTPMessage): Raises an Exception if the request cannot be assembled. """ if self.content == CONTENT_MISSING: - raise proxy.ProxyError(502, "Cannot assemble flow with CONTENT_MISSING") + raise proxy.ProxyError( + 502, + "Cannot assemble flow with CONTENT_MISSING" + ) head = self._assemble_head() if self.content: return head + self.content @@ -822,8 +856,9 @@ class HTTPResponse(HTTPMessage): pairs = [pair.partition("=") for pair in header.split(';')] cookie_name = pairs[0][0] # the key of the first key/value pairs cookie_value = pairs[0][2] # the value of the first key/value pairs - cookie_parameters = {key.strip().lower(): value.strip() for key, sep, value in - pairs[1:]} + cookie_parameters = { + key.strip().lower(): value.strip() for key, sep, value in pairs[1:] + } cookies.append((cookie_name, (cookie_value, cookie_parameters))) return dict(cookies) @@ -856,7 +891,8 @@ class HTTPFlow(Flow): self.response = None """@type: HTTPResponse""" - self.intercepting = False # FIXME: Should that rather be an attribute of Flow? + # FIXME: Should that rather be an attribute of Flow? + self.intercepting = False _stateobject_attributes = Flow._stateobject_attributes.copy() _stateobject_attributes.update( @@ -944,7 +980,9 @@ class HTTPFlow(Flow): class HttpAuthenticationError(Exception): def __init__(self, auth_headers=None): - super(HttpAuthenticationError, self).__init__("Proxy Authentication Required") + super(HttpAuthenticationError, self).__init__( + "Proxy Authentication Required" + ) self.headers = auth_headers self.code = 407 @@ -1114,7 +1152,12 @@ class HTTPHandler(ProtocolHandler): def handle_server_reconnect(self, state): if state["state"] == "connect": - send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False) + send_connect_request( + self.c.server_conn, + state["host"], + state["port"], + update_state=False + ) else: # pragma: nocover raise RuntimeError("Unknown State: %s" % state["state"]) @@ -1138,11 +1181,11 @@ class HTTPHandler(ProtocolHandler): self.c.log(message_debug, level="debug") if flow: - # TODO: no flows without request or with both request and response at the moment. + # TODO: no flows without request or with both request and response + # at the moment. if flow.request and not flow.response: flow.error = Error(message or message_debug) self.c.channel.ask("error", flow) - try: code = getattr(error, "code", 502) headers = getattr(error, "headers", None) @@ -1156,12 +1199,22 @@ class HTTPHandler(ProtocolHandler): def send_error(self, code, message, headers): response = http_status.RESPONSES.get(code, "Unknown") - html_content = '<html><head>\n<title>%d %s</title>\n</head>\n<body>\n%s\n</body>\n</html>' % \ - (code, response, message) + html_content = """ + <html> + <head> + <title>%d %s</title> + </head> + <body %s</body> + </html> + """ % (code, response, message) self.c.client_conn.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response)) - self.c.client_conn.wfile.write("Server: %s\r\n" % self.c.config.server_version) + self.c.client_conn.wfile.write( + "Server: %s\r\n" % self.c.config.server_version + ) self.c.client_conn.wfile.write("Content-type: text/html\r\n") - self.c.client_conn.wfile.write("Content-Length: %d\r\n" % len(html_content)) + self.c.client_conn.wfile.write( + "Content-Length: %d\r\n" % len(html_content) + ) if headers: for key, value in headers.items(): self.c.client_conn.wfile.write("%s: %s\r\n" % (key, value)) @@ -1201,11 +1254,15 @@ class HTTPHandler(ProtocolHandler): # Now we can process the request. if request.form_in == "authority": if self.c.client_conn.ssl_established: - raise http.HttpError(400, "Must not CONNECT on already encrypted connection") + raise http.HttpError( + 400, + "Must not CONNECT on already encrypted connection" + ) if self.c.config.mode == "regular": self.c.set_server_address((request.host, request.port)) - flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow + # Update server_conn attribute on the flow + flow.server_conn = self.c.server_conn self.c.establish_server_connection() self.c.client_conn.send( 'HTTP/1.1 200 Connection established\r\n' + @@ -1217,7 +1274,9 @@ class HTTPHandler(ProtocolHandler): elif self.c.config.mode == "upstream": return None else: - pass # CONNECT should never occur if we don't expect absolute-form requests + # CONNECT should never occur if we don't expect absolute-form + # requests + pass elif request.form_in == self.expected_form_in: @@ -1225,61 +1284,77 @@ class HTTPHandler(ProtocolHandler): if request.form_in == "absolute": if request.scheme != "http": - raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme) + raise http.HttpError( + 400, + "Invalid request scheme: %s" % request.scheme + ) if self.c.config.mode == "regular": - # Update info so that an inline script sees the correct value at flow.server_conn + # Update info so that an inline script sees the correct + # value at flow.server_conn self.c.set_server_address((request.host, request.port)) flow.server_conn = self.c.server_conn return None - - raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" % - (self.expected_form_in, request.form_in)) + raise http.HttpError( + 400, "Invalid HTTP request form (expected: %s, got: %s)" % ( + self.expected_form_in, request.form_in + ) + ) def process_server_address(self, flow): # Depending on the proxy mode, server handling is entirely different - # We provide a mostly unified API to the user, which needs to be unfiddled here + # We provide a mostly unified API to the user, which needs to be + # unfiddled here # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 ) address = netlib.tcp.Address((flow.request.host, flow.request.port)) ssl = (flow.request.scheme == "https") if self.c.config.mode == "upstream": - - # The connection to the upstream proxy may have a state we may need to take into account. + # The connection to the upstream proxy may have a state we may need + # to take into account. connected_to = None for s in flow.server_conn.state: if s[0] == "http" and s[1]["state"] == "connect": connected_to = tcp.Address((s[1]["host"], s[1]["port"])) - # We need to reconnect if the current flow either requires a (possibly impossible) - # change to the connection state, e.g. the host has changed but we already CONNECTed somewhere else. + # We need to reconnect if the current flow either requires a + # (possibly impossible) change to the connection state, e.g. the + # host has changed but we already CONNECTed somewhere else. needs_server_change = ( ssl != self.c.server_conn.ssl_established or - (connected_to and address != connected_to) # HTTP proxying is "stateless", CONNECT isn't. + # HTTP proxying is "stateless", CONNECT isn't. + (connected_to and address != connected_to) ) if needs_server_change: # force create new connection to the proxy server to reset state self.live.change_server(self.c.server_conn.address, force=True) if ssl: - send_connect_request(self.c.server_conn, address.host, address.port) + send_connect_request( + self.c.server_conn, + address.host, + address.port + ) self.c.establish_ssl(server=True) else: - # If we're not in upstream mode, we just want to update the host and possibly establish TLS. - self.live.change_server(address, ssl=ssl) # this is a no op if the addresses match. + # If we're not in upstream mode, we just want to update the host and + # possibly establish TLS. This is a no op if the addresses match. + self.live.change_server(address, ssl=ssl) flow.server_conn = self.c.server_conn def send_response_to_client(self, flow): if not flow.response.stream: # no streaming: - # we already received the full response from the server and can send it to the client straight away. + # we already received the full response from the server and can send + # it to the client straight away. self.c.client_conn.send(flow.response.assemble()) else: # streaming: - # First send the headers and then transfer the response incrementally: + # First send the headers and then transfer the response + # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, @@ -1293,7 +1368,8 @@ class HTTPHandler(ProtocolHandler): def check_close_connection(self, flow): """ - Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so. + Checks if the connection should be closed depending on the HTTP + semantics. Returns True, if so. """ close_connection = ( http.connection_close(flow.request.httpversion, flow.request.headers) or |