diff options
| -rw-r--r-- | mitmproxy/addons/dumper.py | 3 | ||||
| -rw-r--r-- | mitmproxy/flow.py | 5 | ||||
| -rw-r--r-- | mitmproxy/io_compat.py | 1 | ||||
| -rw-r--r-- | mitmproxy/proxy/protocol/http.py | 193 | ||||
| -rw-r--r-- | mitmproxy/proxy/protocol/http1.py | 2 | ||||
| -rw-r--r-- | mitmproxy/proxy/protocol/http2.py | 7 | ||||
| -rw-r--r-- | mitmproxy/tools/console/common.py | 9 |
7 files changed, 120 insertions, 100 deletions
diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index d690c000..fb92c629 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -116,7 +116,8 @@ class Dumper: else: client = "" - method = flow.request.method + pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else '' + method = flow.request.method + pushed method_color = dict( GET="green", DELETE="red" diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index ff7a2b4a..18395be3 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -7,7 +7,7 @@ from mitmproxy import stateobject from mitmproxy import connections from mitmproxy import version -from typing import Optional # noqa +from typing import Optional, Dict # noqa class Error(stateobject.StateObject): @@ -83,6 +83,7 @@ class Flow(stateobject.StateObject): self._backup = None # type: Optional[Flow] self.reply = None # type: Optional[controller.Reply] self.marked = False # type: bool + self.metadata = dict() # type: Dict[str, str] _stateobject_attributes = dict( id=str, @@ -92,6 +93,7 @@ class Flow(stateobject.StateObject): type=str, intercepted=bool, marked=bool, + metadata=dict, ) def get_state(self): @@ -120,6 +122,7 @@ class Flow(stateobject.StateObject): f.live = False f.client_conn = self.client_conn.copy() f.server_conn = self.server_conn.copy() + f.metadata = self.metadata.copy() if self.error: f.error = self.error.copy() diff --git a/mitmproxy/io_compat.py b/mitmproxy/io_compat.py index e1ca27b2..b1b5a296 100644 --- a/mitmproxy/io_compat.py +++ b/mitmproxy/io_compat.py @@ -69,6 +69,7 @@ def convert_018_019(data): data["client_conn"]["sni"] = None data["client_conn"]["cipher_name"] = None data["client_conn"]["tls_version"] = None + data["metadata"] = dict() return data diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index a47fb455..542f6a94 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -12,14 +12,14 @@ from mitmproxy.net import websockets class _HttpTransmissionLayer(base.Layer): - def read_request_headers(self): + def read_request_headers(self, flow): raise NotImplementedError() def read_request_body(self, request): raise NotImplementedError() - def read_request(self): - request = self.read_request_headers() + def read_request(self, f): + request = self.read_request_headers(f) request.data.content = b"".join( self.read_request_body(request) ) @@ -125,7 +125,7 @@ class HttpLayer(base.Layer): def __init__(self, ctx, mode): super().__init__(ctx) self.mode = mode - + self.flow = None # type: http.HTTPFlow self.__initial_server_conn = None "Contains the original destination in transparent mode, which needs to be restored" "if an inline script modified the target server for a single http request" @@ -140,105 +140,112 @@ class HttpLayer(base.Layer): self.__initial_server_tls = self.server_tls self.__initial_server_conn = self.server_conn while True: - f = http.HTTPFlow(self.client_conn, self.server_conn, live=self) - try: - request = self.get_request_from_client(f) - # Make sure that the incoming request matches our expectations - self.validate_request(request) - except exceptions.HttpReadDisconnect: - # don't throw an error for disconnects that happen before/between requests. + self.flow = http.HTTPFlow(self.client_conn, self.server_conn, live=self) + if not self._process_flow(self.flow): return - except exceptions.HttpException as e: - # We optimistically guess there might be an HTTP client on the - # other end - self.send_error_response(400, repr(e)) - raise exceptions.ProtocolException( - "HTTP protocol error in client request: {}".format(e) + + def _process_flow(self, f): + try: + request = self.get_request_from_client(f) + # Make sure that the incoming request matches our expectations + self.validate_request(request) + except exceptions.HttpReadDisconnect: + # don't throw an error for disconnects that happen before/between requests. + return False + except exceptions.HttpException as e: + # We optimistically guess there might be an HTTP client on the + # other end + self.send_error_response(400, repr(e)) + raise exceptions.ProtocolException( + "HTTP protocol error in client request: {}".format(e) + ) + + self.log("request", "debug", [repr(request)]) + + # Handle Proxy Authentication + # Proxy Authentication conceptually does not work in transparent mode. + # We catch this misconfiguration on startup. Here, we sort out requests + # after a successful CONNECT request (which do not need to be validated anymore) + if not (self.http_authenticated or self.authenticate(request)): + return False + + f.request = request + + try: + # Regular Proxy Mode: Handle CONNECT + if self.mode == "regular" and request.first_line_format == "authority": + self.handle_regular_mode_connect(request) + return False + except (exceptions.ProtocolException, exceptions.NetlibException) as e: + # HTTPS tasting means that ordinary errors like resolution and + # connection errors can happen here. + self.send_error_response(502, repr(e)) + f.error = flow.Error(str(e)) + self.channel.ask("error", f) + return False + + # update host header in reverse proxy mode + if self.config.options.mode == "reverse": + f.request.headers["Host"] = self.config.upstream_server.address.host + + # set upstream auth + if self.mode == "upstream" and self.config.upstream_auth is not None: + f.request.headers["Proxy-Authorization"] = self.config.upstream_auth + self.process_request_hook(f) + + try: + if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers): + # We only support RFC6455 with WebSockets version 13 + # allow inline scripts to manipulate the client handshake + self.channel.ask("websocket_handshake", f) + + if not f.response: + self.establish_server_connection( + f.request.host, + f.request.port, + f.request.scheme ) + self.get_response_from_server(f) + else: + # response was set by an inline script. + # we now need to emulate the responseheaders hook. + self.channel.ask("responseheaders", f) - self.log("request", "debug", [repr(request)]) + self.log("response", "debug", [repr(f.response)]) + self.channel.ask("response", f) + self.send_response_to_client(f) - # Handle Proxy Authentication - # Proxy Authentication conceptually does not work in transparent mode. - # We catch this misconfiguration on startup. Here, we sort out requests - # after a successful CONNECT request (which do not need to be validated anymore) - if not (self.http_authenticated or self.authenticate(request)): - return + if self.check_close_connection(f): + return False + + # Handle 101 Switching Protocols + if f.response.status_code == 101: + self.handle_101_switching_protocols(f) + return False # should never be reached - f.request = request - - try: - # Regular Proxy Mode: Handle CONNECT - if self.mode == "regular" and request.first_line_format == "authority": - self.handle_regular_mode_connect(request) - return - except (exceptions.ProtocolException, exceptions.NetlibException) as e: - # HTTPS tasting means that ordinary errors like resolution and - # connection errors can happen here. - self.send_error_response(502, repr(e)) + # Upstream Proxy Mode: Handle CONNECT + if f.request.first_line_format == "authority" and f.response.status_code == 200: + self.handle_upstream_mode_connect(f.request.copy()) + return False + + except (exceptions.ProtocolException, exceptions.NetlibException) as e: + self.send_error_response(502, repr(e)) + if not f.response: f.error = flow.Error(str(e)) self.channel.ask("error", f) - return + return False + else: + raise exceptions.ProtocolException( + "Error in HTTP connection: %s" % repr(e) + ) + finally: + if f: + f.live = False - # update host header in reverse proxy mode - if self.config.options.mode == "reverse": - f.request.headers["Host"] = self.config.upstream_server.address.host - - # set upstream auth - if self.mode == "upstream" and self.config.upstream_auth is not None: - f.request.headers["Proxy-Authorization"] = self.config.upstream_auth - self.process_request_hook(f) - - try: - if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers): - # We only support RFC6455 with WebSockets version 13 - # allow inline scripts to manipulate the client handshake - self.channel.ask("websocket_handshake", f) - - if not f.response: - self.establish_server_connection( - f.request.host, - f.request.port, - f.request.scheme - ) - self.get_response_from_server(f) - else: - # response was set by an inline script. - # we now need to emulate the responseheaders hook. - self.channel.ask("responseheaders", f) - - self.log("response", "debug", [repr(f.response)]) - self.channel.ask("response", f) - self.send_response_to_client(f) - - if self.check_close_connection(f): - return - - # Handle 101 Switching Protocols - if f.response.status_code == 101: - return self.handle_101_switching_protocols(f) - - # Upstream Proxy Mode: Handle CONNECT - if f.request.first_line_format == "authority" and f.response.status_code == 200: - self.handle_upstream_mode_connect(f.request.copy()) - return - - except (exceptions.ProtocolException, exceptions.NetlibException) as e: - self.send_error_response(502, repr(e)) - if not f.response: - f.error = flow.Error(str(e)) - self.channel.ask("error", f) - return - else: - raise exceptions.ProtocolException( - "Error in HTTP connection: %s" % repr(e) - ) - finally: - if f: - f.live = False + return True def get_request_from_client(self, f): - request = self.read_request() + request = self.read_request(f) f.request = request self.channel.ask("requestheaders", f) if request.headers.get("expect", "").lower() == "100-continue": diff --git a/mitmproxy/proxy/protocol/http1.py b/mitmproxy/proxy/protocol/http1.py index 713c48a7..b1fd0ecd 100644 --- a/mitmproxy/proxy/protocol/http1.py +++ b/mitmproxy/proxy/protocol/http1.py @@ -9,7 +9,7 @@ class Http1Layer(httpbase._HttpTransmissionLayer): super().__init__(ctx) self.mode = mode - def read_request_headers(self): + def read_request_headers(self, flow): return http.HTTPRequest.wrap( http1.read_request_head(self.client_conn.rfile) ) diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index d7d3d651..f440d100 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -243,6 +243,7 @@ class Http2Layer(base.Layer): def _handle_pushed_stream_received(self, event, h2_connection): # pushed stream ids should be unique and not dependent on race conditions # only the parent stream id must be looked up first + parent_eid = self.server_to_client_stream_ids[event.parent_stream_id] with self.client_conn.h2.lock: self.client_conn.h2.push_stream(parent_eid, event.pushed_stream_id, event.headers) @@ -455,9 +456,13 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr raise exceptions.Http2ZombieException("Connection already dead") @detect_zombie_stream - def read_request_headers(self): + def read_request_headers(self, flow): self.request_arrived.wait() self.raise_zombie() + + if self.pushed: + flow.metadata['h2-pushed-stream'] = True + first_line_format, method, scheme, host, port, path = http2.parse_headers(self.request_headers) return http.HTTPRequest( first_line_format, diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 78860702..08bf0b67 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -327,7 +327,7 @@ def export_to_clip_or_file(key, scope, flow, writer): @lru_cache(maxsize=800) -def raw_format_flow(f): +def raw_format_flow(f, flow): f = dict(f) pile = [] req = [] @@ -346,7 +346,9 @@ def raw_format_flow(f): if f["req_is_replay"]: req.append(fcol(SYMBOL_REPLAY, "replay")) - req.append(fcol(f["req_method"], "method")) + + pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in flow.metadata else '' + req.append(fcol(f["req_method"] + pushed, "method")) preamble = sum(i[1] for i in req) + len(req) - 1 @@ -451,10 +453,11 @@ def format_flow(f, focus, extended=False, hostheader=False, max_url_len=False): resp_clen = contentdesc, roundtrip = roundtrip, )) + t = f.response.headers.get("content-type") if t: d["resp_ctype"] = t.split(";")[0] else: d["resp_ctype"] = "" - return raw_format_flow(tuple(sorted(d.items()))) + return raw_format_flow(tuple(sorted(d.items())), f) |