diff options
| -rw-r--r-- | mitmproxy/protocol/http2.py | 2 | ||||
| -rw-r--r-- | mitmproxy/web/app.py | 65 | ||||
| -rw-r--r-- | mitmproxy/web/master.py | 4 | ||||
| -rw-r--r-- | netlib/http/headers.py | 9 | ||||
| -rw-r--r-- | netlib/http/http2/utils.py | 6 | ||||
| -rw-r--r-- | netlib/multidict.py | 24 | ||||
| -rw-r--r-- | test/mitmproxy/test_web_master.py | 2 | ||||
| -rw-r--r-- | tox.ini | 2 |
8 files changed, 68 insertions, 46 deletions
diff --git a/mitmproxy/protocol/http2.py b/mitmproxy/protocol/http2.py index 9515eef9..b6623aa3 100644 --- a/mitmproxy/protocol/http2.py +++ b/mitmproxy/protocol/http2.py @@ -448,7 +448,7 @@ class Http2SingleStreamLayer(http._HttpTransmissionLayer, basethread.BaseThread) status_code = int(self.response_headers.get(':status', 502)) headers = self.response_headers.copy() - headers.clear(":status") + headers.pop(":status", None) return models.HTTPResponse( http_version=b"HTTP/2.0", diff --git a/mitmproxy/web/app.py b/mitmproxy/web/app.py index a2798472..f9d0dca6 100644 --- a/mitmproxy/web/app.py +++ b/mitmproxy/web/app.py @@ -12,34 +12,57 @@ from io import BytesIO from mitmproxy.flow import FlowWriter, FlowReader from mitmproxy import filt +from mitmproxy import models from netlib import version -def _strip_content(flow_state): +def convert_flow_to_json_dict(flow): + # type: (models.Flow) -> dict """ Remove flow message content and cert to save transmission space. Args: - flow_state: The original flow state. Will be left unmodified + flow: The original flow. """ - for attr in ("request", "response"): - if attr in flow_state: - message = flow_state[attr] - if message is None: - continue - if message["content"]: - message["contentLength"] = len(message["content"]) - else: - message["contentLength"] = None - del message["content"] - - if "backup" in flow_state: - del flow_state["backup"] - flow_state["modified"] = True - - flow_state.get("server_conn", {}).pop("cert", None) - - return flow_state + f = { + "id": flow.id, + "intercepted": flow.intercepted, + "client_conn": flow.client_conn.get_state(), + "server_conn": flow.server_conn.get_state(), + "type": flow.type + } + if flow.error: + f["error"] = flow.error.get_state() + + if isinstance(flow, models.HTTPFlow): + if flow.request: + f["request"] = { + "method": flow.request.method, + "scheme": flow.request.scheme, + "host": flow.request.host, + "port": flow.request.port, + "path": flow.request.path, + "http_version": flow.request.http_version, + "headers": tuple(flow.request.headers.items(True)), + "contentLength": len(flow.request.content) if flow.request.content is not None else None, + "timestamp_start": flow.request.timestamp_start, + "timestamp_end": flow.request.timestamp_end, + "is_replay": flow.request.is_replay, + } + if flow.response: + f["response"] = { + "http_version": flow.response.http_version, + "status_code": flow.response.status_code, + "reason": flow.response.reason, + "headers": tuple(flow.response.headers.items(True)), + "contentLength": len(flow.response.content) if flow.response.content is not None else None, + "timestamp_start": flow.response.timestamp_start, + "timestamp_end": flow.response.timestamp_end, + "is_replay": flow.response.is_replay, + } + f.get("server_conn", {}).pop("cert", None) + + return f class APIError(tornado.web.HTTPError): @@ -158,7 +181,7 @@ class Flows(RequestHandler): def get(self): self.write(dict( - data=[_strip_content(f.get_state()) for f in self.state.flows] + data=[convert_flow_to_json_dict(f) for f in self.state.flows] )) diff --git a/mitmproxy/web/master.py b/mitmproxy/web/master.py index d034a24b..737bb95f 100644 --- a/mitmproxy/web/master.py +++ b/mitmproxy/web/master.py @@ -27,7 +27,7 @@ class WebFlowView(flow.FlowView): app.ClientConnection.broadcast( type="UPDATE_FLOWS", cmd="add", - data=app._strip_content(f.get_state()) + data=app.convert_flow_to_json_dict(f) ) def _update(self, f): @@ -35,7 +35,7 @@ class WebFlowView(flow.FlowView): app.ClientConnection.broadcast( type="UPDATE_FLOWS", cmd="update", - data=app._strip_content(f.get_state()) + data=app.convert_flow_to_json_dict(f) ) def _remove(self, f): diff --git a/netlib/http/headers.py b/netlib/http/headers.py index f052a53b..413add87 100644 --- a/netlib/http/headers.py +++ b/netlib/http/headers.py @@ -148,6 +148,15 @@ class Headers(multidict.MultiDict): value = _always_bytes(value) super(Headers, self).insert(index, key, value) + def items(self, multi=False): + if multi: + return ( + (_native(k), _native(v)) + for k, v in self.fields + ) + else: + return super(Headers, self).items() + def replace(self, pattern, repl, flags=0): """ Replaces a regular expression pattern with repl in each "name: value" diff --git a/netlib/http/http2/utils.py b/netlib/http/http2/utils.py index 4c01952d..164bacc8 100644 --- a/netlib/http/http2/utils.py +++ b/netlib/http/http2/utils.py @@ -7,9 +7,9 @@ def parse_headers(headers): scheme = headers.get(':scheme', 'https').encode() path = headers.get(':path', '/').encode() - headers.clear(":method") - headers.clear(":scheme") - headers.clear(":path") + headers.pop(":method", None) + headers.pop(":scheme", None) + headers.pop(":path", None) host = None port = None diff --git a/netlib/multidict.py b/netlib/multidict.py index 50c879d9..51053ff6 100644 --- a/netlib/multidict.py +++ b/netlib/multidict.py @@ -170,18 +170,10 @@ class _MultiDict(MutableMapping, basetypes.Serializable): else: return super(_MultiDict, self).items() - def clear(self, key): - """ - Removes all items with the specified key, and does not raise an - exception if the key does not exist. - """ - if key in self: - del self[key] - def collect(self): """ Returns a list of (key, value) tuples, where values are either - singular if threre is only one matching item for a key, or a list + singular if there is only one matching item for a key, or a list if there are more than one. The order of the keys matches the order in the underlying fields list. """ @@ -204,18 +196,16 @@ class _MultiDict(MutableMapping, basetypes.Serializable): .. code-block:: python # Simple dict with duplicate values. - >>> d - MultiDictView[("name", "value"), ("a", "false"), ("a", "42")] + >>> d = MultiDict([("name", "value"), ("a", False), ("a", 42)]) >>> d.to_dict() { "name": "value", - "a": ["false", "42"] + "a": [False, 42] } """ - d = {} - for k, v in self.collect(): - d[k] = v - return d + return { + k: v for k, v in self.collect() + } def get_state(self): return self.fields @@ -307,4 +297,4 @@ class MultiDictView(_MultiDict): @fields.setter def fields(self, value): - return self._setter(value) + self._setter(value) diff --git a/test/mitmproxy/test_web_master.py b/test/mitmproxy/test_web_master.py index 98f53c93..f0fafe24 100644 --- a/test/mitmproxy/test_web_master.py +++ b/test/mitmproxy/test_web_master.py @@ -13,5 +13,5 @@ class TestWebMaster(mastertest.MasterTest): def test_basic(self): m = self.mkmaster(None) for i in (1, 2, 3): - self.dummy_cycle(m, 1, "") + self.dummy_cycle(m, 1, b"") assert len(m.state.flows) == i @@ -16,7 +16,7 @@ commands = [testenv:py35] setenv = - TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py + TESTS = test/netlib test/pathod/ test/mitmproxy/script test/mitmproxy/test_contentview.py test/mitmproxy/test_custom_contentview.py test/mitmproxy/test_app.py test/mitmproxy/test_controller.py test/mitmproxy/test_fuzzing.py test/mitmproxy/test_script.py test/mitmproxy/test_web_app.py test/mitmproxy/test_utils.py test/mitmproxy/test_stateobject.py test/mitmproxy/test_cmdline.py test/mitmproxy/test_contrib_tnetstring.py test/mitmproxy/test_proxy.py test/mitmproxy/test_protocol_http1.py test/mitmproxy/test_platform_pf.py test/mitmproxy/test_server.py test/mitmproxy/test_filt.py test/mitmproxy/test_flow_export.py test/mitmproxy/test_web_master.py HOME = {envtmpdir} [testenv:docs] |