From 3eac72f1a317f0890f74cd26e5ab955dbc2303f2 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 17 Dec 2016 10:34:53 +0100 Subject: http2: faster timeouts --- mitmproxy/proxy/protocol/http2.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index 41707096..45e55955 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -328,7 +328,7 @@ class Http2Layer(base.Layer): try: while True: - r = tcp.ssl_read_select(conns, 1) + r = tcp.ssl_read_select(conns, 0.1) for conn in r: source_conn = self.client_conn if conn == self.client_conn.connection else self.server_conn other_conn = self.server_conn if conn == self.client_conn.connection else self.client_conn @@ -580,7 +580,7 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr def read_response_body(self, request, response): while True: try: - yield self.response_data_queue.get(timeout=1) + yield self.response_data_queue.get(timeout=0.1) except queue.Empty: # pragma: no cover pass if self.response_data_finished.is_set(): -- cgit v1.2.3 From 377be68cac921917b241f8aee2170f7084dbf93d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 17 Dec 2016 16:01:44 +0100 Subject: appveyor: don't build installer for PRs --- .appveyor.yml | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/.appveyor.yml b/.appveyor.yml index b95b23e3..3725ea3f 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -34,7 +34,7 @@ test_script: - ps: | if( - ($Env:TOXENV -match "py35") -and + ($Env:TOXENV -match "py35") -and !$Env:APPVEYOR_PULL_REQUEST_NUMBER -and (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) ) { tox -e rtool -- decrypt release\installbuilder\license.xml.enc release\installbuilder\license.xml -- cgit v1.2.3 From ffcf0609285048ed2868548fe8e162b52cce1f25 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 17 Dec 2016 13:49:18 +0100 Subject: display ALPN information --- mitmproxy/connections.py | 16 ++++++++++++++-- mitmproxy/io_compat.py | 2 ++ mitmproxy/test/tflow.py | 4 +++- mitmproxy/tools/console/flowdetailview.py | 4 ++++ 4 files changed, 23 insertions(+), 3 deletions(-) diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py index c7941ad9..fc637420 100644 --- a/mitmproxy/connections.py +++ b/mitmproxy/connections.py @@ -22,6 +22,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): timestamp_end: Connection end timestamp sni: Server Name Indication sent by client during the TLS handshake cipher_name: The current used cipher + alpn_proto_negotiated: The negotiated application protocol tls_version: TLS version """ @@ -44,14 +45,16 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): self.timestamp_ssl_setup = None self.sni = None self.cipher_name = None + self.alpn_proto_negotiated = None self.tls_version = None def connected(self): return bool(self.connection) and not self.finished def __repr__(self): - return "".format( + return "".format( ssl="[ssl] " if self.ssl_established else "", + alpn="[ALPN: {}] ".format(self.alpn_proto_negotiated) if self.alpn_proto_negotiated else "", address=repr(self.address) ) @@ -68,6 +71,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): timestamp_end=float, sni=str, cipher_name=str, + alpn_proto_negotiated=str, tls_version=str, ) @@ -97,6 +101,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): timestamp_ssl_setup=None, sni=None, cipher_name=None, + alpn_proto_negotiated=None, tls_version=None, )) @@ -109,6 +114,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): else: self.sni = None self.cipher_name = self.connection.get_cipher_name() + self.alpn_proto_negotiated = self.get_alpn_proto_negotiated() self.tls_version = self.connection.get_protocol_version_name() def finish(self): @@ -128,6 +134,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): ssl_established: True if TLS is established, False otherwise cert: The certificate presented by the remote during the TLS handshake sni: Server Name Indication sent by the proxy during the TLS handshake + alpn_proto_negotiated: The negotiated application protocol via: The underlying server connection (e.g. the connection to the upstream proxy in upstream proxy mode) timestamp_start: Connection start timestamp timestamp_tcp_setup: TCP ACK received timestamp @@ -138,6 +145,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def __init__(self, address, source_address=None, spoof_source_address=None): tcp.TCPClient.__init__(self, address, source_address, spoof_source_address) + self.alpn_proto_negotiated = None self.via = None self.timestamp_start = None self.timestamp_end = None @@ -154,8 +162,9 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): ssl = "[ssl] " else: ssl = "" - return "".format( + return "".format( ssl=ssl, + alpn="[ALPN: {}] ".format(self.alpn_proto_negotiated) if self.alpn_proto_negotiated else "", address=repr(self.address) ) @@ -170,6 +179,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): ssl_established=bool, cert=certs.SSLCert, sni=str, + alpn_proto_negotiated=str, timestamp_start=float, timestamp_tcp_setup=float, timestamp_ssl_setup=float, @@ -189,6 +199,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): ip_address=dict(address=address, use_ipv6=False), cert=None, sni=None, + alpn_proto_negotiated=None, source_address=dict(address=('', 0), use_ipv6=False), ssl_established=False, timestamp_start=None, @@ -228,6 +239,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) self.sni = sni + self.alpn_proto_negotiated = self.get_alpn_proto_negotiated() self.timestamp_ssl_setup = time.time() def finish(self): diff --git a/mitmproxy/io_compat.py b/mitmproxy/io_compat.py index 20ee8824..14f4928c 100644 --- a/mitmproxy/io_compat.py +++ b/mitmproxy/io_compat.py @@ -67,8 +67,10 @@ def convert_017_018(data): def convert_018_019(data): data["version"] = (0, 19) data["client_conn"]["sni"] = None + data["client_conn"]["alpn_proto_negotiated"] = None data["client_conn"]["cipher_name"] = None data["client_conn"]["tls_version"] = None + data["server_conn"]["alpn_proto_negotiated"] = None data["mode"] = "regular" data["metadata"] = dict() return data diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py index 281047f5..f100f62e 100644 --- a/mitmproxy/test/tflow.py +++ b/mitmproxy/test/tflow.py @@ -72,6 +72,7 @@ def tclient_conn(): timestamp_end=3, sni="address", cipher_name="cipher", + alpn_proto_negotiated=None, tls_version="TLSv1.2", )) c.reply = controller.DummyReply() @@ -93,7 +94,8 @@ def tserver_conn(): timestamp_end=4, ssl_established=False, sni="address", - via=None + alpn_proto_negotiated=None, + via=None, )) c.reply = controller.DummyReply() return c diff --git a/mitmproxy/tools/console/flowdetailview.py b/mitmproxy/tools/console/flowdetailview.py index 7677efe4..8c08d6ee 100644 --- a/mitmproxy/tools/console/flowdetailview.py +++ b/mitmproxy/tools/console/flowdetailview.py @@ -31,6 +31,8 @@ def flowdetails(state, flow): ["Address", repr(sc.address)], ["Resolved Address", repr(sc.ip_address)], ] + if sc.alpn_proto_negotiated: + parts.append(["ALPN", sc.alpn_proto_negotiated]) text.extend( common.format_keyvals(parts, key="key", val="text", indent=4) @@ -94,6 +96,8 @@ def flowdetails(state, flow): parts.append(["Server Name Indication", cc.sni]) if cc.cipher_name: parts.append(["Cipher Name", cc.cipher_name]) + if cc.alpn_proto_negotiated: + parts.append(["ALPN", cc.alpn_proto_negotiated]) text.extend( common.format_keyvals(parts, key="key", val="text", indent=4) -- cgit v1.2.3 From c50feb6a408eb672cd0b21ebfa3037967cb4cf74 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 17 Dec 2016 17:59:23 +0100 Subject: bump h2 This takes care of https://github.com/python-hyper/hyper-h2/issues/422 --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 2f05eddc..08708b31 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ setup( "cryptography>=1.3, <1.8", "cssutils>=1.0.1, <1.1", "Flask>=0.10.1, <0.12", - "h2>=2.5.0, <3", + "h2>=2.5.1, <3", "html2text>=2016.1.8, <=2016.9.19", "hyperframe>=4.0.1, <5", "jsbeautifier>=1.6.3, <1.7", -- cgit v1.2.3 From 5cfc728d2e25b2663697075226d9310dbee5efc6 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 17 Dec 2016 18:28:34 +0100 Subject: don't log full tracebacks --- mitmproxy/proxy/protocol/http2.py | 3 --- mitmproxy/proxy/protocol/http_replay.py | 6 ++---- mitmproxy/proxy/server.py | 3 +-- 3 files changed, 3 insertions(+), 9 deletions(-) diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index 45e55955..b7548221 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -1,6 +1,5 @@ import threading import time -import traceback import functools from typing import Dict, Callable, Any, List # noqa @@ -358,7 +357,6 @@ class Http2Layer(base.Layer): self._cleanup_streams() except Exception as e: # pragma: no cover self.log(repr(e), "info") - self.log(traceback.format_exc(), "debug") self._kill_all_streams() @@ -624,7 +622,6 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr pass except exceptions.ProtocolException as e: # pragma: no cover self.log(repr(e), "info") - self.log(traceback.format_exc(), "debug") except exceptions.SetServerNotAllowedException as e: # pragma: no cover self.log("Changing the Host server for HTTP/2 connections not allowed: {}".format(e), "info") except exceptions.Kill: diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py index 2e4c91b0..38f511c4 100644 --- a/mitmproxy/proxy/protocol/http_replay.py +++ b/mitmproxy/proxy/protocol/http_replay.py @@ -1,5 +1,3 @@ -import traceback - from mitmproxy import log from mitmproxy import controller from mitmproxy import exceptions @@ -107,10 +105,10 @@ class RequestReplayThread(basethread.BaseThread): "log", log.LogEntry("Connection killed", "info") ) - except Exception: + except Exception as e: self.channel.tell( "log", - log.LogEntry(traceback.format_exc(), "error") + log.LogEntry(repr(e), "error") ) finally: r.first_line_format = first_line_format_backup diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index fc00a633..97018dad 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -120,7 +120,6 @@ class ConnectionHandler: except exceptions.Kill: self.log("Connection killed", "info") except exceptions.ProtocolException as e: - if isinstance(e, exceptions.ClientHandshakeException): self.log( "Client Handshake failed. " @@ -134,7 +133,7 @@ class ConnectionHandler: else: self.log(str(e), "warn") - self.log(traceback.format_exc(), "debug") + self.log(repr(e), "debug") # If an error propagates to the topmost level, # we send an HTTP error response, which is both # understandable by HTTP clients and humans. -- cgit v1.2.3 From 975d1b87a3c0531e196084434a168644d9c187f9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 17 Dec 2016 18:59:50 +0100 Subject: fix #1867 (#1873) --- mitmproxy/proxy/protocol/http.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index 50d64e17..59de070f 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -1,6 +1,5 @@ import h2.exceptions import time -import traceback import enum from mitmproxy import connections # noqa @@ -431,7 +430,7 @@ class HttpLayer(base.Layer): response = http.make_error_response(code, message, headers) self.send_response(response) except (exceptions.NetlibException, h2.exceptions.H2Error, exceptions.Http2ProtocolException): - self.log(traceback.format_exc(), "debug") + self.log("Failed to send error response to client: {}".format(message), "debug") def change_upstream_proxy_server(self, address) -> None: # Make set_upstream_proxy_server always available, -- cgit v1.2.3 From c4e90000210c392464944261c44e0bf1ed08608c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 19 Dec 2016 01:15:10 +0100 Subject: fix #1858 --- examples/complex/har_dump.py | 2 +- mitmproxy/addons/serverplayback.py | 30 +++++++++++++++++------------- mitmproxy/net/http/request.py | 9 +++++++-- test/mitmproxy/net/http/test_request.py | 4 ++-- test/mitmproxy/test_examples.py | 4 ++-- 5 files changed, 29 insertions(+), 20 deletions(-) diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index aeb154d2..62011903 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -136,7 +136,7 @@ def response(flow): if flow.request.method in ["POST", "PUT", "PATCH"]: params = [ - {"name": a.decode("utf8", "surrogateescape"), "value": b.decode("utf8", "surrogateescape")} + {"name": a, "value": b} for a, b in flow.request.urlencoded_form.items(multi=True) ] entry["request"]["postData"] = { diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 0b52918c..f2b5f206 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -1,9 +1,10 @@ -import urllib import hashlib +import urllib +from typing import Any # noqa +from typing import List # noqa -from mitmproxy.utils import strutils -from mitmproxy import exceptions from mitmproxy import ctx +from mitmproxy import exceptions from mitmproxy import io @@ -36,17 +37,20 @@ class ServerPlayback: _, _, path, _, query, _ = urllib.parse.urlparse(r.url) queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True) - key = [str(r.port), str(r.scheme), str(r.method), str(path)] + key = [str(r.port), str(r.scheme), str(r.method), str(path)] # type: List[Any] if not self.options.server_replay_ignore_content: - form_contents = r.urlencoded_form or r.multipart_form - if self.options.server_replay_ignore_payload_params and form_contents: - params = [ - strutils.always_bytes(i) - for i in self.options.server_replay_ignore_payload_params - ] - for p in form_contents.items(multi=True): - if p[0] not in params: - key.append(p) + if self.options.server_replay_ignore_payload_params and r.multipart_form: + key.extend( + (k, v) + for k, v in r.multipart_form.items(multi=True) + if k.decode(errors="replace") not in self.options.server_replay_ignore_payload_params + ) + elif self.options.server_replay_ignore_payload_params and r.urlencoded_form: + key.extend( + (k, v) + for k, v in r.urlencoded_form.items(multi=True) + if k not in self.options.server_replay_ignore_payload_params + ) else: key.append(str(r.raw_content)) diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index c3d85363..7cc4def7 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -350,6 +350,8 @@ class Request(message.Message): The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. An empty multidict.MultiDictView if the content-type indicates non-form data or the content could not be parsed. + + Starting with mitmproxy 1.0, key and value are strings. """ return multidict.MultiDictView( self._get_urlencoded_form, @@ -360,7 +362,7 @@ class Request(message.Message): is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower() if is_valid_content_type: try: - return tuple(mitmproxy.net.http.url.decode(self.content)) + return tuple(mitmproxy.net.http.url.decode(self.content.decode())) except ValueError: pass return () @@ -381,7 +383,10 @@ class Request(message.Message): def multipart_form(self): """ The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. - None if the content-type indicates non-form data. + An empty multidict.MultiDictView if the content-type indicates non-form data + or the content could not be parsed. + + Key and value are bytes. """ return multidict.MultiDictView( self._get_multipart_form, diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py index 9c0ec333..dfa76ba8 100644 --- a/test/mitmproxy/net/http/test_request.py +++ b/test/mitmproxy/net/http/test_request.py @@ -255,11 +255,11 @@ class TestRequestUtils: assert not request.urlencoded_form request.headers["Content-Type"] = "application/x-www-form-urlencoded" - assert list(request.urlencoded_form.items()) == [(b"foobar", b"baz")] + assert list(request.urlencoded_form.items()) == [("foobar", "baz")] def test_set_urlencoded_form(self): request = treq() - request.urlencoded_form = [(b'foo', b'bar'), (b'rab', b'oof')] + request.urlencoded_form = [('foo', 'bar'), ('rab', 'oof')] assert request.headers["Content-Type"] == "application/x-www-form-urlencoded" assert request.content diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index 610c9dad..3930e8df 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -68,11 +68,11 @@ class TestScripts(mastertest.MasterTest): f = tflow.tflow(req=tutils.treq(headers=form_header)) m.request(f) - assert f.request.urlencoded_form[b"mitmproxy"] == b"rocks" + assert f.request.urlencoded_form["mitmproxy"] == "rocks" f.request.headers["content-type"] = "" m.request(f) - assert list(f.request.urlencoded_form.items()) == [(b"foo", b"bar")] + assert list(f.request.urlencoded_form.items()) == [("foo", "bar")] def test_modify_querystring(self): m, sc = tscript("simple/modify_querystring.py") -- cgit v1.2.3 From fbce37054fa289eb5d7a9c791db9d4a71b96ecea Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 19 Dec 2016 16:06:18 +0100 Subject: fix #1877 --- mitmproxy/net/http/message.py | 6 +++++- test/mitmproxy/net/http/test_message.py | 9 +++++++++ 2 files changed, 14 insertions(+), 1 deletion(-) diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index d3d6898d..166f919a 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -103,7 +103,11 @@ class Message(serializable.Serializable): ce = self.headers.get("content-encoding") if ce: try: - return encoding.decode(self.raw_content, ce) + content = encoding.decode(self.raw_content, ce) + # A client may illegally specify a byte -> str encoding here (e.g. utf8) + if isinstance(content, str): + raise ValueError("Invalid Content-Encoding: {}".format(ce)) + return content except ValueError: if strict: raise diff --git a/test/mitmproxy/net/http/test_message.py b/test/mitmproxy/net/http/test_message.py index 69d029d9..a001e734 100644 --- a/test/mitmproxy/net/http/test_message.py +++ b/test/mitmproxy/net/http/test_message.py @@ -141,6 +141,15 @@ class TestMessageContentEncoding: assert r.headers["content-encoding"] assert r.get_content(strict=False) == b"foo" + def test_utf8_as_ce(self): + r = tutils.tresp() + r.headers["content-encoding"] = "utf8" + r.raw_content = b"foo" + with tutils.raises(ValueError): + assert r.content + assert r.headers["content-encoding"] + assert r.get_content(strict=False) == b"foo" + def test_cannot_decode(self): r = tutils.tresp() r.encode("gzip") -- cgit v1.2.3 From 4f3b50e417ceee79a358296a35e94f8ce13fc91b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 19 Dec 2016 15:36:46 +0100 Subject: fix io compat with previous releases --- mitmproxy/io_compat.py | 10 ++-- test/mitmproxy/data/dumpfile-018 | 85 +++++++++++++++++++++++++++++++ test/mitmproxy/test_flow_format_compat.py | 8 +++ 3 files changed, 99 insertions(+), 4 deletions(-) create mode 100644 test/mitmproxy/data/dumpfile-018 diff --git a/mitmproxy/io_compat.py b/mitmproxy/io_compat.py index 14f4928c..7a2e4773 100644 --- a/mitmproxy/io_compat.py +++ b/mitmproxy/io_compat.py @@ -65,7 +65,11 @@ def convert_017_018(data): def convert_018_019(data): - data["version"] = (0, 19) + # convert_unicode needs to be called for every dual release and the first py3-only release + data = convert_unicode(data) + + data["request"].pop("stickyauth", None) + data["request"].pop("stickycookie", None) data["client_conn"]["sni"] = None data["client_conn"]["alpn_proto_negotiated"] = None data["client_conn"]["cipher_name"] = None @@ -73,6 +77,7 @@ def convert_018_019(data): data["server_conn"]["alpn_proto_negotiated"] = None data["mode"] = "regular" data["metadata"] = dict() + data["version"] = (0, 19) return data @@ -138,7 +143,4 @@ def migrate_flow(flow_data): raise ValueError( "{} cannot read files serialized with version {}.".format(version.MITMPROXY, v) ) - # TODO: This should finally be moved in the converter for the first py3-only release. - # It's here so that a py2 0.18 dump can be read by py3 0.18 and vice versa. - flow_data = convert_unicode(flow_data) return flow_data diff --git a/test/mitmproxy/data/dumpfile-018 b/test/mitmproxy/data/dumpfile-018 new file mode 100644 index 00000000..abe8b0b1 --- /dev/null +++ b/test/mitmproxy/data/dumpfile-018 @@ -0,0 +1,85 @@ +5243:5:error;0:~11:intercepted;5:false!6:marked;5:false!2:id;36:55367415-10f5-4938-b69f-8a523394f947;7:request;396:10:stickyauth;5:false!7:content;0:,15:timestamp_start;18:1482157523.9086578^9:is_replay;5:false!4:path;1:/,4:host;15:www.example.com,17:first_line_format;8:relative;12:stickycookie;5:false!12:http_version;8:HTTP/1.1,6:method;3:GET,4:port;3:443#13:timestamp_end;18:1482157523.9086578^6:scheme;5:https,7:headers;82:29:10:User-Agent,11:curl/7.35.0,]26:4:Host,15:www.example.com,]15:6:Accept,3:*/*,]]}8:response;1851:6:reason;2:OK,12:http_version;8:HTTP/1.1,13:timestamp_end;17:1482157524.361187^11:status_code;3:200#7:content;1270: + + + Example Domain + + + + + + + + +
+

Example Domain

+

This domain is established to be used for illustrative examples in documents. You may use this + domain in examples without prior coordination or asking for permission.

+

More information...

+
+ + +,7:headers;410:25:13:Accept-Ranges,5:bytes,]35:13:Cache-Control,14:max-age=604800,]28:12:Content-Type,9:text/html,]40:4:Date,29:Mon, 19 Dec 2016 14:25:24 GMT,]22:4:Etag,11:"359670651",]43:7:Expires,29:Mon, 26 Dec 2016 14:25:24 GMT,]50:13:Last-Modified,29:Fri, 09 Aug 2013 23:54:35 GMT,]27:6:Server,14:ECS (iad/18CB),]26:4:Vary,15:Accept-Encoding,]16:7:X-Cache,3:HIT,]25:17:x-ec-custom-error,1:1,]25:14:Content-Length,4:1270,]]15:timestamp_start;17:1482157524.361187^}4:type;4:http;11:server_conn;2570:15:ssl_established;4:true!7:address;58:7:address;25:15:www.example.com;3:443#]8:use_ipv6;5:false!}10:ip_address;56:7:address;23:13:93.184.216.34;3:443#]8:use_ipv6;5:false!}3:via;0:~14:source_address;57:7:address;24:12:10.67.53.133;5:52775#]8:use_ipv6;5:false!}13:timestamp_end;0:~4:cert;2122:-----BEGIN CERTIFICATE----- +MIIF8jCCBNqgAwIBAgIQDmTF+8I2reFLFyrrQceMsDANBgkqhkiG9w0BAQsFADBw +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMS8wLQYDVQQDEyZEaWdpQ2VydCBTSEEyIEhpZ2ggQXNz +dXJhbmNlIFNlcnZlciBDQTAeFw0xNTExMDMwMDAwMDBaFw0xODExMjgxMjAwMDBa +MIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxML +TG9zIEFuZ2VsZXMxPDA6BgNVBAoTM0ludGVybmV0IENvcnBvcmF0aW9uIGZvciBB +c3NpZ25lZCBOYW1lcyBhbmQgTnVtYmVyczETMBEGA1UECxMKVGVjaG5vbG9neTEY +MBYGA1UEAxMPd3d3LmV4YW1wbGUub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAs0CWL2FjPiXBl61lRfvvE0KzLJmG9LWAC3bcBjgsH6NiVVo2dt6u +Xfzi5bTm7F3K7srfUBYkLO78mraM9qizrHoIeyofrV/n+pZZJauQsPjCPxMEJnRo +D8Z4KpWKX0LyDu1SputoI4nlQ/htEhtiQnuoBfNZxF7WxcxGwEsZuS1KcXIkHl5V +RJOreKFHTaXcB1qcZ/QRaBIv0yhxvK1yBTwWddT4cli6GfHcCe3xGMaSL328Fgs3 +jYrvG29PueB6VJi/tbbPu6qTfwp/H1brqdjh29U52Bhb0fJkM9DWxCP/Cattcc7a +z8EXnCO+LK8vkhw/kAiJWPKx4RBvgy73nwIDAQABo4ICUDCCAkwwHwYDVR0jBBgw +FoAUUWj/kK8CB3U8zNllZGKiErhZcjswHQYDVR0OBBYEFKZPYB4fLdHn8SOgKpUW +5Oia6m5IMIGBBgNVHREEejB4gg93d3cuZXhhbXBsZS5vcmeCC2V4YW1wbGUuY29t +ggtleGFtcGxlLmVkdYILZXhhbXBsZS5uZXSCC2V4YW1wbGUub3Jngg93d3cuZXhh +bXBsZS5jb22CD3d3dy5leGFtcGxlLmVkdYIPd3d3LmV4YW1wbGUubmV0MA4GA1Ud +DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0f +BG4wbDA0oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItaGEtc2Vy +dmVyLWc0LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTIt +aGEtc2VydmVyLWc0LmNybDBMBgNVHSAERTBDMDcGCWCGSAGG/WwBATAqMCgGCCsG +AQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAgGBmeBDAECAjCB +gwYIKwYBBQUHAQEEdzB1MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2Vy +dC5jb20wTQYIKwYBBQUHMAKGQWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9E +aWdpQ2VydFNIQTJIaWdoQXNzdXJhbmNlU2VydmVyQ0EuY3J0MAwGA1UdEwEB/wQC +MAAwDQYJKoZIhvcNAQELBQADggEBAISomhGn2L0LJn5SJHuyVZ3qMIlRCIdvqe0Q +6ls+C8ctRwRO3UU3x8q8OH+2ahxlQmpzdC5al4XQzJLiLjiJ2Q1p+hub8MFiMmVP +PZjb2tZm2ipWVuMRM+zgpRVM6nVJ9F3vFfUSHOb4/JsEIUvPY+d8/Krc+kPQwLvy +ieqRbcuFjmqfyPmUv1U9QoI4TQikpw7TZU0zYZANP4C/gj4Ry48/znmUaRvy2kvI +l7gRQ21qJTK5suoiYoYNo3J9T+pXPGU7Lydz/HwW+w0DpArtAaukI8aNX4ohFUKS +wDSiIIWIWJiJGbEeIO0TIFwEVWTOnbNl/faPXpk5IRXicapqiII= +-----END CERTIFICATE----- +,15:timestamp_start;18:1482157523.9086578^3:sni;15:www.example.com;19:timestamp_ssl_setup;17:1482157524.260993^19:timestamp_tcp_setup;18:1482157524.0081189^}11:client_conn;216:15:ssl_established;4:true!7:address;53:7:address;20:9:127.0.0.1;5:52774#]8:use_ipv6;5:false!}10:clientcert;0:~13:timestamp_end;0:~15:timestamp_start;18:1482157522.8949482^19:timestamp_ssl_setup;18:1482157523.9086578^}7:version;13:1:0#2:18#1:2#]} \ No newline at end of file diff --git a/test/mitmproxy/test_flow_format_compat.py b/test/mitmproxy/test_flow_format_compat.py index 0bb9cf15..a787278d 100644 --- a/test/mitmproxy/test_flow_format_compat.py +++ b/test/mitmproxy/test_flow_format_compat.py @@ -11,6 +11,14 @@ def test_load(): assert flows[0].request.url == "https://example.com/" +def test_load_018(): + with open(tutils.test_data.path("mitmproxy/data/dumpfile-018"), "rb") as f: + flow_reader = io.FlowReader(f) + flows = list(flow_reader.stream()) + assert len(flows) == 1 + assert flows[0].request.url == "https://www.example.com/" + + def test_cannot_convert(): with open(tutils.test_data.path("mitmproxy/data/dumpfile-010"), "rb") as f: flow_reader = io.FlowReader(f) -- cgit v1.2.3 From 98a079aa692fe532e54a5b919d939bf7772af8fd Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 19 Dec 2016 15:50:33 +0100 Subject: rename logging.py example. this conflicted with Python's stdlib --- docs/scripting/overview.rst | 4 ++-- examples/simple/log_events.py | 12 ++++++++++++ examples/simple/logging.py | 12 ------------ 3 files changed, 14 insertions(+), 14 deletions(-) create mode 100644 examples/simple/log_events.py delete mode 100644 examples/simple/logging.py diff --git a/docs/scripting/overview.rst b/docs/scripting/overview.rst index 84f2135e..6ec0caaa 100644 --- a/docs/scripting/overview.rst +++ b/docs/scripting/overview.rst @@ -85,8 +85,8 @@ and mitmproxy console can place script output in the event buffer. Here's how this looks: -.. literalinclude:: ../../examples/simple/logging.py - :caption: :src:`examples/simple/logging.py` +.. literalinclude:: ../../examples/simple/log_events.py + :caption: :src:`examples/simple/log_events.py` :language: python The ``ctx`` module also exposes the mitmproxy master object at ``ctx.master`` diff --git a/examples/simple/log_events.py b/examples/simple/log_events.py new file mode 100644 index 00000000..ab1baf75 --- /dev/null +++ b/examples/simple/log_events.py @@ -0,0 +1,12 @@ +""" +It is recommended to use `ctx.log` for logging within a script. +This goes to the event log in mitmproxy and to stdout in mitmdump. + +If you want to help us out: https://github.com/mitmproxy/mitmproxy/issues/1530 :-) +""" +from mitmproxy import ctx + + +def start(): + ctx.log.info("This is some informative text.") + ctx.log.error("This is an error.") diff --git a/examples/simple/logging.py b/examples/simple/logging.py deleted file mode 100644 index ab1baf75..00000000 --- a/examples/simple/logging.py +++ /dev/null @@ -1,12 +0,0 @@ -""" -It is recommended to use `ctx.log` for logging within a script. -This goes to the event log in mitmproxy and to stdout in mitmdump. - -If you want to help us out: https://github.com/mitmproxy/mitmproxy/issues/1530 :-) -""" -from mitmproxy import ctx - - -def start(): - ctx.log.info("This is some informative text.") - ctx.log.error("This is an error.") -- cgit v1.2.3 From f997b7fe14d40b8f5be61511e6b4956d92ef21db Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 20 Dec 2016 16:49:38 +0100 Subject: always decode alpn where required --- mitmproxy/connections.py | 21 +++++++++++++++++---- mitmproxy/test/tflow.py | 2 +- mitmproxy/tools/console/flowdetailview.py | 3 ++- mitmproxy/tools/web/app.py | 6 ++++++ 4 files changed, 26 insertions(+), 6 deletions(-) diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py index fc637420..9c4bca2f 100644 --- a/mitmproxy/connections.py +++ b/mitmproxy/connections.py @@ -6,6 +6,7 @@ import os from mitmproxy import stateobject from mitmproxy import certs from mitmproxy.net import tcp +from mitmproxy.utils import strutils class ClientConnection(tcp.BaseHandler, stateobject.StateObject): @@ -52,9 +53,15 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): return bool(self.connection) and not self.finished def __repr__(self): + if self.alpn_proto_negotiated: + alpn = "[ALPN: {}] ".format( + strutils.bytes_to_escaped_str(self.alpn_proto_negotiated) + ) + else: + alpn = "" return "".format( ssl="[ssl] " if self.ssl_established else "", - alpn="[ALPN: {}] ".format(self.alpn_proto_negotiated) if self.alpn_proto_negotiated else "", + alpn=alpn, address=repr(self.address) ) @@ -71,7 +78,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): timestamp_end=float, sni=str, cipher_name=str, - alpn_proto_negotiated=str, + alpn_proto_negotiated=bytes, tls_version=str, ) @@ -162,9 +169,15 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): ssl = "[ssl] " else: ssl = "" + if self.alpn_proto_negotiated: + alpn = "[ALPN: {}] ".format( + strutils.bytes_to_escaped_str(self.alpn_proto_negotiated) + ) + else: + alpn = "" return "".format( ssl=ssl, - alpn="[ALPN: {}] ".format(self.alpn_proto_negotiated) if self.alpn_proto_negotiated else "", + alpn=alpn, address=repr(self.address) ) @@ -179,7 +192,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): ssl_established=bool, cert=certs.SSLCert, sni=str, - alpn_proto_negotiated=str, + alpn_proto_negotiated=bytes, timestamp_start=float, timestamp_tcp_setup=float, timestamp_ssl_setup=float, diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py index f100f62e..959c9a2c 100644 --- a/mitmproxy/test/tflow.py +++ b/mitmproxy/test/tflow.py @@ -72,7 +72,7 @@ def tclient_conn(): timestamp_end=3, sni="address", cipher_name="cipher", - alpn_proto_negotiated=None, + alpn_proto_negotiated=b"http/1.1", tls_version="TLSv1.2", )) c.reply = controller.DummyReply() diff --git a/mitmproxy/tools/console/flowdetailview.py b/mitmproxy/tools/console/flowdetailview.py index 8c08d6ee..571a5e1b 100644 --- a/mitmproxy/tools/console/flowdetailview.py +++ b/mitmproxy/tools/console/flowdetailview.py @@ -2,6 +2,7 @@ import urwid from mitmproxy.tools.console import common, searchable from mitmproxy.utils import human +from mitmproxy.utils import strutils def maybe_timestamp(base, attr): @@ -77,7 +78,7 @@ def flowdetails(state, flow): parts.append( [ "Alt names", - ", ".join(str(x) for x in c.altnames) + ", ".join(strutils.bytes_to_escaped_str(x) for x in c.altnames) ] ) text.extend( diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index ce18c6f0..c0de4c1f 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -34,6 +34,12 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: "type": flow.type, "modified": flow.modified(), } + # .alpn_proto_negotiated is bytes, we need to decode that. + for conn in "client_conn", "server_conn": + if f[conn]["alpn_proto_negotiated"] is None: + continue + f[conn]["alpn_proto_negotiated"] = \ + f[conn]["alpn_proto_negotiated"].decode(errors="backslashreplace") if flow.error: f["error"] = flow.error.get_state() -- cgit v1.2.3 From 7de3507f9a3f75a890a946e1c6bd50950c527245 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 20 Dec 2016 17:30:35 +0100 Subject: docs: add mitmweb stub --- docs/index.rst | 1 + docs/mitmweb.rst | 16 ++++++++++++++++ docs/screenshots/mitmweb.png | Bin 0 -> 62415 bytes 3 files changed, 17 insertions(+) create mode 100644 docs/mitmweb.rst create mode 100644 docs/screenshots/mitmweb.png diff --git a/docs/index.rst b/docs/index.rst index cd32a1f6..a40a5f62 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -17,6 +17,7 @@ mitmproxy mitmdump + mitmweb config .. toctree:: diff --git a/docs/mitmweb.rst b/docs/mitmweb.rst new file mode 100644 index 00000000..e761fdbc --- /dev/null +++ b/docs/mitmweb.rst @@ -0,0 +1,16 @@ +.. _mitmweb: +.. program:: mitmweb + +mitmweb +======= + +**mitmweb** is mitmproxy's web-based user interface that allows interactive examination and modification of HTTP traffic. Like mitmproxy, it differs from mitmdump in that all flows are kept in memory, which means that it's intended for taking and manipulating +small-ish samples. + +.. warning:: + + Mitmweb is currently in beta. We consider it stable for all features currently exposed in the UI, but it still misses a lot of mitmproxy's features. + + +.. image:: screenshots/mitmweb.png + diff --git a/docs/screenshots/mitmweb.png b/docs/screenshots/mitmweb.png new file mode 100644 index 00000000..5f0cc925 Binary files /dev/null and b/docs/screenshots/mitmweb.png differ -- cgit v1.2.3 From 18f200907484333f737a94c80a40bfe1563781b5 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 20 Dec 2016 17:34:35 +0100 Subject: update installation page We emphasize the binary installation because that is usually the simplest method to install mitmproxy on all major platforms. --- docs/install.rst | 129 +++++++++++++++++++++++++++++-------------------------- 1 file changed, 69 insertions(+), 60 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 1fe09aca..70b52731 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -3,117 +3,125 @@ Installation ============ -.. _install-ubuntu: +Please follow the steps for your operating system. -Installation On Ubuntu ----------------------- +Once installation is complete, you can run :ref:`mitmproxy`, :ref:`mitmdump` or :ref:`mitmweb` from a terminal. -Ubuntu comes with Python but we need to install pip, python-dev and several libraries. -This was tested on a fully patched installation of Ubuntu 16.04. + +.. _install-macos: + +Installation on macOS +--------------------- + +You can use Homebrew to install everything: .. code:: bash - sudo apt-get install python3-pip python3-dev libffi-dev libssl-dev libtiff5-dev libjpeg8-dev zlib1g-dev libwebp-dev - sudo pip3 install mitmproxy # or pip install --user mitmproxy + brew install mitmproxy -On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install a newer version of Python. -mitmproxy requires Python 3.5 or higher. Please take a look at pyenv_. -Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``. +Or you can download the pre-built binary packages from `mitmproxy.org`_. -Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal. +.. _install-windows: -.. _install-fedora: +Installation on Windows +----------------------- -Installation On Fedora ----------------------- +The recommended way to install mitmproxy on Windows is to use the installer provided at `mitmproxy.org`_. After installation, you'll find shortcuts for :ref:`mitmweb` (the web-based interface) and :ref:`mitmdump` in the start menu. Both executables are added to your PATH and can be invoked from the command line. -Fedora comes with Python but we need to install pip, python-dev and several libraries. -This was tested on a fully patched installation of Fedora 24. +.. note:: + mitmproxy's console interface is not supported on Windows, but you can use mitmweb (the web-based interface) and mitmdump. -.. code:: bash +.. _install-linux: - sudo dnf install make gcc redhat-rpm-config python3-pip python3-devel libffi-devel openssl-devel libtiff-devel libjpeg-devel zlib-devel libwebp-devel openjpeg2-devel - sudo pip3 install mitmproxy # or pip install --user mitmproxy +Installation on Linux +--------------------- -Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``. +The recommended way to run mitmproxy on Linux is to use the pre-built binaries provided at `mitmproxy.org`_. -Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal. +Our pre-built binaries provide you with the latest version of mitmproxy, a self-contained Python 3.5 environment and a recent version of OpenSSL that supports HTTP/2. Of course, you can also install mitmproxy from source if you prefer that (see :ref:`install-advanced`). +.. _install-advanced: +Advanced Installation +--------------------- -.. _install-arch: +.. _install-docker: -Installation On Arch Linux --------------------------- +Docker Images +^^^^^^^^^^^^^ -mitmproxy has been added into the [community] repository. Use pacman to install it: +You can also use the official mitmproxy images from `DockerHub`_. +That being said, our portable binaries are just as easy to install and even easier to use. 😊 ->>> sudo pacman -S mitmproxy -Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal. +.. _install-arch: -.. _install-macos: +Installation on Arch Linux +^^^^^^^^^^^^^^^^^^^^^^^^^^ + +mitmproxy has been added into the [community] repository. Use pacman to install it: -Installation On macOS ------------------------- +>>> sudo pacman -S mitmproxy -You can use Homebrew to install everything: -.. code:: bash - brew install mitmproxy -Or you can download the pre-built binary packages from `mitmproxy.org`_. +.. _install-source-ubuntu: -Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal. +Installation from Source on Ubuntu +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Ubuntu comes with Python but we need to install pip3, python3-dev and several libraries. +This was tested on a fully patched installation of Ubuntu 16.04. +.. code:: bash -.. _install-windows: + sudo apt-get install python3-pip python3-dev libffi-dev libssl-dev libtiff5-dev libjpeg8-dev zlib1g-dev libwebp-dev + sudo pip3 install mitmproxy # or pip3 install --user mitmproxy -Installation On Windows ------------------------ +On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install a newer version of Python. +mitmproxy requires Python 3.5 or higher. Please take a look at pyenv_. +Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``. -.. note:: - Please note that mitmdump is the only component of mitmproxy that is supported on Windows at - the moment. +.. _install-source-fedora: - **There is no interactive user interface on Windows.** +Installation from Source on Fedora +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ +Fedora comes with Python but we need to install pip3, python3-dev and several libraries. +This was tested on a fully patched installation of Fedora 24. -First, install the latest version of Python 3.5 from the `Python website`_. -If you already have an older version of Python 3.5 installed, make sure to install pip_ -(pip is included in Python by default). If pip aborts with an error, make sure you are using the current version of pip. +.. code:: bash -.. code:: powershell + sudo dnf install make gcc redhat-rpm-config python3-pip python3-devel libffi-devel openssl-devel libtiff-devel libjpeg-devel zlib-devel libwebp-devel openjpeg2-devel + sudo pip3 install mitmproxy # or pip3 install --user mitmproxy - python -m pip install --upgrade pip +Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``. -Next, add Python and the Python Scripts directory to your **PATH** variable. -You can do this easily by running the following in powershell: +.. _install-source-windows: -.. code:: powershell +🐱💻 Installation from Source on Windows +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. note:: + mitmproxy's console interface is not supported on Windows, but you can use mitmweb (the web-based interface) and mitmdump. - [Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27;C:\Python27\Scripts", "User") +First, install the latest version of Python 3.5 or later from the `Python website`_. During installation, make sure to select `Add Python to PATH`. Now, you can install mitmproxy by running .. code:: powershell - pip install mitmproxy - -Once the installation is complete, you can run :ref:`mitmdump` from a command prompt. - + pip3 install mitmproxy -.. _install-source: +.. _install-dev-version: -Installation From Source ------------------------- +Latest Development Version +^^^^^^^^^^^^^^^^^^^^^^^^^^ If you would like to install mitmproxy directly from the master branch on GitHub or would like to -get set up to contribute to the project, install the dependencies as you would for a regular -mitmproxy installation. Then see the Hacking_ section of the README on GitHub. +get set up to contribute to the project, install the dependencies as you would for a regular installation from source. Then see the Hacking_ section of the README on GitHub. You can check your system information by running: ``mitmproxy --sysinfo`` @@ -122,3 +130,4 @@ You can check your system information by running: ``mitmproxy --sysinfo`` .. _`Python website`: https://www.python.org/downloads/windows/ .. _pip: https://pip.pypa.io/en/latest/installing.html .. _pyenv: https://github.com/yyuu/pyenv +.. _DockerHub: https://hub.docker.com/r/mitmproxy/mitmproxy/ -- cgit v1.2.3 From 42e9448adef2eb9b14616f71472ecae9cd267a84 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 21 Dec 2016 22:27:51 +0100 Subject: reflow docs --- docs/install.rst | 60 ++++++++++++++++++++++++++++++++++++-------------------- docs/mitmweb.rst | 10 ++++++---- 2 files changed, 45 insertions(+), 25 deletions(-) diff --git a/docs/install.rst b/docs/install.rst index 70b52731..93b3c364 100644 --- a/docs/install.rst +++ b/docs/install.rst @@ -5,7 +5,8 @@ Installation Please follow the steps for your operating system. -Once installation is complete, you can run :ref:`mitmproxy`, :ref:`mitmdump` or :ref:`mitmweb` from a terminal. +Once installation is complete, you can run :ref:`mitmproxy`, :ref:`mitmdump` or +:ref:`mitmweb` from a terminal. .. _install-macos: @@ -27,19 +28,28 @@ Or you can download the pre-built binary packages from `mitmproxy.org`_. Installation on Windows ----------------------- -The recommended way to install mitmproxy on Windows is to use the installer provided at `mitmproxy.org`_. After installation, you'll find shortcuts for :ref:`mitmweb` (the web-based interface) and :ref:`mitmdump` in the start menu. Both executables are added to your PATH and can be invoked from the command line. +The recommended way to install mitmproxy on Windows is to use the installer +provided at `mitmproxy.org`_. After installation, you'll find shortcuts for +:ref:`mitmweb` (the web-based interface) and :ref:`mitmdump` in the start menu. +Both executables are added to your PATH and can be invoked from the command +line. .. note:: - mitmproxy's console interface is not supported on Windows, but you can use mitmweb (the web-based interface) and mitmdump. + mitmproxy's console interface is not supported on Windows, but you can use + mitmweb (the web-based interface) and mitmdump. .. _install-linux: Installation on Linux --------------------- -The recommended way to run mitmproxy on Linux is to use the pre-built binaries provided at `mitmproxy.org`_. +The recommended way to run mitmproxy on Linux is to use the pre-built binaries +provided at `mitmproxy.org`_. -Our pre-built binaries provide you with the latest version of mitmproxy, a self-contained Python 3.5 environment and a recent version of OpenSSL that supports HTTP/2. Of course, you can also install mitmproxy from source if you prefer that (see :ref:`install-advanced`). +Our pre-built binaries provide you with the latest version of mitmproxy, a +self-contained Python 3.5 environment and a recent version of OpenSSL that +supports HTTP/2. Of course, you can also install mitmproxy from source if you +prefer that (see :ref:`install-advanced`). .. _install-advanced: @@ -51,9 +61,8 @@ Advanced Installation Docker Images ^^^^^^^^^^^^^ -You can also use the official mitmproxy images from `DockerHub`_. -That being said, our portable binaries are just as easy to install and even easier to use. 😊 - +You can also use the official mitmproxy images from `DockerHub`_. That being +said, our portable binaries are just as easy to install and even easier to use. 😊 .. _install-arch: @@ -66,31 +75,32 @@ mitmproxy has been added into the [community] repository. Use pacman to install >>> sudo pacman -S mitmproxy - .. _install-source-ubuntu: Installation from Source on Ubuntu ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Ubuntu comes with Python but we need to install pip3, python3-dev and several libraries. -This was tested on a fully patched installation of Ubuntu 16.04. +Ubuntu comes with Python but we need to install pip3, python3-dev and several +libraries. This was tested on a fully patched installation of Ubuntu 16.04. .. code:: bash sudo apt-get install python3-pip python3-dev libffi-dev libssl-dev libtiff5-dev libjpeg8-dev zlib1g-dev libwebp-dev sudo pip3 install mitmproxy # or pip3 install --user mitmproxy -On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install a newer version of Python. -mitmproxy requires Python 3.5 or higher. Please take a look at pyenv_. -Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``. +On older Ubuntu versions, e.g., **12.04** and **14.04**, you may need to install +a newer version of Python. mitmproxy requires Python 3.5 or higher. Please take +a look at pyenv_. Make sure to have an up-to-date version of pip by running +``pip3 install -U pip``. + .. _install-source-fedora: Installation from Source on Fedora ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ -Fedora comes with Python but we need to install pip3, python3-dev and several libraries. -This was tested on a fully patched installation of Fedora 24. +Fedora comes with Python but we need to install pip3, python3-dev and several +libraries. This was tested on a fully patched installation of Fedora 24. .. code:: bash @@ -99,15 +109,19 @@ This was tested on a fully patched installation of Fedora 24. Make sure to have an up-to-date version of pip by running ``pip3 install -U pip``. + + .. _install-source-windows: 🐱💻 Installation from Source on Windows ^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ .. note:: - mitmproxy's console interface is not supported on Windows, but you can use mitmweb (the web-based interface) and mitmdump. + mitmproxy's console interface is not supported on Windows, but you can use + mitmweb (the web-based interface) and mitmdump. -First, install the latest version of Python 3.5 or later from the `Python website`_. During installation, make sure to select `Add Python to PATH`. +First, install the latest version of Python 3.5 or later from the `Python +website`_. During installation, make sure to select `Add Python to PATH`. Now, you can install mitmproxy by running @@ -115,14 +129,18 @@ Now, you can install mitmproxy by running pip3 install mitmproxy + + .. _install-dev-version: Latest Development Version ^^^^^^^^^^^^^^^^^^^^^^^^^^ -If you would like to install mitmproxy directly from the master branch on GitHub or would like to -get set up to contribute to the project, install the dependencies as you would for a regular installation from source. Then see the Hacking_ section of the README on GitHub. -You can check your system information by running: ``mitmproxy --sysinfo`` +If you would like to install mitmproxy directly from the master branch on GitHub +or would like to get set up to contribute to the project, install the +dependencies as you would for a regular installation from source. Then see the +Hacking_ section of the README on GitHub. You can check your system information +by running: ``mitmproxy --sysinfo`` .. _Hacking: https://github.com/mitmproxy/mitmproxy/blob/master/README.rst#hacking diff --git a/docs/mitmweb.rst b/docs/mitmweb.rst index e761fdbc..e57af059 100644 --- a/docs/mitmweb.rst +++ b/docs/mitmweb.rst @@ -4,13 +4,15 @@ mitmweb ======= -**mitmweb** is mitmproxy's web-based user interface that allows interactive examination and modification of HTTP traffic. Like mitmproxy, it differs from mitmdump in that all flows are kept in memory, which means that it's intended for taking and manipulating -small-ish samples. +**mitmweb** is mitmproxy's web-based user interface that allows interactive +examination and modification of HTTP traffic. Like mitmproxy, it differs from +mitmdump in that all flows are kept in memory, which means that it's intended +for taking and manipulating small-ish samples. .. warning:: - Mitmweb is currently in beta. We consider it stable for all features currently exposed in the UI, but it still misses a lot of mitmproxy's features. + Mitmweb is currently in beta. We consider it stable for all features currently + exposed in the UI, but it still misses a lot of mitmproxy's features. .. image:: screenshots/mitmweb.png - -- cgit v1.2.3 From a196493a7a43ae45c4f6164b63513e670906acba Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 21 Dec 2016 23:13:53 +0100 Subject: remove output of webmaster during tests --- test/mitmproxy/test_web_app.py | 1 + 1 file changed, 1 insertion(+) diff --git a/test/mitmproxy/test_web_app.py b/test/mitmproxy/test_web_app.py index 6cf6ce26..8c15968b 100644 --- a/test/mitmproxy/test_web_app.py +++ b/test/mitmproxy/test_web_app.py @@ -20,6 +20,7 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): def get_app(self): o = options.Options() m = webmaster.WebMaster(o, proxy.DummyServer()) + m.addons.remove(m.addons.get('termlog')) f = tflow.tflow(resp=True) f.id = "42" m.view.add(f) -- cgit v1.2.3 From 24751965f99daf83c82d9c1581f712ca24da9d3d Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 21 Dec 2016 23:22:14 +0100 Subject: remove output of dumper during tests --- mitmproxy/tools/dump.py | 8 +++++--- mitmproxy/tools/web/master.py | 5 +++-- test/mitmproxy/test_tools_dump.py | 3 ++- test/mitmproxy/test_web_app.py | 3 +-- 4 files changed, 11 insertions(+), 8 deletions(-) diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index 90332627..4e2844a1 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -30,12 +30,14 @@ class Options(options.Options): class DumpMaster(master.Master): - def __init__(self, options, server): + def __init__(self, options, server, with_termlog=True, with_dumper=True): master.Master.__init__(self, options, server) self.has_errored = False - self.addons.add(termlog.TermLog()) + if with_termlog: + self.addons.add(termlog.TermLog()) self.addons.add(*addons.default_addons()) - self.addons.add(dumper.Dumper()) + if with_dumper: + self.addons.add(dumper.Dumper()) # This line is just for type hinting self.options = self.options # type: Options diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 68a3db70..e6c15a0a 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -15,7 +15,7 @@ from mitmproxy.tools.web import app class WebMaster(master.Master): - def __init__(self, options, server): + def __init__(self, options, server, with_termlog=True): super().__init__(options, server) self.view = view.View() self.view.sig_view_add.connect(self._sig_view_add) @@ -34,8 +34,9 @@ class WebMaster(master.Master): intercept.Intercept(), self.view, self.events, - termlog.TermLog(), ) + if with_termlog: + self.addons.add(termlog.TermLog()) self.app = app.Application( self, self.options.wdebug ) diff --git a/test/mitmproxy/test_tools_dump.py b/test/mitmproxy/test_tools_dump.py index 1488f33b..2e64d2d2 100644 --- a/test/mitmproxy/test_tools_dump.py +++ b/test/mitmproxy/test_tools_dump.py @@ -11,7 +11,8 @@ from . import mastertest class TestDumpMaster(mastertest.MasterTest): def mkmaster(self, flt, **options): o = dump.Options(filtstr=flt, verbosity=-1, flow_detail=0, **options) - return dump.DumpMaster(o, proxy.DummyServer()) + m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=False, with_dumper=False) + return m def test_read(self): with tutils.tmpdir() as t: diff --git a/test/mitmproxy/test_web_app.py b/test/mitmproxy/test_web_app.py index 8c15968b..61cf6993 100644 --- a/test/mitmproxy/test_web_app.py +++ b/test/mitmproxy/test_web_app.py @@ -19,8 +19,7 @@ def json(resp: httpclient.HTTPResponse): class TestApp(tornado.testing.AsyncHTTPTestCase): def get_app(self): o = options.Options() - m = webmaster.WebMaster(o, proxy.DummyServer()) - m.addons.remove(m.addons.get('termlog')) + m = webmaster.WebMaster(o, proxy.DummyServer(), with_termlog=False) f = tflow.tflow(resp=True) f.id = "42" m.view.add(f) -- cgit v1.2.3