From 77bb0b74ed9cf156490f52a162a4d1856694db4f Mon Sep 17 00:00:00 2001 From: Youhei Sakurai Date: Fri, 27 Feb 2015 02:44:47 +0900 Subject: Maybe it should work; https://github.com/mitmproxy/mitmproxy/issues/319 --- doc-src/features/responsestreaming.html | 3 +++ examples/stream_modify.py | 11 +++++++++++ libmproxy/protocol/http.py | 7 ++++++- 3 files changed, 20 insertions(+), 1 deletion(-) create mode 100644 examples/stream_modify.py diff --git a/doc-src/features/responsestreaming.html b/doc-src/features/responsestreaming.html index 47fafef7..59847c8f 100644 --- a/doc-src/features/responsestreaming.html +++ b/doc-src/features/responsestreaming.html @@ -40,6 +40,9 @@ Responses that should be tagged for streaming by setting their respective .strea $!example("examples/stream.py")!$ +In addition, if the .stream attribute is set to callable(), .stream will work as a hook in chunk data processing. + +$!example("examples/stream_modify.py")!$

Implementation Details

diff --git a/examples/stream_modify.py b/examples/stream_modify.py new file mode 100644 index 00000000..517f730a --- /dev/null +++ b/examples/stream_modify.py @@ -0,0 +1,11 @@ +def modify(chunks): + """ + chunks is a generator that can be used to iterate over all chunks. + Each chunk is a (prefix, content, suffix) tuple. + For example, in the case of chunked transfer encoding: ("3\r\n","foo","\r\n") + """ + for prefix, content, suffix in chunks: + yield prefix, content.replace("foo","bar"), suffix + +def responseheaders(ctx, flow): + flow.response.stream = modify diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 046d0b42..fff2b84f 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1332,7 +1332,12 @@ class HTTPHandler(ProtocolHandler): # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) - for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, + for chunk in hasattr(flow.response.stream, "__call__") and \ + flow.response.stream(http.read_http_body_chunked(self.c.server_conn.rfile, + flow.response.headers, + self.c.config.body_size_limit, flow.request.method, + flow.response.code, False, 4096)) or \ + http.read_http_body_chunked(self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, flow.response.code, False, 4096): -- cgit v1.2.3 From 1d42c1b3c4f97a9e0d08bf55580b17ce144bd723 Mon Sep 17 00:00:00 2001 From: Youhei Sakurai Date: Fri, 27 Feb 2015 03:17:40 +0900 Subject: Modify example to notify incompatibility with --stream SIZE command line option; https://github.com/mitmproxy/mitmproxy/issues/319 --- examples/stream_modify.py | 8 ++++++++ 1 file changed, 8 insertions(+) diff --git a/examples/stream_modify.py b/examples/stream_modify.py index 517f730a..a28d95c7 100644 --- a/examples/stream_modify.py +++ b/examples/stream_modify.py @@ -1,3 +1,10 @@ +""" +This inline script won't work with --stream SIZE command line option. + +That's because flow.response.stream will be overwritten to True if the +command line option exists. +""" + def modify(chunks): """ chunks is a generator that can be used to iterate over all chunks. @@ -9,3 +16,4 @@ def modify(chunks): def responseheaders(ctx, flow): flow.response.stream = modify + flow.response.stream_large_bodies = 1024 # = 1KB -- cgit v1.2.3 From 3a78c95d0a202b486d062951ee0593f43a16eac2 Mon Sep 17 00:00:00 2001 From: Marcelo Glezer Date: Thu, 26 Feb 2015 18:14:20 -0300 Subject: added to flowlist / flowdetail time elapsed between request sent and response received --- libmproxy/console/common.py | 7 ++++++- libmproxy/utils.py | 12 ++++++++++++ test/test_utils.py | 13 +++++++++++++ 3 files changed, 31 insertions(+), 1 deletion(-) diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index a2cfd57b..8f074cfd 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -163,6 +163,7 @@ def raw_format_flow(f, focus, extended, padding): resp.append(fcol(f["resp_ctype"], rc)) resp.append(fcol(f["resp_clen"], rc)) resp.append(fcol(f["resp_rate"], rc)) + resp.append(fcol(f["roundtrip"], rc)) elif f["err_msg"]: resp.append(fcol(SYMBOL_RETURN, "error")) @@ -345,19 +346,23 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2): contentdesc = "[content missing]" else: contentdesc = "[no content]" - + duration = 0 if f.response.timestamp_end: delta = f.response.timestamp_end - f.response.timestamp_start + if f.request.timestamp_end: + duration = f.response.timestamp_end - f.request.timestamp_end else: delta = 0 size = f.response.size() rate = utils.pretty_size(size / ( delta if delta > 0 else 1 ) ) + roundtrip = utils.pretty_duration(duration) d.update(dict( resp_code = f.response.code, resp_is_replay = f.response.is_replay, resp_clen = contentdesc, resp_rate = "{0}/s".format(rate), + roundtrip = roundtrip, )) t = f.response.headers["content-type"] if t: diff --git a/libmproxy/utils.py b/libmproxy/utils.py index 33af035f..76e99c34 100644 --- a/libmproxy/utils.py +++ b/libmproxy/utils.py @@ -79,6 +79,18 @@ def pretty_size(size): x = int(x) return str(x) + suf +def pretty_duration(secs): + formatters = [ + (100, "{:.0f}s"), + (10, "{:2.1f}s"), + (1, "{:1.2f}s"), + ] + + for limit, formatter in formatters: + if secs >= limit: + return formatter.format(secs) + #less than 1 sec + return "{:.0f}ms".format(secs*1000) class Data: def __init__(self, name): diff --git a/test/test_utils.py b/test/test_utils.py index d99a146d..45bfb4f7 100644 --- a/test/test_utils.py +++ b/test/test_utils.py @@ -50,6 +50,19 @@ def test_urldecode(): s = "one=two&three=four" assert len(utils.urldecode(s)) == 2 +def test_pretty_duration(): + assert utils.pretty_duration(0.00001) == "0ms" + assert utils.pretty_duration(0.0001) == "0ms" + assert utils.pretty_duration(0.001) == "1ms" + assert utils.pretty_duration(0.01) == "10ms" + assert utils.pretty_duration(0.1) == "100ms" + assert utils.pretty_duration(1) == "1.00s" + assert utils.pretty_duration(10) == "10.0s" + assert utils.pretty_duration(100) == "100s" + assert utils.pretty_duration(1000) == "1000s" + assert utils.pretty_duration(10000) == "10000s" + assert utils.pretty_duration(1.123) == "1.12s" + assert utils.pretty_duration(0.123) == "123ms" def test_LRUCache(): class Foo: -- cgit v1.2.3 From 10f81e596bc0db37c62c0326ae6f7d3891f7756c Mon Sep 17 00:00:00 2001 From: Youhei Sakurai Date: Fri, 27 Feb 2015 10:15:07 +0900 Subject: Change from checking __call__ to using callable; https://github.com/mitmproxy/mitmproxy/issues/319 --- doc-src/features/responsestreaming.html | 2 +- libmproxy/protocol/http.py | 2 +- 2 files changed, 2 insertions(+), 2 deletions(-) diff --git a/doc-src/features/responsestreaming.html b/doc-src/features/responsestreaming.html index 59847c8f..20785269 100644 --- a/doc-src/features/responsestreaming.html +++ b/doc-src/features/responsestreaming.html @@ -40,7 +40,7 @@ Responses that should be tagged for streaming by setting their respective .strea $!example("examples/stream.py")!$ -In addition, if the .stream attribute is set to callable(), .stream will work as a hook in chunk data processing. +In addition, if the .stream attribute is callable, .stream will work as a hook in chunk data processing. $!example("examples/stream_modify.py")!$ diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index fff2b84f..78c4ac80 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1332,7 +1332,7 @@ class HTTPHandler(ProtocolHandler): # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) - for chunk in hasattr(flow.response.stream, "__call__") and \ + for chunk in callabe(flow.response.stream) and \ flow.response.stream(http.read_http_body_chunked(self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, -- cgit v1.2.3 From 5916767e036c6c7a816aa964bcd2b2721c7316bb Mon Sep 17 00:00:00 2001 From: Youhei Sakurai Date: Fri, 27 Feb 2015 10:22:27 +0900 Subject: Correct typo; https://github.com/mitmproxy/mitmproxy/issues/319 --- libmproxy/protocol/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 78c4ac80..51fd503f 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1332,7 +1332,7 @@ class HTTPHandler(ProtocolHandler): # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) - for chunk in callabe(flow.response.stream) and \ + for chunk in callable(flow.response.stream) and \ flow.response.stream(http.read_http_body_chunked(self.c.server_conn.rfile, flow.response.headers, self.c.config.body_size_limit, flow.request.method, -- cgit v1.2.3 From 81a274eb51ea7552667a872f0b6db1aeca9315b3 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 27 Feb 2015 09:17:41 +0100 Subject: fix #479 --- libmproxy/proxy/server.py | 7 ++++++- test/test_server.py | 31 +++++++++++++++++++++++++++++++ test/tservers.py | 4 ++-- 3 files changed, 39 insertions(+), 3 deletions(-) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index ea78d964..4e576067 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -285,7 +285,12 @@ class ConnectionHandler: if sni != self.server_conn.sni: self.log("SNI received: %s" % sni, "debug") - self.server_reconnect(sni) # reconnect to upstream server with SNI + # We should only re-establish upstream SSL if one of the following conditions is true: + # - We established SSL with the server previously + # - We initially wanted to establish SSL with the server, + # but the server refused to negotiate without SNI. + if self.server_conn.ssl_established or hasattr(self.server_conn, "may_require_sni"): + self.server_reconnect(sni) # reconnect to upstream server with SNI # Now, change client context to reflect changed certificate: cert, key, chain_file = self.find_cert() new_context = self.client_conn._create_ssl_context( diff --git a/test/test_server.py b/test/test_server.py index a611d30f..e387293f 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -1,5 +1,6 @@ import socket, time from libmproxy.proxy.config import HostMatcher +import libpathod from netlib import tcp, http_auth, http from libpathod import pathoc, pathod from netlib.certutils import SSLCert @@ -332,6 +333,36 @@ class TestReverse(tservers.ReverseProxTest, CommonMixin, TcpMixin): reverse = True +class TestHttps2Http(tservers.ReverseProxTest): + @classmethod + def get_proxy_config(cls): + d = super(TestHttps2Http, cls).get_proxy_config() + d["upstream_server"][0] = True + return d + + def pathoc(self, ssl, sni=None): + """ + Returns a connected Pathoc instance. + """ + p = libpathod.pathoc.Pathoc(("localhost", self.proxy.port), ssl=ssl, sni=sni) + p.connect() + return p + + def test_all(self): + p = self.pathoc(ssl=True) + assert p.request("get:'/p/200'").status_code == 200 + + def test_sni(self): + p = self.pathoc(ssl=True, sni="example.com") + assert p.request("get:'/p/200'").status_code == 200 + assert all("Error in handle_sni" not in msg for msg in self.proxy.log) + + def test_http(self): + p = self.pathoc(ssl=False) + assert p.request("get:'/p/200'").status_code == 400 + + + class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin): ssl = False diff --git a/test/tservers.py b/test/tservers.py index 37929d1a..30c8b52e 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -218,12 +218,12 @@ class ReverseProxTest(ProxTestBase): @classmethod def get_proxy_config(cls): d = ProxTestBase.get_proxy_config() - d["upstream_server"] = ( + d["upstream_server"] = [ True if cls.ssl else False, True if cls.ssl else False, "127.0.0.1", cls.server.port - ) + ] d["mode"] = "reverse" return d -- cgit v1.2.3 From 3323b29f10175d4100eb00a3787fa1c15e71e413 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 27 Feb 2015 12:51:06 +0100 Subject: always include SNI as SAN entry To be as robust as possible, we include the SNI value always as a Subject Alternative Name. Second, we make sure that the server address is in the list as well. --- libmproxy/proxy/server.py | 7 ++++--- 1 file changed, 4 insertions(+), 3 deletions(-) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 4e576067..8544ff72 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -260,11 +260,12 @@ class ConnectionHandler: sans = [] if self.server_conn.ssl_established and (not self.config.no_upstream_cert): upstream_cert = self.server_conn.cert + sans.extend(upstream_cert.altnames) if upstream_cert.cn: + sans.append(host) host = upstream_cert.cn.decode("utf8").encode("idna") - sans = upstream_cert.altnames - elif self.server_conn.sni: - sans = [self.server_conn.sni] + if self.server_conn.sni: + sans.append(self.server_conn.sni) ret = self.config.certstore.get_cert(host, sans) if not ret: -- cgit v1.2.3 From 595bde2202880971907096e4eeddb2b298c02d07 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 27 Feb 2015 12:55:16 +0100 Subject: add example inline script for dns spoofing, refs #486 --- examples/README | 1 + examples/dns_spoofing.py | 35 +++++++++++++++++++++++++++++++++++ 2 files changed, 36 insertions(+) create mode 100644 examples/dns_spoofing.py diff --git a/examples/README b/examples/README index 85ab272a..f24c4de7 100644 --- a/examples/README +++ b/examples/README @@ -1,6 +1,7 @@ # inline script examples add_header.py Simple script that just adds a header to every request. change_upstream_proxy.py Dynamically change the upstream proxy +dns_spoofing.py Use mitmproxy in a DNS spoofing scenario. dup_and_replay.py Duplicates each request, changes it, and then replays the modified request. iframe_injector.py Inject configurable iframe into pages. modify_form.py Modify all form submissions to add a parameter. diff --git a/examples/dns_spoofing.py b/examples/dns_spoofing.py new file mode 100644 index 00000000..cfba7c54 --- /dev/null +++ b/examples/dns_spoofing.py @@ -0,0 +1,35 @@ +""" +This inline scripts makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect +connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the +Host header of the HTTP request. +Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't +know the actual target and cannot construct a certificate that looks valid. +Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well. +Using transparent mode is the better option most of the time. + +Usage: + mitmproxy + -p 80 + -R http://example.com/ // Used as the target location if no Host header is present + mitmproxy + -p 443 + -R https://example.com/ // Used as the target locaction if neither SNI nor host header are present. + +mitmproxy will always connect to the default location first, so it must be reachable. +As a workaround, you can spawn an arbitrary HTTP server and use that for both endpoints, e.g. +mitmproxy -p 80 -R http://localhost:8000 +mitmproxy -p 443 -R https2http://localhost:8000 +""" + + +def request(context, flow): + if flow.client_conn.ssl_established: + # TLS SNI or Host header + flow.request.host = flow.client_conn.connection.get_servername() or flow.request.pretty_host(hostheader=True) + + # If you use a https2http location as default destination, these attributes need to be corrected as well: + flow.request.port = 443 + flow.request.scheme = "https" + else: + # Host header + flow.request.host = flow.request.pretty_host(hostheader=True) \ No newline at end of file -- cgit v1.2.3 From 355f9fc40799c88f413ca260dd93691074fd758e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 27 Feb 2015 14:43:23 +0100 Subject: use entry_points for Windows support --- setup.py | 8 +++++++- 1 file changed, 7 insertions(+), 1 deletion(-) diff --git a/setup.py b/setup.py index 9e22a039..cb0c3f66 100644 --- a/setup.py +++ b/setup.py @@ -36,6 +36,10 @@ for script in scripts: if os.name == "nt": deps.add("pydivert>=0.0.7") # Transparent proxying on Windows +console_scripts = [ + "%s = libmproxy.main:%s" % (s, s) for s in scripts +] + setup( name="mitmproxy", version=version.VERSION, @@ -65,7 +69,9 @@ setup( ], packages=find_packages(), include_package_data=True, - scripts=scripts, + entry_points={ + 'console_scripts': console_scripts + }, install_requires=list(deps), extras_require={ 'dev': [ -- cgit v1.2.3 From e1b6cf940146ca91c6f583a3333b4b50b72875bb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 27 Feb 2015 15:24:27 +0100 Subject: fix #319 --- doc-src/features/responsestreaming.html | 9 +++++---- examples/stream_modify.py | 19 +++++++++++-------- libmproxy/flow.py | 2 +- libmproxy/protocol/http.py | 22 +++++++++++++--------- test/scripts/stream_modify.py | 7 +++++++ test/test_server.py | 6 ++++++ 6 files changed, 43 insertions(+), 22 deletions(-) create mode 100644 test/scripts/stream_modify.py diff --git a/doc-src/features/responsestreaming.html b/doc-src/features/responsestreaming.html index 20785269..6511e913 100644 --- a/doc-src/features/responsestreaming.html +++ b/doc-src/features/responsestreaming.html @@ -40,10 +40,6 @@ Responses that should be tagged for streaming by setting their respective .strea $!example("examples/stream.py")!$ -In addition, if the .stream attribute is callable, .stream will work as a hook in chunk data processing. - -$!example("examples/stream_modify.py")!$ -

Implementation Details

When response streaming is enabled, portions of the code which would have otherwise performed changes @@ -52,6 +48,11 @@ on the response body will see an empty response body instead (libmproxy.pr Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a Transfer-Encoding: chunked header, the response will be streamed one chunk at a time. +

Modifying streamed data

+If the .stream attribute is callable, .stream will work as a hook in chunk data processing. + +$!example("examples/stream_modify.py")!$ + ### See Also - [Ignore Domains](@!urlTo("passthrough.html")!@) diff --git a/examples/stream_modify.py b/examples/stream_modify.py index a28d95c7..56d26e6d 100644 --- a/examples/stream_modify.py +++ b/examples/stream_modify.py @@ -1,10 +1,13 @@ """ -This inline script won't work with --stream SIZE command line option. - -That's because flow.response.stream will be overwritten to True if the -command line option exists. +This inline script modifies a streamed response. +If you do not need streaming, see the modify_response_body example. +Be aware that content replacement isn't trivial: + - If the transfer encoding isn't chunked, you cannot simply change the content length. + - If you want to replace all occurences of "foobar", make sure to catch the cases + where one chunk ends with [...]foo" and the next starts with "bar[...]. """ + def modify(chunks): """ chunks is a generator that can be used to iterate over all chunks. @@ -12,8 +15,8 @@ def modify(chunks): For example, in the case of chunked transfer encoding: ("3\r\n","foo","\r\n") """ for prefix, content, suffix in chunks: - yield prefix, content.replace("foo","bar"), suffix + yield prefix, content.replace("foo", "bar"), suffix + -def responseheaders(ctx, flow): - flow.response.stream = modify - flow.response.stream_large_bodies = 1024 # = 1KB +def responseheaders(context, flow): + flow.response.stream = modify \ No newline at end of file diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 14497964..43580109 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -165,7 +165,7 @@ class StreamLargeBodies(object): r.headers, is_request, flow.request.method, code ) if not (0 <= expected_size <= self.max_size): - r.stream = True + r.stream = r.stream or True # r.stream may already be a callable, which we want to preserve. class ClientPlaybackState: diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 51fd503f..49310ec3 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1332,15 +1332,19 @@ class HTTPHandler(ProtocolHandler): # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) - for chunk in callable(flow.response.stream) and \ - flow.response.stream(http.read_http_body_chunked(self.c.server_conn.rfile, - flow.response.headers, - self.c.config.body_size_limit, flow.request.method, - flow.response.code, False, 4096)) or \ - http.read_http_body_chunked(self.c.server_conn.rfile, - flow.response.headers, - self.c.config.body_size_limit, flow.request.method, - flow.response.code, False, 4096): + + chunks = http.read_http_body_chunked( + self.c.server_conn.rfile, + flow.response.headers, + self.c.config.body_size_limit, + flow.request.method, + flow.response.code, + False, + 4096 + ) + if callable(flow.response.stream): + chunks = flow.response.stream(chunks) + for chunk in chunks: for part in chunk: self.c.client_conn.wfile.write(part) self.c.client_conn.wfile.flush() diff --git a/test/scripts/stream_modify.py b/test/scripts/stream_modify.py new file mode 100644 index 00000000..9a98a7ee --- /dev/null +++ b/test/scripts/stream_modify.py @@ -0,0 +1,7 @@ +def modify(chunks): + for prefix, content, suffix in chunks: + yield prefix, content.replace("foo", "bar"), suffix + + +def responseheaders(context, flow): + flow.response.stream = modify \ No newline at end of file diff --git a/test/test_server.py b/test/test_server.py index e387293f..26770f29 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -266,6 +266,12 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin): assert self.master.state.view[-1].response.content == CONTENT_MISSING self.master.set_stream_large_bodies(None) + def test_stream_modify(self): + self.master.load_script(tutils.test_data.path("scripts/stream_modify.py")) + d = self.pathod('200:b"foo"') + assert d.content == "bar" + self.master.unload_scripts() + class TestHTTPAuth(tservers.HTTPProxTest): authenticator = http_auth.BasicProxyAuth(http_auth.PassManSingleUser("test", "test"), "realm") def test_auth(self): -- cgit v1.2.3 From eec4c539f2f6b8124ab92d6087afb228874a479e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 28 Feb 2015 03:35:28 +0100 Subject: SSLKEYLOGFILE docs --- doc-src/_nav.html | 1 + doc-src/dev/index.py | 1 + doc-src/dev/sslkeylogfile.html | 8 ++++++++ 3 files changed, 10 insertions(+) create mode 100644 doc-src/dev/sslkeylogfile.html diff --git a/doc-src/_nav.html b/doc-src/_nav.html index 6c3afbe1..69175c0c 100644 --- a/doc-src/_nav.html +++ b/doc-src/_nav.html @@ -56,4 +56,5 @@
  • Hacking
    • $!nav("dev/architecture.html", this, state)!$ $!nav("dev/testing.html", this, state)!$ + $!nav("dev/sslkeylogfile.html", this, state)!$ diff --git a/doc-src/dev/index.py b/doc-src/dev/index.py index bb7872c7..0f2a6494 100644 --- a/doc-src/dev/index.py +++ b/doc-src/dev/index.py @@ -3,5 +3,6 @@ from countershape import Page pages = [ Page("testing.html", "Testing"), Page("architecture.html", "Architecture"), + Page("sslkeylogfile.html", "TLS Master Secrets"), # Page("addingviews.html", "Writing Content Views"), ] diff --git a/doc-src/dev/sslkeylogfile.html b/doc-src/dev/sslkeylogfile.html new file mode 100644 index 00000000..1826fc2e --- /dev/null +++ b/doc-src/dev/sslkeylogfile.html @@ -0,0 +1,8 @@ +The SSL master keys can be logged by mitmproxy so that external programs can decrypt TLS connections both from and to the proxy. +Key logging is enabled by setting the environment variable SSLKEYLOGFILE so that it points to a writable +text file. Recent versions of WireShark can use these log files to decrypt packets. +You can specify the key file path in WireShark via
    +Edit → Preferences → Protocols → SSL → (Pre)-Master-Secret log filename. + + Note that SSLKEYLOGFILE is respected by other programs as well, e.g. Firefox and Chrome. +If this creates any issues, you can set MITMPROXY_SSLKEYLOGFILE alternatively. \ No newline at end of file -- cgit v1.2.3 From 0b7b0ac33dd8bb5d057ebc36b8979d5e3ddc0384 Mon Sep 17 00:00:00 2001 From: elitest Date: Sat, 28 Feb 2015 10:16:31 -0600 Subject: Update Config.py to improve cipher selection added support for specifying cipher suites on both sides of the proxy instead of just the one. --- libmproxy/proxy/config.py | 16 +++++++++++----- 1 file changed, 11 insertions(+), 5 deletions(-) diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 84893323..335d2dcf 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -55,7 +55,8 @@ class ProxyConfig: self.host = host self.port = port self.server_version = server_version - self.ciphers = ciphers + self.client_ciphers = client_ciphers + self.server_ciphers = server_ciphers self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit @@ -215,9 +216,14 @@ def ssl_option_group(parser): help="Client certificate directory." ) group.add_argument( - "--ciphers", action="store", - type=str, dest="ciphers", default=None, - help="SSL cipher specification." + "--client-ciphers", action="store", + type=str, dest="client_ciphers", default=None, + help="Proxy client SSL cipher specification." + ) + group.add_argument( + "--server-ciphers", action="store", + type=str, dest="server_ciphers", default=None, + help="Proxy server SSL cipher specification." ) group.add_argument( "--cert-forward", action="store_true", @@ -248,4 +254,4 @@ def ssl_option_group(parser): metavar="PORT", help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " "Defaults to %s." % str(TRANSPARENT_SSL_PORTS) - ) \ No newline at end of file + ) -- cgit v1.2.3 From 6a1e2941496d1606f9a112cb83652e0609be31d6 Mon Sep 17 00:00:00 2001 From: elitest Date: Sat, 28 Feb 2015 11:07:18 -0600 Subject: Update Server.py to improve cipher selection Differentiated client ciphers and added server ciphers. --- libmproxy/proxy/server.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 8544ff72..7306331c 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -189,6 +189,7 @@ class ConnectionHandler: sni, method=self.config.openssl_server_method, options=self.config.openssl_server_options + cipher_list=self.config.server_ciphers, ) except tcp.NetLibError as v: e = ProxyError(502, repr(v)) @@ -210,7 +211,7 @@ class ConnectionHandler: method=self.config.openssl_client_method, options=self.config.openssl_client_options, handle_sni=self.handle_sni, - cipher_list=self.config.ciphers, + cipher_list=self.config.client_ciphers, dhparams=self.config.certstore.dhparams, chain_file=chain_file ) -- cgit v1.2.3 From 09828ff2d9f64b326efbfda0d84d5e461bc672cf Mon Sep 17 00:00:00 2001 From: Marcelo Glezer Date: Sun, 1 Mar 2015 22:19:32 -0300 Subject: changes requested by @mhils --- libmproxy/console/common.py | 11 ++--------- 1 file changed, 2 insertions(+), 9 deletions(-) diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 8f074cfd..fa21c93e 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -162,7 +162,6 @@ def raw_format_flow(f, focus, extended, padding): if f["resp_ctype"]: resp.append(fcol(f["resp_ctype"], rc)) resp.append(fcol(f["resp_clen"], rc)) - resp.append(fcol(f["resp_rate"], rc)) resp.append(fcol(f["roundtrip"], rc)) elif f["err_msg"]: @@ -347,21 +346,15 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2): else: contentdesc = "[no content]" duration = 0 - if f.response.timestamp_end: - delta = f.response.timestamp_end - f.response.timestamp_start - if f.request.timestamp_end: - duration = f.response.timestamp_end - f.request.timestamp_end - else: - delta = 0 + if f.response.timestamp_end and f.request.timestamp_start: + duration = f.response.timestamp_end - f.request.timestamp_start size = f.response.size() - rate = utils.pretty_size(size / ( delta if delta > 0 else 1 ) ) roundtrip = utils.pretty_duration(duration) d.update(dict( resp_code = f.response.code, resp_is_replay = f.response.is_replay, resp_clen = contentdesc, - resp_rate = "{0}/s".format(rate), roundtrip = roundtrip, )) t = f.response.headers["content-type"] -- cgit v1.2.3 From b063d6020f18e8b0f3da56ebad557cec49a7ada5 Mon Sep 17 00:00:00 2001 From: Jim Shaver Date: Sun, 1 Mar 2015 20:12:27 -0600 Subject: specified cipher_list in a few more locations, added a missing comma --- libmproxy/proxy/config.py | 6 ++++-- libmproxy/proxy/server.py | 2 +- setup.py | 0 3 files changed, 5 insertions(+), 3 deletions(-) mode change 100644 => 100755 setup.py diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 335d2dcf..0215f92c 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -45,7 +45,8 @@ class ProxyConfig: authenticator=None, ignore_hosts=[], tcp_hosts=[], - ciphers=None, + client_ciphers=None, + server_ciphers=None, certs=[], certforward=False, ssl_version_client="secure", @@ -189,7 +190,8 @@ def process_proxy_options(parser, options): ignore_hosts=options.ignore_hosts, tcp_hosts=options.tcp_hosts, authenticator=authenticator, - ciphers=options.ciphers, + client_ciphers=options.client_ciphers, + server_ciphers=options.server_ciphers, certs=certs, certforward=options.certforward, ssl_version_client=options.ssl_version_client, diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 7306331c..ebe91d22 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -188,7 +188,7 @@ class ConnectionHandler: self.config.clientcerts, sni, method=self.config.openssl_server_method, - options=self.config.openssl_server_options + options=self.config.openssl_server_options, cipher_list=self.config.server_ciphers, ) except tcp.NetLibError as v: diff --git a/setup.py b/setup.py old mode 100644 new mode 100755 -- cgit v1.2.3 From ac92df0fda7535a17b7ee3522e2d50b7ce2c01a7 Mon Sep 17 00:00:00 2001 From: Jim Shaver Date: Sun, 1 Mar 2015 20:13:47 -0600 Subject: Un-executable'd setup.py --- setup.py | 0 1 file changed, 0 insertions(+), 0 deletions(-) mode change 100755 => 100644 setup.py diff --git a/setup.py b/setup.py old mode 100755 new mode 100644 -- cgit v1.2.3 From 13e74facb6b7af85cd9543ec56e01c3cd9b8270b Mon Sep 17 00:00:00 2001 From: elitest Date: Sun, 1 Mar 2015 20:21:35 -0600 Subject: Update config.py --- libmproxy/proxy/config.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 0215f92c..e8c75bee 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -191,7 +191,7 @@ def process_proxy_options(parser, options): tcp_hosts=options.tcp_hosts, authenticator=authenticator, client_ciphers=options.client_ciphers, - server_ciphers=options.server_ciphers, + server_ciphers=options.server_ciphers, certs=certs, certforward=options.certforward, ssl_version_client=options.ssl_version_client, -- cgit v1.2.3 From c6f54605a72fa577ad1c968eb438f0aad8347c82 Mon Sep 17 00:00:00 2001 From: elitest Date: Sun, 1 Mar 2015 20:49:03 -0600 Subject: Update Config.py to clarify help messages --- libmproxy/proxy/config.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index e8c75bee..a4765852 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -220,12 +220,12 @@ def ssl_option_group(parser): group.add_argument( "--client-ciphers", action="store", type=str, dest="client_ciphers", default=None, - help="Proxy client SSL cipher specification." + help="Client facing SSL cipher specification." ) group.add_argument( "--server-ciphers", action="store", type=str, dest="server_ciphers", default=None, - help="Proxy server SSL cipher specification." + help="Server facing SSL cipher specification." ) group.add_argument( "--cert-forward", action="store_true", -- cgit v1.2.3 From 5e07fe08ea80a860a215fe65b8430698261c7cb7 Mon Sep 17 00:00:00 2001 From: elitest Date: Mon, 2 Mar 2015 00:19:06 -0600 Subject: Update Server.py to fix SNI handling Forgot to change ciphers->client_ciphers. --- libmproxy/proxy/server.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index ebe91d22..cb6d3c70 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -299,7 +299,7 @@ class ConnectionHandler: cert, key, method=self.config.openssl_client_method, options=self.config.openssl_client_options, - cipher_list=self.config.ciphers, + cipher_list=self.config.client_ciphers, dhparams=self.config.certstore.dhparams, chain_file=chain_file ) -- cgit v1.2.3 From 75ba0a92e4dd0f331505f450d6baa89b18abe2f2 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 2 Mar 2015 14:35:50 +0100 Subject: do some housekeeping --- libmproxy/proxy/config.py | 60 +++++++++++++++++++++++------------------------ libmproxy/proxy/server.py | 18 +++++++------- 2 files changed, 39 insertions(+), 39 deletions(-) diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index a4765852..dfde2958 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -45,8 +45,8 @@ class ProxyConfig: authenticator=None, ignore_hosts=[], tcp_hosts=[], - client_ciphers=None, - server_ciphers=None, + ciphers_client=None, + ciphers_server=None, certs=[], certforward=False, ssl_version_client="secure", @@ -56,8 +56,8 @@ class ProxyConfig: self.host = host self.port = port self.server_version = server_version - self.client_ciphers = client_ciphers - self.server_ciphers = server_ciphers + self.ciphers_client = ciphers_client + self.ciphers_server = ciphers_server self.clientcerts = clientcerts self.no_upstream_cert = no_upstream_cert self.body_size_limit = body_size_limit @@ -85,8 +85,8 @@ class ProxyConfig: for spec, cert in certs: self.certstore.add_cert_file(spec, cert) self.certforward = certforward - self.openssl_client_method, self.openssl_client_options = version_to_openssl(ssl_version_client) - self.openssl_server_method, self.openssl_server_options = version_to_openssl(ssl_version_server) + self.openssl_method_client, self.openssl_options_client = version_to_openssl(ssl_version_client) + self.openssl_method_server, self.openssl_options_server = version_to_openssl(ssl_version_server) self.ssl_ports = ssl_ports @@ -190,8 +190,8 @@ def process_proxy_options(parser, options): ignore_hosts=options.ignore_hosts, tcp_hosts=options.tcp_hosts, authenticator=authenticator, - client_ciphers=options.client_ciphers, - server_ciphers=options.server_ciphers, + ciphers_client=options.ciphers_client, + ciphers_server=options.ciphers_server, certs=certs, certforward=options.certforward, ssl_version_client=options.ssl_version_client, @@ -212,25 +212,36 @@ def ssl_option_group(parser): 'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. ' 'Can be passed multiple times.' ) + group.add_argument( + "--cert-forward", action="store_true", + dest="certforward", default=False, + help="Simply forward SSL certificates from upstream." + ) + group.add_argument( + "--ciphers-client", action="store", + type=str, dest="ciphers_client", default=None, + help="Set supported ciphers for client connections. (OpenSSL Syntax)" + ) + group.add_argument( + "--ciphers-server", action="store", + type=str, dest="ciphers_server", default=None, + help="Set supported ciphers for server connections. (OpenSSL Syntax)" + ) group.add_argument( "--client-certs", action="store", type=str, dest="clientcerts", default=None, help="Client certificate directory." ) group.add_argument( - "--client-ciphers", action="store", - type=str, dest="client_ciphers", default=None, - help="Client facing SSL cipher specification." - ) - group.add_argument( - "--server-ciphers", action="store", - type=str, dest="server_ciphers", default=None, - help="Server facing SSL cipher specification." + "--no-upstream-cert", default=False, + action="store_true", dest="no_upstream_cert", + help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--cert-forward", action="store_true", - dest="certforward", default=False, - help="Simply forward SSL certificates from upstream." + "--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS), + metavar="PORT", + help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " + "Defaults to %s." % str(TRANSPARENT_SSL_PORTS) ) group.add_argument( "--ssl-version-client", dest="ssl_version_client", @@ -246,14 +257,3 @@ def ssl_option_group(parser): help="Set supported SSL/TLS version for server connections. " "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure." ) - group.add_argument( - "--no-upstream-cert", default=False, - action="store_true", dest="no_upstream_cert", - help="Don't connect to upstream server to look up certificate details." - ) - group.add_argument( - "--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS), - metavar="PORT", - help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " - "Defaults to %s." % str(TRANSPARENT_SSL_PORTS) - ) diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index cb6d3c70..896dd024 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -187,9 +187,9 @@ class ConnectionHandler: self.server_conn.establish_ssl( self.config.clientcerts, sni, - method=self.config.openssl_server_method, - options=self.config.openssl_server_options, - cipher_list=self.config.server_ciphers, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + cipher_list=self.config.ciphers_server, ) except tcp.NetLibError as v: e = ProxyError(502, repr(v)) @@ -208,10 +208,10 @@ class ConnectionHandler: try: self.client_conn.convert_to_ssl( cert, key, - method=self.config.openssl_client_method, - options=self.config.openssl_client_options, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, handle_sni=self.handle_sni, - cipher_list=self.config.client_ciphers, + cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, chain_file=chain_file ) @@ -297,9 +297,9 @@ class ConnectionHandler: cert, key, chain_file = self.find_cert() new_context = self.client_conn._create_ssl_context( cert, key, - method=self.config.openssl_client_method, - options=self.config.openssl_client_options, - cipher_list=self.config.client_ciphers, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, dhparams=self.config.certstore.dhparams, chain_file=chain_file ) -- cgit v1.2.3