Merge pull request #2 from mitmproxy/master

update to mitmproxy/master

18 files changed, 220 insertions, 51 deletions

diff --git a/doc-src/_nav.html b/doc-src/_nav.html
index 6c3afbe1..69175c0c 100644
--- a/doc-src/_nav.html
+++ b/doc-src/_nav.html
@@ -56,4 +56,5 @@
     <li class="nav-header">Hacking</li>
         $!nav("dev/architecture.html", this, state)!$
         $!nav("dev/testing.html", this, state)!$
+        $!nav("dev/sslkeylogfile.html", this, state)!$
 </ul>
diff --git a/doc-src/dev/index.py b/doc-src/dev/index.py
index bb7872c7..0f2a6494 100644
--- a/doc-src/dev/index.py
+++ b/doc-src/dev/index.py
@@ -3,5 +3,6 @@ from countershape import Page
 pages = [
      Page("testing.html", "Testing"),
      Page("architecture.html", "Architecture"),
+     Page("sslkeylogfile.html", "TLS Master Secrets"),
 #    Page("addingviews.html", "Writing Content Views"),
 ]
diff --git a/doc-src/dev/sslkeylogfile.html b/doc-src/dev/sslkeylogfile.html
new file mode 100644
index 00000000..1826fc2e
--- /dev/null
+++ b/doc-src/dev/sslkeylogfile.html
@@ -0,0 +1,8 @@
+The SSL master keys can be logged by mitmproxy so that external programs can decrypt TLS connections both from and to the proxy.
+Key logging is enabled by setting the environment variable <samp>SSLKEYLOGFILE</samp> so that it points to a writable
+text file. Recent versions of WireShark can use these log files to decrypt packets.
+You can specify the key file path in WireShark via<br>
+<samp>Edit → Preferences → Protocols → SSL → (Pre)-Master-Secret log filename</samp>.
+
+ Note that <samp>SSLKEYLOGFILE</samp> is respected by other programs as well, e.g. Firefox and Chrome.
+If this creates any issues, you can set <samp>MITMPROXY_SSLKEYLOGFILE</samp> alternatively.
\ No newline at end of file
diff --git a/doc-src/features/responsestreaming.html b/doc-src/features/responsestreaming.html
index 47fafef7..6511e913 100644
--- a/doc-src/features/responsestreaming.html
+++ b/doc-src/features/responsestreaming.html
@@ -40,7 +40,6 @@ Responses that should be tagged for streaming by setting their respective .strea
 
 $!example("examples/stream.py")!$
 
-
 <h2>Implementation Details</h2>
 
 When response streaming is enabled, portions of the code which would have otherwise performed changes
@@ -49,6 +48,11 @@ on the response body will see an empty response body instead (<code>libmproxy.pr
 Streamed responses are usually sent in chunks of 4096 bytes. If the response is sent with a <code>Transfer-Encoding:
     chunked</code> header, the response will be streamed one chunk at a time.
 
+<h2>Modifying streamed data</h2>
+If the <code>.stream</code> attribute is callable, .stream will work as a hook in chunk data processing.
+
+$!example("examples/stream_modify.py")!$
+
 ### See Also
 
 - [Ignore Domains](@!urlTo("passthrough.html")!@)
diff --git a/examples/README b/examples/README
index 85ab272a..f24c4de7 100644
--- a/examples/README
+++ b/examples/README
@@ -1,6 +1,7 @@
 # inline script examples
 add_header.py             Simple script that just adds a header to every request.
 change_upstream_proxy.py  Dynamically change the upstream proxy
+dns_spoofing.py           Use mitmproxy in a DNS spoofing scenario.
 dup_and_replay.py         Duplicates each request, changes it, and then replays the modified request.
 iframe_injector.py        Inject configurable iframe into pages.
 modify_form.py            Modify all form submissions to add a parameter.
diff --git a/examples/dns_spoofing.py b/examples/dns_spoofing.py
new file mode 100644
index 00000000..cfba7c54
--- /dev/null
+++ b/examples/dns_spoofing.py
@@ -0,0 +1,35 @@
+"""
+This inline scripts makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect
+connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the
+Host header of the HTTP request.
+Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't
+know the actual target and cannot construct a certificate that looks valid.
+Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well.
+Using transparent mode is the better option most of the time.
+
+Usage:
+    mitmproxy
+        -p 80
+        -R http://example.com/  // Used as the target location if no Host header is present
+    mitmproxy
+        -p 443
+        -R https://example.com/ // Used as the target locaction if neither SNI nor host header are present.
+
+mitmproxy will always connect to the default location first, so it must be reachable.
+As a workaround, you can spawn an arbitrary HTTP server and use that for both endpoints, e.g.
+mitmproxy -p  80 -R       http://localhost:8000
+mitmproxy -p 443 -R https2http://localhost:8000
+"""
+
+
+def request(context, flow):
+    if flow.client_conn.ssl_established:
+        # TLS SNI or Host header
+        flow.request.host = flow.client_conn.connection.get_servername() or flow.request.pretty_host(hostheader=True)
+
+        # If you use a https2http location as default destination, these attributes need to be corrected as well:
+        flow.request.port = 443
+        flow.request.scheme = "https"
+    else:
+        # Host header
+        flow.request.host = flow.request.pretty_host(hostheader=True)
\ No newline at end of file
diff --git a/examples/stream_modify.py b/examples/stream_modify.py
new file mode 100644
index 00000000..56d26e6d
--- /dev/null
+++ b/examples/stream_modify.py
@@ -0,0 +1,22 @@
+"""
+This inline script modifies a streamed response.
+If you do not need streaming, see the modify_response_body example.
+Be aware that content replacement isn't trivial:
+    - If the transfer encoding isn't chunked, you cannot simply change the content length.
+    - If you want to replace all occurences of "foobar", make sure to catch the cases
+      where one chunk ends with [...]foo" and the next starts with "bar[...].
+"""
+
+
+def modify(chunks):
+    """
+    chunks is a generator that can be used to iterate over all chunks.
+    Each chunk is a (prefix, content, suffix) tuple. 
+    For example, in the case of chunked transfer encoding: ("3\r\n","foo","\r\n")
+    """
+    for prefix, content, suffix in chunks:
+        yield prefix, content.replace("foo", "bar"), suffix
+
+
+def responseheaders(context, flow):
+    flow.response.stream = modify
\ No newline at end of file
diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py
index a2cfd57b..fa21c93e 100644
--- a/libmproxy/console/common.py
+++ b/libmproxy/console/common.py
@@ -162,7 +162,7 @@ def raw_format_flow(f, focus, extended, padding):
         if f["resp_ctype"]:
             resp.append(fcol(f["resp_ctype"], rc))
         resp.append(fcol(f["resp_clen"], rc))
-        resp.append(fcol(f["resp_rate"], rc))
+        resp.append(fcol(f["roundtrip"], rc))
 
     elif f["err_msg"]:
         resp.append(fcol(SYMBOL_RETURN, "error"))
@@ -345,19 +345,17 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
             contentdesc = "[content missing]"
         else:
             contentdesc = "[no content]"
-
-        if f.response.timestamp_end:
-            delta = f.response.timestamp_end - f.response.timestamp_start
-        else:
-            delta = 0
+        duration = 0
+        if f.response.timestamp_end and f.request.timestamp_start:
+            duration = f.response.timestamp_end - f.request.timestamp_start
         size = f.response.size()
-        rate = utils.pretty_size(size / ( delta if delta > 0 else 1 ) )
+        roundtrip = utils.pretty_duration(duration)
 
         d.update(dict(
             resp_code = f.response.code,
             resp_is_replay = f.response.is_replay,
             resp_clen = contentdesc,
-            resp_rate = "{0}/s".format(rate),
+            roundtrip = roundtrip,
         ))
         t = f.response.headers["content-type"]
         if t:
diff --git a/libmproxy/flow.py b/libmproxy/flow.py
index 14497964..43580109 100644
--- a/libmproxy/flow.py
+++ b/libmproxy/flow.py
@@ -165,7 +165,7 @@ class StreamLargeBodies(object):
             r.headers, is_request, flow.request.method, code
         )
         if not (0 <= expected_size <= self.max_size):
-            r.stream = True
+            r.stream = r.stream or True  # r.stream may already be a callable, which we want to preserve.
 
 
 class ClientPlaybackState:
diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py
index 046d0b42..49310ec3 100644
--- a/libmproxy/protocol/http.py
+++ b/libmproxy/protocol/http.py
@@ -1332,10 +1332,19 @@ class HTTPHandler(ProtocolHandler):
             # incrementally:
             h = flow.response._assemble_head(preserve_transfer_encoding=True)
             self.c.client_conn.send(h)
-            for chunk in http.read_http_body_chunked(self.c.server_conn.rfile,
-                                                     flow.response.headers,
-                                                     self.c.config.body_size_limit, flow.request.method,
-                                                     flow.response.code, False, 4096):
+
+            chunks = http.read_http_body_chunked(
+                self.c.server_conn.rfile,
+                flow.response.headers,
+                self.c.config.body_size_limit,
+                flow.request.method,
+                flow.response.code,
+                False,
+                4096
+            )
+            if callable(flow.response.stream):
+                chunks = flow.response.stream(chunks)
+            for chunk in chunks:
                 for part in chunk:
                     self.c.client_conn.wfile.write(part)
                 self.c.client_conn.wfile.flush()
diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py
index 84893323..dfde2958 100644
--- a/libmproxy/proxy/config.py
+++ b/libmproxy/proxy/config.py
@@ -45,7 +45,8 @@ class ProxyConfig:
             authenticator=None,
             ignore_hosts=[],
             tcp_hosts=[],
-            ciphers=None,
+            ciphers_client=None,
+            ciphers_server=None,
             certs=[],
             certforward=False,
             ssl_version_client="secure",
@@ -55,7 +56,8 @@ class ProxyConfig:
         self.host = host
         self.port = port
         self.server_version = server_version
-        self.ciphers = ciphers
+        self.ciphers_client = ciphers_client
+        self.ciphers_server = ciphers_server
         self.clientcerts = clientcerts
         self.no_upstream_cert = no_upstream_cert
         self.body_size_limit = body_size_limit
@@ -83,8 +85,8 @@ class ProxyConfig:
         for spec, cert in certs:
             self.certstore.add_cert_file(spec, cert)
         self.certforward = certforward
-        self.openssl_client_method, self.openssl_client_options = version_to_openssl(ssl_version_client)
-        self.openssl_server_method, self.openssl_server_options = version_to_openssl(ssl_version_server)
+        self.openssl_method_client, self.openssl_options_client = version_to_openssl(ssl_version_client)
+        self.openssl_method_server, self.openssl_options_server = version_to_openssl(ssl_version_server)
         self.ssl_ports = ssl_ports
 
 
@@ -188,7 +190,8 @@ def process_proxy_options(parser, options):
         ignore_hosts=options.ignore_hosts,
         tcp_hosts=options.tcp_hosts,
         authenticator=authenticator,
-        ciphers=options.ciphers,
+        ciphers_client=options.ciphers_client,
+        ciphers_server=options.ciphers_server,
         certs=certs,
         certforward=options.certforward,
         ssl_version_client=options.ssl_version_client,
@@ -210,19 +213,35 @@ def ssl_option_group(parser):
              'Can be passed multiple times.'
     )
     group.add_argument(
+        "--cert-forward", action="store_true",
+        dest="certforward", default=False,
+        help="Simply forward SSL certificates from upstream."
+    )
+    group.add_argument(
+        "--ciphers-client", action="store",
+        type=str, dest="ciphers_client", default=None,
+        help="Set supported ciphers for client connections. (OpenSSL Syntax)"
+    )
+    group.add_argument(
+        "--ciphers-server", action="store",
+        type=str, dest="ciphers_server", default=None,
+        help="Set supported ciphers for server connections. (OpenSSL Syntax)"
+    )
+    group.add_argument(
         "--client-certs", action="store",
         type=str, dest="clientcerts", default=None,
         help="Client certificate directory."
     )
     group.add_argument(
-        "--ciphers", action="store",
-        type=str, dest="ciphers", default=None,
-        help="SSL cipher specification."
+        "--no-upstream-cert", default=False,
+        action="store_true", dest="no_upstream_cert",
+        help="Don't connect to upstream server to look up certificate details."
     )
     group.add_argument(
-        "--cert-forward", action="store_true",
-        dest="certforward", default=False,
-        help="Simply forward SSL certificates from upstream."
+        "--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS),
+        metavar="PORT",
+        help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. "
+             "Defaults to %s." % str(TRANSPARENT_SSL_PORTS)
     )
     group.add_argument(
         "--ssl-version-client", dest="ssl_version_client",
@@ -238,14 +257,3 @@ def ssl_option_group(parser):
         help="Set supported SSL/TLS version for server connections. "
              "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure."
     )
-    group.add_argument(
-        "--no-upstream-cert", default=False,
-        action="store_true", dest="no_upstream_cert",
-        help="Don't connect to upstream server to look up certificate details."
-    )
-    group.add_argument(
-        "--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS),
-        metavar="PORT",
-        help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. "
-             "Defaults to %s." % str(TRANSPARENT_SSL_PORTS)
-    )
\ No newline at end of file
diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py
index ea78d964..896dd024 100644
--- a/libmproxy/proxy/server.py
+++ b/libmproxy/proxy/server.py
@@ -187,8 +187,9 @@ class ConnectionHandler:
                 self.server_conn.establish_ssl(
                     self.config.clientcerts,
                     sni,
-                    method=self.config.openssl_server_method,
-                    options=self.config.openssl_server_options
+                    method=self.config.openssl_method_server,
+                    options=self.config.openssl_options_server,
+                    cipher_list=self.config.ciphers_server,
                 )
             except tcp.NetLibError as v:
                 e = ProxyError(502, repr(v))
@@ -207,10 +208,10 @@ class ConnectionHandler:
             try:
                 self.client_conn.convert_to_ssl(
                     cert, key,
-                    method=self.config.openssl_client_method,
-                    options=self.config.openssl_client_options,
+                    method=self.config.openssl_method_client,
+                    options=self.config.openssl_options_client,
                     handle_sni=self.handle_sni,
-                    cipher_list=self.config.ciphers,
+                    cipher_list=self.config.ciphers_client,
                     dhparams=self.config.certstore.dhparams,
                     chain_file=chain_file
                 )
@@ -260,11 +261,12 @@ class ConnectionHandler:
             sans = []
             if self.server_conn.ssl_established and (not self.config.no_upstream_cert):
                 upstream_cert = self.server_conn.cert
+                sans.extend(upstream_cert.altnames)
                 if upstream_cert.cn:
+                    sans.append(host)
                     host = upstream_cert.cn.decode("utf8").encode("idna")
-                sans = upstream_cert.altnames
-            elif self.server_conn.sni:
-                sans = [self.server_conn.sni]
+            if self.server_conn.sni:
+                sans.append(self.server_conn.sni)
 
             ret = self.config.certstore.get_cert(host, sans)
             if not ret:
@@ -285,14 +287,19 @@ class ConnectionHandler:
 
             if sni != self.server_conn.sni:
                 self.log("SNI received: %s" % sni, "debug")
-                self.server_reconnect(sni)  # reconnect to upstream server with SNI
+                # We should only re-establish upstream SSL if one of the following conditions is true:
+                #   - We established SSL with the server previously
+                #   - We initially wanted to establish SSL with the server,
+                #     but the server refused to negotiate without SNI.
+                if self.server_conn.ssl_established or hasattr(self.server_conn, "may_require_sni"):
+                    self.server_reconnect(sni)  # reconnect to upstream server with SNI
                 # Now, change client context to reflect changed certificate:
                 cert, key, chain_file = self.find_cert()
                 new_context = self.client_conn._create_ssl_context(
                     cert, key,
-                    method=self.config.openssl_client_method,
-                    options=self.config.openssl_client_options,
-                    cipher_list=self.config.ciphers,
+                    method=self.config.openssl_method_client,
+                    options=self.config.openssl_options_client,
+                    cipher_list=self.config.ciphers_client,
                     dhparams=self.config.certstore.dhparams,
                     chain_file=chain_file
                 )
diff --git a/libmproxy/utils.py b/libmproxy/utils.py
index 33af035f..76e99c34 100644
--- a/libmproxy/utils.py
+++ b/libmproxy/utils.py
@@ -79,6 +79,18 @@ def pretty_size(size):
                 x = int(x)
             return str(x) + suf
 
+def pretty_duration(secs):
+    formatters = [
+        (100, "{:.0f}s"), 
+        (10, "{:2.1f}s"), 
+        (1, "{:1.2f}s"),         
+    ]
+
+    for limit, formatter in formatters:
+        if secs >= limit:
+            return formatter.format(secs)
+    #less than 1 sec 
+    return "{:.0f}ms".format(secs*1000)
 
 class Data:
     def __init__(self, name):
diff --git a/setup.py b/setup.py
index 9e22a039..cb0c3f66 100644
--- a/setup.py
+++ b/setup.py
@@ -36,6 +36,10 @@ for script in scripts:
 if os.name == "nt":
     deps.add("pydivert>=0.0.7")  # Transparent proxying on Windows
 
+console_scripts = [
+    "%s = libmproxy.main:%s" % (s, s) for s in scripts
+]
+
 setup(
     name="mitmproxy",
     version=version.VERSION,
@@ -65,7 +69,9 @@ setup(
     ],
     packages=find_packages(),
     include_package_data=True,
-    scripts=scripts,
+    entry_points={
+        'console_scripts': console_scripts
+    },
     install_requires=list(deps),
     extras_require={
         'dev': [
diff --git a/test/scripts/stream_modify.py b/test/scripts/stream_modify.py
new file mode 100644
index 00000000..9a98a7ee
--- /dev/null
+++ b/test/scripts/stream_modify.py
@@ -0,0 +1,7 @@
+def modify(chunks):
+    for prefix, content, suffix in chunks:
+        yield prefix, content.replace("foo", "bar"), suffix
+
+
+def responseheaders(context, flow):
+    flow.response.stream = modify
\ No newline at end of file
diff --git a/test/test_server.py b/test/test_server.py
index a611d30f..26770f29 100644
--- a/test/test_server.py
+++ b/test/test_server.py
@@ -1,5 +1,6 @@
 import socket, time
 from libmproxy.proxy.config import HostMatcher
+import libpathod
 from netlib import tcp, http_auth, http
 from libpathod import pathoc, pathod
 from netlib.certutils import SSLCert
@@ -265,6 +266,12 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin):
         assert self.master.state.view[-1].response.content == CONTENT_MISSING
         self.master.set_stream_large_bodies(None)
 
+    def test_stream_modify(self):
+        self.master.load_script(tutils.test_data.path("scripts/stream_modify.py"))
+        d = self.pathod('200:b"foo"')
+        assert d.content == "bar"
+        self.master.unload_scripts()
+
 class TestHTTPAuth(tservers.HTTPProxTest):
     authenticator = http_auth.BasicProxyAuth(http_auth.PassManSingleUser("test", "test"), "realm")
     def test_auth(self):
@@ -332,6 +339,36 @@ class TestReverse(tservers.ReverseProxTest, CommonMixin, TcpMixin):
     reverse = True
 
 
+class TestHttps2Http(tservers.ReverseProxTest):
+    @classmethod
+    def get_proxy_config(cls):
+        d = super(TestHttps2Http, cls).get_proxy_config()
+        d["upstream_server"][0] = True
+        return d
+
+    def pathoc(self, ssl, sni=None):
+        """
+            Returns a connected Pathoc instance.
+        """
+        p = libpathod.pathoc.Pathoc(("localhost", self.proxy.port), ssl=ssl, sni=sni)
+        p.connect()
+        return p
+
+    def test_all(self):
+        p = self.pathoc(ssl=True)
+        assert p.request("get:'/p/200'").status_code == 200
+
+    def test_sni(self):
+        p = self.pathoc(ssl=True, sni="example.com")
+        assert p.request("get:'/p/200'").status_code == 200
+        assert all("Error in handle_sni" not in msg for msg in self.proxy.log)
+
+    def test_http(self):
+        p = self.pathoc(ssl=False)
+        assert p.request("get:'/p/200'").status_code == 400
+
+
+
 class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin):
     ssl = False
 
diff --git a/test/test_utils.py b/test/test_utils.py
index d99a146d..45bfb4f7 100644
--- a/test/test_utils.py
+++ b/test/test_utils.py
@@ -50,6 +50,19 @@ def test_urldecode():
     s = "one=two&three=four"
     assert len(utils.urldecode(s)) == 2
 
+def test_pretty_duration():
+    assert utils.pretty_duration(0.00001) == "0ms"
+    assert utils.pretty_duration(0.0001) == "0ms"
+    assert utils.pretty_duration(0.001) == "1ms"
+    assert utils.pretty_duration(0.01) == "10ms"
+    assert utils.pretty_duration(0.1) == "100ms"
+    assert utils.pretty_duration(1) == "1.00s"
+    assert utils.pretty_duration(10) == "10.0s"
+    assert utils.pretty_duration(100) == "100s"
+    assert utils.pretty_duration(1000) == "1000s"
+    assert utils.pretty_duration(10000) == "10000s"    
+    assert utils.pretty_duration(1.123) == "1.12s"
+    assert utils.pretty_duration(0.123) == "123ms"
 
 def test_LRUCache():
     class Foo:
diff --git a/test/tservers.py b/test/tservers.py
index 37929d1a..30c8b52e 100644
--- a/test/tservers.py
+++ b/test/tservers.py
@@ -218,12 +218,12 @@ class ReverseProxTest(ProxTestBase):
     @classmethod
     def get_proxy_config(cls):
         d = ProxTestBase.get_proxy_config()
-        d["upstream_server"] = (
+        d["upstream_server"] = [
             True if cls.ssl else False,
             True if cls.ssl else False,
             "127.0.0.1",
             cls.server.port
-        )
+        ]
         d["mode"] = "reverse"
         return d