Merge branch 'master' into 0.10

6 files changed, 61 insertions, 35 deletions

diff --git a/doc-src/transparent/osx.html b/doc-src/transparent/osx.html
index 77eea63b..205e4c76 100644
--- a/doc-src/transparent/osx.html
+++ b/doc-src/transparent/osx.html
@@ -67,3 +67,15 @@ rdr on en2 inet proto tcp to any port 443 -> 127.0.0.1 port 8080
 
 
 </ol>
+
+Note that the **rdr** rules in the pf.conf given above only apply to inbound
+traffic. This means that they will NOT redirect traffic coming from the box
+running pf itself. We can't distinguish between an outbound connection from a
+non-mitmproxy app, and an outbound connection from mitmproxy itself - if you
+want to intercept your OSX traffic, you should use an external host to run
+mitmproxy. None the less, pf is flexible to cater for a range of creative
+possibilities, like intercepting traffic emanating from VMs.  See the
+**pf.conf** man page for more.
+
+
+
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py
index 127affc9..4b5b0f5f 100644
--- a/libmproxy/cmdline.py
+++ b/libmproxy/cmdline.py
@@ -193,6 +193,11 @@ def common_options(parser):
         help="Reverse proxy to upstream server: http[s]://host[:port]"
     )
     parser.add_argument(
+        "-F",
+        action="store", dest="forward_proxy", default=None,
+        help="Proxy to unconditionally forward to: http[s]://host[:port]"
+    )
+    parser.add_argument(
         "-q",
         action="store_true", dest="quiet",
         help="Quiet."
diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py
index 006303a7..951d2c2a 100644
--- a/libmproxy/console/common.py
+++ b/libmproxy/console/common.py
@@ -190,7 +190,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2):
 
         delta = f.response.timestamp_end - f.response.timestamp_start
         size = len(f.response.content) + f.response.get_header_size()
-        rate = utils.pretty_size(size / delta)
+        rate = utils.pretty_size(size / ( delta if delta > 0 else 1 ) )
 
         d.update(dict(
             resp_code = f.response.code,
diff --git a/libmproxy/flow.py b/libmproxy/flow.py
index b6770806..96103ddb 100644
--- a/libmproxy/flow.py
+++ b/libmproxy/flow.py
@@ -1581,6 +1581,13 @@ class FlowMaster(controller.Master):
         self.run_script_hook("clientdisconnect", r)
         r.reply()
 
+    def handle_serverconnection(self, sc):
+        # To unify the mitmproxy script API, we call the script hook "serverconnect" rather than "serverconnection".
+        # As things are handled differently in libmproxy (ClientConnect + ClientDisconnect vs ServerConnection class),
+        # there is no "serverdisonnect" event at the moment.
+        self.run_script_hook("serverconnect", sc)
+        sc.reply()
+
     def handle_error(self, r):
         f = self.state.add_error(r)
         if f:
diff --git a/libmproxy/proxy.py b/libmproxy/proxy.py
index 4fdeeb65..ab2cdb7c 100644
--- a/libmproxy/proxy.py
+++ b/libmproxy/proxy.py
@@ -23,13 +23,14 @@ class Log:
 
 
 class ProxyConfig:
-    def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, transparent_proxy=None, authenticator=None):
+    def __init__(self, certfile = None, cacert = None, clientcerts = None, no_upstream_cert=False, body_size_limit = None, reverse_proxy=None, forward_proxy=None, transparent_proxy=None, authenticator=None):
         self.certfile = certfile
         self.cacert = cacert
         self.clientcerts = clientcerts
         self.no_upstream_cert = no_upstream_cert
         self.body_size_limit = body_size_limit
         self.reverse_proxy = reverse_proxy
+        self.forward_proxy = forward_proxy
         self.transparent_proxy = transparent_proxy
         self.authenticator = authenticator
         self.certstore = certutils.CertStore()
@@ -158,6 +159,7 @@ class ProxyHandler(tcp.BaseHandler):
         if not self.server_conn:
             try:
                 self.server_conn = ServerConnection(self.config, scheme, host, port, sni)
+                self.channel.ask(self.server_conn)
                 self.server_conn.connect()
             except tcp.NetLibError, v:
                 raise ProxyError(502, v)
@@ -219,7 +221,12 @@ class ProxyHandler(tcp.BaseHandler):
                     # the case, we want to reconnect without sending an error
                     # to the client.
                     while 1:
-                        sc = self.get_server_connection(cc, scheme, host, port, self.sni)
+                        if self.config.forward_proxy:
+                            forward_scheme, forward_host, forward_port = self.config.forward_proxy
+                            sc = self.get_server_connection(cc, forward_scheme, forward_host, forward_port, self.sni)
+                        else:
+                            sc = self.get_server_connection(cc, scheme, host, port, self.sni)
+                            
                         sc.send(request)
                         if sc.requestcount == 1: # add timestamps only for first request (others are not directly affected)
                             request.tcp_setup_timestamp = sc.tcp_setup_timestamp
@@ -258,13 +265,13 @@ class ProxyHandler(tcp.BaseHandler):
                 else:
                     response = response_reply
                     self.send_response(response)
-                    if request and http.request_connection_close(request.httpversion, request.headers):
+                    if request and http.connection_close(request.httpversion, request.headers):
                         return
                     # We could keep the client connection when the server
                     # connection needs to go away.  However, we want to mimic
                     # behaviour as closely as possible to the client, so we
                     # disconnect.
-                    if http.response_connection_close(response.httpversion, response.headers):
+                    if http.connection_close(response.httpversion, response.headers):
                         return
         except (IOError, ProxyError, http.HttpError, tcp.NetLibError), e:
             if hasattr(e, "code"):
@@ -310,6 +317,17 @@ class ProxyHandler(tcp.BaseHandler):
                 raise ProxyError(502, "Unable to generate dummy cert.")
             return ret
 
+    def establish_ssl(self, client_conn, host, port):
+        dummycert = self.find_cert(client_conn, host, port, host)
+        sni = HandleSNI(
+            self, client_conn, host, port,
+            dummycert, self.config.certfile or self.config.cacert
+        )
+        try:
+            self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)
+        except tcp.NetLibError, v:
+            raise ProxyError(400, str(v))
+
     def get_line(self, fp):
         """
             Get a line, possibly preceded by a blank.
@@ -329,15 +347,7 @@ class ProxyHandler(tcp.BaseHandler):
         if port in self.config.transparent_proxy["sslports"]:
             scheme = "https"
             if not self.ssl_established:
-                dummycert = self.find_cert(client_conn, host, port, host)
-                sni = HandleSNI(
-                    self, client_conn, host, port,
-                    dummycert, self.config.certfile or self.config.cacert
-                )
-                try:
-                    self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)
-                except tcp.NetLibError, v:
-                    raise ProxyError(400, str(v))
+                self.establish_ssl(client_conn, host, port)
         else:
             scheme = "http"
         line = self.get_line(self.rfile)
@@ -372,15 +382,7 @@ class ProxyHandler(tcp.BaseHandler):
                             '\r\n'
                             )
                 self.wfile.flush()
-                dummycert = self.find_cert(client_conn, host, port, host)
-                sni = HandleSNI(
-                    self, client_conn, host, port,
-                    dummycert, self.config.certfile or self.config.cacert
-                )
-                try:
-                    self.convert_to_ssl(dummycert, self.config.certfile or self.config.cacert, handle_sni=sni)
-                except tcp.NetLibError, v:
-                    raise ProxyError(400, str(v))
+                self.establish_ssl(client_conn, host, port)
                 self.proxy_connect_state = (host, port, httpversion)
                 line = self.rfile.readline(line)
 
@@ -414,10 +416,12 @@ class ProxyHandler(tcp.BaseHandler):
                     )
 
     def read_request_reverse(self, client_conn):
+        scheme, host, port = self.config.reverse_proxy
+        if scheme.lower() == "https" and not self.ssl_established:
+            self.establish_ssl(client_conn, host, port)
         line = self.get_line(self.rfile)
         if line == "":
             return None
-        scheme, host, port = self.config.reverse_proxy
         r = http.parse_init_http(line)
         if not r:
             raise ProxyError(400, "Bad HTTP request line: %s"%repr(line))
@@ -427,7 +431,7 @@ class ProxyHandler(tcp.BaseHandler):
                     self.rfile, self.wfile, headers, httpversion, self.config.body_size_limit
                 )
         return flow.Request(
-                    client_conn, httpversion, host, port, "http", method, path, headers, content,
+                    client_conn, httpversion, host, port, scheme, method, path, headers, content,
                     self.rfile.first_byte_timestamp, utils.timestamp()
                )
 
@@ -595,6 +599,13 @@ def process_proxy_options(parser, options):
     else:
         rp = None
 
+    if options.forward_proxy:
+        fp = utils.parse_proxy_spec(options.forward_proxy)
+        if not fp:
+            return parser.error("Invalid forward proxy specification: %s"%options.forward_proxy)
+    else:
+        fp = None
+
     if options.clientcerts:
         options.clientcerts = os.path.expanduser(options.clientcerts)
         if not os.path.exists(options.clientcerts) or not os.path.isdir(options.clientcerts):
@@ -624,6 +635,7 @@ def process_proxy_options(parser, options):
         body_size_limit = body_size_limit,
         no_upstream_cert = options.no_upstream_cert,
         reverse_proxy = rp,
+        forward_proxy = fp,
         transparent_proxy = trans,
         authenticator = authenticator
     )
diff --git a/test/test_server.py b/test/test_server.py
index 85c20737..d832ff18 100644
--- a/test/test_server.py
+++ b/test/test_server.py
@@ -191,16 +191,6 @@ class TestHTTPS(tservers.HTTPProxTest, CommonMixin):
         assert p.request("get:/:i0,'invalid\r\n\r\n'").status_code == 400
 
 
-class TestHTTPSNoUpstream(tservers.HTTPProxTest, CommonMixin):
-    ssl = True
-    no_upstream_cert = True
-    def test_cert_gen_error(self):
-        f = self.pathoc_raw()
-        f.connect((u"foo..bar".encode("utf8"), 0))
-        f.request("get:/")
-        assert "dummy cert" in "".join(self.proxy.log)
-
-
 class TestHTTPSCertfile(tservers.HTTPProxTest, CommonMixin):
     ssl = True
     certfile = True