From a653f314ff1c14b9f7acc5bfe1eaa78bcc4ad260 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Fri, 4 Nov 2016 23:01:46 +1300
Subject: proxy.protocol.http: flatten for refactoring

Flatten all of _process_flow, so we can see what's going on in there.
---
 mitmproxy/proxy/protocol/http.py | 339 +++++++++++++++++----------------------
 1 file changed, 150 insertions(+), 189 deletions(-)

diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index 5412827f..97105324 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -18,14 +18,6 @@ class _HttpTransmissionLayer(base.Layer):
     def read_request_body(self, request):
         raise NotImplementedError()
 
-    def read_request(self, f):
-        request = self.read_request_headers(f)
-        request.data.content = b"".join(
-            self.read_request_body(request)
-        )
-        request.timestamp_end = time.time()
-        return request
-
     def send_request(self, request):
         raise NotImplementedError()
 
@@ -146,9 +138,39 @@ class HttpLayer(base.Layer):
 
     def _process_flow(self, f):
         try:
-            request = self.get_request_from_client(f)
+            request = self.read_request_headers(f)
+            request.data.content = b"".join(
+                self.read_request_body(request)
+            )
+            request.timestamp_end = time.time()
+            f.request = request
+            self.channel.ask("requestheaders", f)
+            if request.headers.get("expect", "").lower() == "100-continue":
+                # TODO: We may have to use send_response_headers for HTTP2 here.
+                self.send_response(http.expect_continue_response)
+                request.headers.pop("expect")
+                request.content = b"".join(self.read_request_body(request))
+                request.timestamp_end = time.time()
+
             # Make sure that the incoming request matches our expectations
-            self.validate_request(request)
+            if request.first_line_format == "absolute" and request.scheme != "http":
+                raise exceptions.HttpException("Invalid request scheme: %s" % request.scheme)
+
+            expected_request_forms = {
+                "regular": ("authority", "absolute",),
+                "upstream": ("authority", "absolute"),
+                "transparent": ("relative",)
+            }
+
+            allowed_request_forms = expected_request_forms[self.mode]
+            if request.first_line_format not in allowed_request_forms:
+                err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
+                    " or ".join(allowed_request_forms), request.first_line_format
+                )
+                raise exceptions.HttpException(err_message)
+
+            if self.mode == "regular" and request.first_line_format == "absolute":
+                request.first_line_format = "relative"
         except exceptions.HttpReadDisconnect:
             # don't throw an error for disconnects that happen before/between requests.
             return False
@@ -174,7 +196,11 @@ class HttpLayer(base.Layer):
         try:
             # Regular Proxy Mode: Handle CONNECT
             if self.mode == "regular" and request.first_line_format == "authority":
-                self.handle_regular_mode_connect(request)
+                self.http_authenticated = True
+                self.set_server((request.host, request.port))
+                self.send_response(http.make_connect_response(request.data.http_version))
+                layer = self.ctx.next_layer(self)
+                layer()
                 return False
         except (exceptions.ProtocolException, exceptions.NetlibException) as e:
             # HTTPS tasting means that ordinary errors like resolution and
@@ -191,7 +217,25 @@ class HttpLayer(base.Layer):
         # set upstream auth
         if self.mode == "upstream" and self.config.upstream_auth is not None:
             f.request.headers["Proxy-Authorization"] = self.config.upstream_auth
-        self.process_request_hook(f)
+
+        # Determine .scheme, .host and .port attributes for inline scripts.
+        # For absolute-form requests, they are directly given in the request.
+        # For authority-form requests, we only need to determine the request scheme.
+        # For relative-form requests, we need to determine host and port as
+        # well.
+        if self.mode == "regular":
+            pass  # only absolute-form at this point, nothing to do here.
+        elif self.mode == "upstream":
+            pass
+        else:
+            # Setting request.host also updates the host header, which we want to preserve
+            host_header = f.request.headers.get("host", None)
+            f.request.host = self.__initial_server_conn.address.host
+            f.request.port = self.__initial_server_conn.address.port
+            if host_header:
+                f.request.headers["host"] = host_header
+            f.request.scheme = "https" if self.__initial_server_tls else "http"
+        self.channel.ask("request", f)
 
         try:
             if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers):
@@ -205,7 +249,54 @@ class HttpLayer(base.Layer):
                     f.request.port,
                     f.request.scheme
                 )
-                self.get_response_from_server(f)
+
+                def get_response():
+                    self.send_request(f.request)
+                    f.response = self.read_response_headers()
+
+                try:
+                    get_response()
+                except exceptions.NetlibException as e:
+                    self.log(
+                        "server communication error: %s" % repr(e),
+                        level="debug"
+                    )
+                    # In any case, we try to reconnect at least once. This is
+                    # necessary because it might be possible that we already
+                    # initiated an upstream connection after clientconnect that
+                    # has already been expired, e.g consider the following event
+                    # log:
+                    # > clientconnect (transparent mode destination known)
+                    # > serverconnect (required for client tls handshake)
+                    # > read n% of large request
+                    # > server detects timeout, disconnects
+                    # > read (100-n)% of large request
+                    # > send large request upstream
+
+                    if isinstance(e, exceptions.Http2ProtocolException):
+                        # do not try to reconnect for HTTP2
+                        raise exceptions.ProtocolException("First and only attempt to get response via HTTP2 failed.")
+
+                    self.disconnect()
+                    self.connect()
+                    get_response()
+
+                # call the appropriate script hook - this is an opportunity for an
+                # inline script to set f.stream = True
+                self.channel.ask("responseheaders", f)
+
+                if f.response.stream:
+                    f.response.data.content = None
+                else:
+                    f.response.data.content = b"".join(self.read_response_body(
+                        f.request,
+                        f.response
+                    ))
+                f.response.timestamp_end = time.time()
+
+                # no further manipulation of self.server_conn beyond this point
+                # we can safely set it as the final attribute value here.
+                f.server_conn = self.server_conn
             else:
                 # response was set by an inline script.
                 # we now need to emulate the responseheaders hook.
@@ -213,19 +304,57 @@ class HttpLayer(base.Layer):
 
             self.log("response", "debug", [repr(f.response)])
             self.channel.ask("response", f)
-            self.send_response_to_client(f)
+
+            if not f.response.stream:
+                # no streaming:
+                # we already received the full response from the server and can
+                # send it to the client straight away.
+                self.send_response(f.response)
+            else:
+                # streaming:
+                # First send the headers and then transfer the response incrementally
+                self.send_response_headers(f.response)
+                chunks = self.read_response_body(
+                    f.request,
+                    f.response
+                )
+                if callable(f.response.stream):
+                    chunks = f.response.stream(chunks)
+                self.send_response_body(f.response, chunks)
+                f.response.timestamp_end = time.time()
 
             if self.check_close_connection(f):
                 return False
 
             # Handle 101 Switching Protocols
             if f.response.status_code == 101:
-                self.handle_101_switching_protocols(f)
+                """
+                    Handle a successful HTTP 101 Switching Protocols Response, received after
+                    e.g. a WebSocket upgrade request.
+                """
+                # Check for WebSockets handshake
+                is_websockets = (
+                    f and
+                    websockets.check_handshake(f.request.headers) and
+                    websockets.check_handshake(f.response.headers)
+                )
+                if is_websockets and not self.config.options.websockets:
+                    self.log(
+                        "Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.",
+                        "info"
+                    )
+
+                if is_websockets and self.config.options.websockets:
+                    layer = pwebsockets.WebSocketsLayer(self, f)
+                else:
+                    layer = self.ctx.next_layer(self)
+                layer()
                 return False  # should never be reached
 
             # Upstream Proxy Mode: Handle CONNECT
             if f.request.first_line_format == "authority" and f.response.status_code == 200:
-                self.handle_upstream_mode_connect(f.request.copy())
+                layer = UpstreamConnectLayer(self, f.request)
+                layer()
                 return False
 
         except (exceptions.ProtocolException, exceptions.NetlibException) as e:
@@ -244,131 +373,20 @@ class HttpLayer(base.Layer):
 
         return True
 
-    def get_request_from_client(self, f):
-        request = self.read_request(f)
-        f.request = request
-        self.channel.ask("requestheaders", f)
-        if request.headers.get("expect", "").lower() == "100-continue":
-            # TODO: We may have to use send_response_headers for HTTP2 here.
-            self.send_response(http.expect_continue_response)
-            request.headers.pop("expect")
-            request.content = b"".join(self.read_request_body(request))
-            request.timestamp_end = time.time()
-        return request
-
-    def send_error_response(self, code, message, headers=None):
+    def send_error_response(self, code, message, headers=None) -> None:
         try:
             response = http.make_error_response(code, message, headers)
             self.send_response(response)
         except (exceptions.NetlibException, h2.exceptions.H2Error, exceptions.Http2ProtocolException):
             self.log(traceback.format_exc(), "debug")
 
-    def change_upstream_proxy_server(self, address):
+    def change_upstream_proxy_server(self, address) -> None:
         # Make set_upstream_proxy_server always available,
         # even if there's no UpstreamConnectLayer
         if address != self.server_conn.address:
-            return self.set_server(address)
+            self.set_server(address)
 
-    def handle_regular_mode_connect(self, request):
-        self.http_authenticated = True
-        self.set_server((request.host, request.port))
-        self.send_response(http.make_connect_response(request.data.http_version))
-        layer = self.ctx.next_layer(self)
-        layer()
-
-    def handle_upstream_mode_connect(self, connect_request):
-        layer = UpstreamConnectLayer(self, connect_request)
-        layer()
-
-    def send_response_to_client(self, f):
-        if not f.response.stream:
-            # no streaming:
-            # we already received the full response from the server and can
-            # send it to the client straight away.
-            self.send_response(f.response)
-        else:
-            # streaming:
-            # First send the headers and then transfer the response incrementally
-            self.send_response_headers(f.response)
-            chunks = self.read_response_body(
-                f.request,
-                f.response
-            )
-            if callable(f.response.stream):
-                chunks = f.response.stream(chunks)
-            self.send_response_body(f.response, chunks)
-            f.response.timestamp_end = time.time()
-
-    def get_response_from_server(self, f):
-        def get_response():
-            self.send_request(f.request)
-            f.response = self.read_response_headers()
-
-        try:
-            get_response()
-        except exceptions.NetlibException as e:
-            self.log(
-                "server communication error: %s" % repr(e),
-                level="debug"
-            )
-            # In any case, we try to reconnect at least once. This is
-            # necessary because it might be possible that we already
-            # initiated an upstream connection after clientconnect that
-            # has already been expired, e.g consider the following event
-            # log:
-            # > clientconnect (transparent mode destination known)
-            # > serverconnect (required for client tls handshake)
-            # > read n% of large request
-            # > server detects timeout, disconnects
-            # > read (100-n)% of large request
-            # > send large request upstream
-
-            if isinstance(e, exceptions.Http2ProtocolException):
-                # do not try to reconnect for HTTP2
-                raise exceptions.ProtocolException("First and only attempt to get response via HTTP2 failed.")
-
-            self.disconnect()
-            self.connect()
-            get_response()
-
-        # call the appropriate script hook - this is an opportunity for an
-        # inline script to set f.stream = True
-        self.channel.ask("responseheaders", f)
-
-        if f.response.stream:
-            f.response.data.content = None
-        else:
-            f.response.data.content = b"".join(self.read_response_body(
-                f.request,
-                f.response
-            ))
-        f.response.timestamp_end = time.time()
-
-        # no further manipulation of self.server_conn beyond this point
-        # we can safely set it as the final attribute value here.
-        f.server_conn = self.server_conn
-
-    def process_request_hook(self, f):
-        # Determine .scheme, .host and .port attributes for inline scripts.
-        # For absolute-form requests, they are directly given in the request.
-        # For authority-form requests, we only need to determine the request scheme.
-        # For relative-form requests, we need to determine host and port as
-        # well.
-        if self.mode == "regular":
-            pass  # only absolute-form at this point, nothing to do here.
-        elif self.mode == "upstream":
-            pass
-        else:
-            # Setting request.host also updates the host header, which we want to preserve
-            host_header = f.request.headers.get("host", None)
-            f.request.host = self.__initial_server_conn.address.host
-            f.request.port = self.__initial_server_conn.address.port
-            if host_header:
-                f.request.headers["host"] = host_header
-            f.request.scheme = "https" if self.__initial_server_tls else "http"
-        self.channel.ask("request", f)
-
-    def establish_server_connection(self, host, port, scheme):
+    def establish_server_connection(self, host: str, port: int, scheme: str):
         address = tcp.Address((host, port))
         tls = (scheme == "https")
 
@@ -385,42 +403,8 @@ class HttpLayer(base.Layer):
                 self.connect()
             if tls:
                 raise exceptions.HttpProtocolException("Cannot change scheme in upstream proxy mode.")
-            """
-            # This is a very ugly (untested) workaround to solve a very ugly problem.
-            if self.server_conn and self.server_conn.tls_established and not ssl:
-                self.disconnect()
-                self.connect()
-            elif ssl and not hasattr(self, "connected_to") or self.connected_to != address:
-                if self.server_conn.tls_established:
-                    self.disconnect()
-                    self.connect()
-
-                self.send_request(make_connect_request(address))
-                tls_layer = TlsLayer(self, False, True)
-                tls_layer._establish_tls_with_server()
-            """
-
-    def validate_request(self, request):
-        if request.first_line_format == "absolute" and request.scheme != "http":
-            raise exceptions.HttpException("Invalid request scheme: %s" % request.scheme)
-
-        expected_request_forms = {
-            "regular": ("authority", "absolute",),
-            "upstream": ("authority", "absolute"),
-            "transparent": ("relative",)
-        }
-
-        allowed_request_forms = expected_request_forms[self.mode]
-        if request.first_line_format not in allowed_request_forms:
-            err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
-                " or ".join(allowed_request_forms), request.first_line_format
-            )
-            raise exceptions.HttpException(err_message)
 
-        if self.mode == "regular" and request.first_line_format == "absolute":
-            request.first_line_format = "relative"
-
-    def authenticate(self, request):
+    def authenticate(self, request) -> bool:
         if self.config.authenticator:
             if self.config.authenticator.authenticate(request.headers):
                 self.config.authenticator.clean(request.headers)
@@ -439,26 +423,3 @@ class HttpLayer(base.Layer):
                     ))
                 return False
         return True
-
-    def handle_101_switching_protocols(self, f):
-        """
-        Handle a successful HTTP 101 Switching Protocols Response, received after e.g. a WebSocket upgrade request.
-        """
-        # Check for WebSockets handshake
-        is_websockets = (
-            f and
-            websockets.check_handshake(f.request.headers) and
-            websockets.check_handshake(f.response.headers)
-        )
-        if is_websockets and not self.config.options.websockets:
-            self.log(
-                "Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.",
-                "info"
-            )
-
-        if is_websockets and self.config.options.websockets:
-            layer = pwebsockets.WebSocketsLayer(self, f)
-        else:
-            layer = self.ctx.next_layer(self)
-
-        layer()
-- 
cgit v1.2.3


From 4eea265925cc775f7d8023744b3147881aa17b6b Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 5 Nov 2016 08:09:46 +1300
Subject: Remove unused protocol attribute on connections.

---
 mitmproxy/connections.py | 2 --
 1 file changed, 2 deletions(-)

diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py
index f3a75222..c7941ad9 100644
--- a/mitmproxy/connections.py
+++ b/mitmproxy/connections.py
@@ -42,7 +42,6 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
         self.timestamp_start = time.time()
         self.timestamp_end = None
         self.timestamp_ssl_setup = None
-        self.protocol = None
         self.sni = None
         self.cipher_name = None
         self.tls_version = None
@@ -144,7 +143,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
         self.timestamp_end = None
         self.timestamp_tcp_setup = None
         self.timestamp_ssl_setup = None
-        self.protocol = None
 
     def connected(self):
         return bool(self.connection) and not self.finished
-- 
cgit v1.2.3


From 53b77fc47580d110b02f1ea4bcdf7d0cf73fc4b2 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 5 Nov 2016 09:37:54 +1300
Subject: proxy.protocol.http: cleanups, extract request validation

---
 mitmproxy/proxy/protocol/http.py | 63 ++++++++++++++++++++++------------------
 1 file changed, 35 insertions(+), 28 deletions(-)

diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index 97105324..17d09b07 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -112,12 +112,34 @@ class UpstreamConnectLayer(base.Layer):
         self.server_conn.address = address
 
 
+FIRSTLINES = set(["absolute", "relative", "authority"])
+# At this point, we see only a subset of the proxy modes
+MODES = set(["regular", "transparent", "upstream"])
+MODE_REQUEST_FORMS = {
+    "regular": ("authority", "absolute"),
+    "transparent": ("relative"),
+    "upstream": ("authority", "absolute"),
+}
+
+
+def validate_request_form(mode, request):
+    if request.first_line_format == "absolute" and request.scheme != "http":
+        raise exceptions.HttpException("Invalid request scheme: %s" % request.scheme)
+    allowed_request_forms = MODE_REQUEST_FORMS[mode]
+    if request.first_line_format not in allowed_request_forms:
+        err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
+            " or ".join(allowed_request_forms), request.first_line_format
+        )
+        raise exceptions.HttpException(err_message)
+
+
 class HttpLayer(base.Layer):
 
     def __init__(self, ctx, mode):
         super().__init__(ctx)
+        if mode not in MODES:
+            raise exceptions.ServerException("Invalid mode: %s"%mode)
         self.mode = mode
-        self.flow = None  # type: http.HTTPFlow
         self.__initial_server_conn = None
         "Contains the original destination in transparent mode, which needs to be restored"
         "if an inline script modified the target server for a single http request"
@@ -125,23 +147,21 @@ class HttpLayer(base.Layer):
         # see https://github.com/mitmproxy/mitmproxy/issues/925
         self.__initial_server_tls = None
         # Requests happening after CONNECT do not need Proxy-Authorization headers.
-        self.http_authenticated = False
+        self.connect_request = False
 
     def __call__(self):
         if self.mode == "transparent":
             self.__initial_server_tls = self.server_tls
             self.__initial_server_conn = self.server_conn
         while True:
-            self.flow = http.HTTPFlow(self.client_conn, self.server_conn, live=self)
-            if not self._process_flow(self.flow):
+            flow = http.HTTPFlow(self.client_conn, self.server_conn, live=self)
+            if not self._process_flow(flow):
                 return
 
     def _process_flow(self, f):
         try:
             request = self.read_request_headers(f)
-            request.data.content = b"".join(
-                self.read_request_body(request)
-            )
+            request.data.content = b"".join(self.read_request_body(request))
             request.timestamp_end = time.time()
             f.request = request
             self.channel.ask("requestheaders", f)
@@ -152,25 +172,11 @@ class HttpLayer(base.Layer):
                 request.content = b"".join(self.read_request_body(request))
                 request.timestamp_end = time.time()
 
-            # Make sure that the incoming request matches our expectations
-            if request.first_line_format == "absolute" and request.scheme != "http":
-                raise exceptions.HttpException("Invalid request scheme: %s" % request.scheme)
-
-            expected_request_forms = {
-                "regular": ("authority", "absolute",),
-                "upstream": ("authority", "absolute"),
-                "transparent": ("relative",)
-            }
-
-            allowed_request_forms = expected_request_forms[self.mode]
-            if request.first_line_format not in allowed_request_forms:
-                err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
-                    " or ".join(allowed_request_forms), request.first_line_format
-                )
-                raise exceptions.HttpException(err_message)
+            validate_request_form(self.mode, request)
 
             if self.mode == "regular" and request.first_line_format == "absolute":
                 request.first_line_format = "relative"
+
         except exceptions.HttpReadDisconnect:
             # don't throw an error for disconnects that happen before/between requests.
             return False
@@ -188,7 +194,7 @@ class HttpLayer(base.Layer):
         # Proxy Authentication conceptually does not work in transparent mode.
         # We catch this misconfiguration on startup. Here, we sort out requests
         # after a successful CONNECT request (which do not need to be validated anymore)
-        if not (self.http_authenticated or self.authenticate(request)):
+        if not self.connect_request and not self.authenticate(request):
             return False
 
         f.request = request
@@ -196,7 +202,7 @@ class HttpLayer(base.Layer):
         try:
             # Regular Proxy Mode: Handle CONNECT
             if self.mode == "regular" and request.first_line_format == "authority":
-                self.http_authenticated = True
+                self.connect_request = True
                 self.set_server((request.host, request.port))
                 self.send_response(http.make_connect_response(request.data.http_version))
                 layer = self.ctx.next_layer(self)
@@ -275,7 +281,9 @@ class HttpLayer(base.Layer):
 
                     if isinstance(e, exceptions.Http2ProtocolException):
                         # do not try to reconnect for HTTP2
-                        raise exceptions.ProtocolException("First and only attempt to get response via HTTP2 failed.")
+                        raise exceptions.ProtocolException(
+                            "First and only attempt to get response via HTTP2 failed."
+                        )
 
                     self.disconnect()
                     self.connect()
@@ -334,13 +342,12 @@ class HttpLayer(base.Layer):
                 """
                 # Check for WebSockets handshake
                 is_websockets = (
-                    f and
                     websockets.check_handshake(f.request.headers) and
                     websockets.check_handshake(f.response.headers)
                 )
                 if is_websockets and not self.config.options.websockets:
                     self.log(
-                        "Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.",
+                        "Client requested WebSocket connection, but the protocol is disabled.",
                         "info"
                     )
 
-- 
cgit v1.2.3


From 82ac7d05a65f5bfd95a20112da09e2dc40960f07 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 5 Nov 2016 10:10:59 +1300
Subject: Bug: ask requestheaders before request body is read

Also add the beginnings of a test suite to exercise issues like this.
---
 mitmproxy/proxy/protocol/http.py     |  5 ++--
 test/mitmproxy/test_eventsequence.py | 48 ++++++++++++++++++++++++++++++++++++
 2 files changed, 51 insertions(+), 2 deletions(-)
 create mode 100644 test/mitmproxy/test_eventsequence.py

diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index 17d09b07..4caaf1e3 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -161,10 +161,11 @@ class HttpLayer(base.Layer):
     def _process_flow(self, f):
         try:
             request = self.read_request_headers(f)
-            request.data.content = b"".join(self.read_request_body(request))
-            request.timestamp_end = time.time()
             f.request = request
             self.channel.ask("requestheaders", f)
+
+            request.data.content = b"".join(self.read_request_body(request))
+            request.timestamp_end = time.time()
             if request.headers.get("expect", "").lower() == "100-continue":
                 # TODO: We may have to use send_response_headers for HTTP2 here.
                 self.send_response(http.expect_continue_response)
diff --git a/test/mitmproxy/test_eventsequence.py b/test/mitmproxy/test_eventsequence.py
new file mode 100644
index 00000000..7fdbce1b
--- /dev/null
+++ b/test/mitmproxy/test_eventsequence.py
@@ -0,0 +1,48 @@
+from mitmproxy import events
+import contextlib
+from . import tservers
+
+
+class EAddon:
+    def __init__(self, handlers):
+        self.failure = None
+        self.handlers = handlers
+        for i in events.Events:
+            def mkprox():
+                evt = i
+
+                def prox(*args, **kwargs):
+                    if evt in self.handlers:
+                        try:
+                            handlers[evt](*args, **kwargs)
+                        except AssertionError as e:
+                            self.failure = e
+                return prox
+            setattr(self, i, mkprox())
+
+    def fail(self):
+        pass
+
+
+class SequenceTester:
+    @contextlib.contextmanager
+    def events(self, **kwargs):
+        m = EAddon(kwargs)
+        self.master.addons.add(m)
+        yield
+        self.master.addons.remove(m)
+        if m.failure:
+            raise m.failure
+
+
+class TestBasic(tservers.HTTPProxyTest, SequenceTester):
+    def test_requestheaders(self):
+
+        def req(f):
+            assert f.request.headers
+            assert not f.request.content
+
+        with self.events(requestheaders=req):
+            p = self.pathoc()
+            with p.connect():
+                assert p.request("get:'%s/p/200':b@10" % self.server.urlbase).status_code == 200
-- 
cgit v1.2.3


From 39589404209a9980c0a07137f367f70c103e3113 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 5 Nov 2016 10:59:56 +1300
Subject: Test failure during 100-continue

Also:

- Remove duplicate and unused code
- Tighten scope of HttpReadDisconnect handler - we only want to ignore this for
the initial read, not for the entire block that includes things like the expect
handling.
---
 mitmproxy/proxy/modes/socks_proxy.py |  3 ---
 mitmproxy/proxy/protocol/http.py     | 21 ++++++++----------
 test/mitmproxy/test_eventsequence.py | 42 ++++++++++++++++++++++++++----------
 3 files changed, 40 insertions(+), 26 deletions(-)

diff --git a/mitmproxy/proxy/modes/socks_proxy.py b/mitmproxy/proxy/modes/socks_proxy.py
index 04258037..3121b731 100644
--- a/mitmproxy/proxy/modes/socks_proxy.py
+++ b/mitmproxy/proxy/modes/socks_proxy.py
@@ -5,9 +5,6 @@ from mitmproxy.net import socks
 
 class Socks5Proxy(protocol.Layer, protocol.ServerConnectionMixin):
 
-    def __init__(self, ctx):
-        super().__init__(ctx)
-
     def __call__(self):
         try:
             # Parse Client Greeting
diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index 4caaf1e3..9fe83ff6 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -160,27 +160,24 @@ class HttpLayer(base.Layer):
 
     def _process_flow(self, f):
         try:
-            request = self.read_request_headers(f)
+            try:
+                request = self.read_request_headers(f)
+            except exceptions.HttpReadDisconnect:
+                # don't throw an error for disconnects that happen before/between requests.
+                return False
+
             f.request = request
             self.channel.ask("requestheaders", f)
 
-            request.data.content = b"".join(self.read_request_body(request))
-            request.timestamp_end = time.time()
             if request.headers.get("expect", "").lower() == "100-continue":
                 # TODO: We may have to use send_response_headers for HTTP2 here.
                 self.send_response(http.expect_continue_response)
                 request.headers.pop("expect")
-                request.content = b"".join(self.read_request_body(request))
-                request.timestamp_end = time.time()
-
-            validate_request_form(self.mode, request)
 
-            if self.mode == "regular" and request.first_line_format == "absolute":
-                request.first_line_format = "relative"
+            request.data.content = b"".join(self.read_request_body(request))
+            request.timestamp_end = time.time()
 
-        except exceptions.HttpReadDisconnect:
-            # don't throw an error for disconnects that happen before/between requests.
-            return False
+            validate_request_form(self.mode, request)
         except exceptions.HttpException as e:
             # We optimistically guess there might be an HTTP client on the
             # other end
diff --git a/test/mitmproxy/test_eventsequence.py b/test/mitmproxy/test_eventsequence.py
index 7fdbce1b..31c57e82 100644
--- a/test/mitmproxy/test_eventsequence.py
+++ b/test/mitmproxy/test_eventsequence.py
@@ -3,15 +3,17 @@ import contextlib
 from . import tservers
 
 
-class EAddon:
-    def __init__(self, handlers):
+class Eventer:
+    def __init__(self, **handlers):
         self.failure = None
+        self.called = []
         self.handlers = handlers
-        for i in events.Events:
+        for i in events.Events - {"tick"}:
             def mkprox():
                 evt = i
 
                 def prox(*args, **kwargs):
+                    self.called.append(evt)
                     if evt in self.handlers:
                         try:
                             handlers[evt](*args, **kwargs)
@@ -26,23 +28,41 @@ class EAddon:
 
 class SequenceTester:
     @contextlib.contextmanager
-    def events(self, **kwargs):
-        m = EAddon(kwargs)
-        self.master.addons.add(m)
+    def addon(self, addon):
+        self.master.addons.add(addon)
         yield
-        self.master.addons.remove(m)
-        if m.failure:
-            raise m.failure
+        self.master.addons.remove(addon)
+        if addon.failure:
+            raise addon.failure
 
 
 class TestBasic(tservers.HTTPProxyTest, SequenceTester):
     def test_requestheaders(self):
 
-        def req(f):
+        def hdrs(f):
             assert f.request.headers
             assert not f.request.content
 
-        with self.events(requestheaders=req):
+        def req(f):
+            assert f.request.headers
+            assert f.request.content
+
+        with self.addon(Eventer(requestheaders=hdrs, request=req)):
             p = self.pathoc()
             with p.connect():
                 assert p.request("get:'%s/p/200':b@10" % self.server.urlbase).status_code == 200
+
+    def test_100_continue_fail(self):
+        e = Eventer()
+        with self.addon(e):
+            p = self.pathoc()
+            with p.connect():
+                p.request(
+                    """
+                        get:'%s/p/200'
+                        h'expect'='100-continue'
+                        h'content-length'='1000'
+                        da
+                    """ % self.server.urlbase
+                )
+            assert e.called[-1] == "requestheaders"
-- 
cgit v1.2.3


From fbaade429845546d751110caa0f886f7b1a62717 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 5 Nov 2016 16:20:38 +1300
Subject: Remove promotion to raw TCP based on heuristics

This seems terribly dangerous to me. Let's expand explicit control instead.
---
 mitmproxy/proxy/protocol/http.py |  2 +-
 mitmproxy/proxy/root_context.py  | 11 +----------
 2 files changed, 2 insertions(+), 11 deletions(-)

diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index 9fe83ff6..3bc33ab0 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -138,7 +138,7 @@ class HttpLayer(base.Layer):
     def __init__(self, ctx, mode):
         super().__init__(ctx)
         if mode not in MODES:
-            raise exceptions.ServerException("Invalid mode: %s"%mode)
+            raise exceptions.ServerException("Invalid mode: %s" % mode)
         self.mode = mode
         self.__initial_server_conn = None
         "Contains the original destination in transparent mode, which needs to be restored"
diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py
index eacf7e0b..4362347b 100644
--- a/mitmproxy/proxy/root_context.py
+++ b/mitmproxy/proxy/root_context.py
@@ -90,16 +90,7 @@ class RootContext:
             if alpn == b'http/1.1':
                 return protocol.Http1Layer(top_layer, 'transparent')
 
-        # 6. Check for raw tcp mode
-        is_ascii = (
-            len(d) == 3 and
-            # expect A-Za-z
-            all(65 <= x <= 90 or 97 <= x <= 122 for x in d)
-        )
-        if self.config.options.rawtcp and not is_ascii:
-            return protocol.RawTCPLayer(top_layer)
-
-        # 7. Assume HTTP1 by default
+        # 6. Assume HTTP1 by default
         return protocol.Http1Layer(top_layer, 'transparent')
 
     def log(self, msg, level, subs=()):
-- 
cgit v1.2.3


From 5be35d258fd95f9f99ee6e8a32dc234f6b868385 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 12 Nov 2016 10:06:57 +1300
Subject: Use an enum for http protocol modes

---
 mitmproxy/proxy/protocol/http.py | 33 +++++++++++++++++----------------
 mitmproxy/proxy/root_context.py  | 11 ++++++-----
 2 files changed, 23 insertions(+), 21 deletions(-)

diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index 3bc33ab0..e974ffe2 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -1,6 +1,8 @@
 import h2.exceptions
 import time
 import traceback
+import enum
+
 from mitmproxy import exceptions
 from mitmproxy import http
 from mitmproxy import flow
@@ -112,13 +114,18 @@ class UpstreamConnectLayer(base.Layer):
         self.server_conn.address = address
 
 
+class HTTPMode(enum.Enum):
+    regular = 1
+    transparent = 2
+    upstream = 3
+
+
 FIRSTLINES = set(["absolute", "relative", "authority"])
 # At this point, we see only a subset of the proxy modes
-MODES = set(["regular", "transparent", "upstream"])
 MODE_REQUEST_FORMS = {
-    "regular": ("authority", "absolute"),
-    "transparent": ("relative"),
-    "upstream": ("authority", "absolute"),
+    HTTPMode.regular: ("authority", "absolute"),
+    HTTPMode.transparent: ("relative"),
+    HTTPMode.upstream: ("authority", "absolute"),
 }
 
 
@@ -137,8 +144,6 @@ class HttpLayer(base.Layer):
 
     def __init__(self, ctx, mode):
         super().__init__(ctx)
-        if mode not in MODES:
-            raise exceptions.ServerException("Invalid mode: %s" % mode)
         self.mode = mode
         self.__initial_server_conn = None
         "Contains the original destination in transparent mode, which needs to be restored"
@@ -150,7 +155,7 @@ class HttpLayer(base.Layer):
         self.connect_request = False
 
     def __call__(self):
-        if self.mode == "transparent":
+        if self.mode == HTTPMode.transparent:
             self.__initial_server_tls = self.server_tls
             self.__initial_server_conn = self.server_conn
         while True:
@@ -199,7 +204,7 @@ class HttpLayer(base.Layer):
 
         try:
             # Regular Proxy Mode: Handle CONNECT
-            if self.mode == "regular" and request.first_line_format == "authority":
+            if self.mode is HTTPMode.regular and request.first_line_format == "authority":
                 self.connect_request = True
                 self.set_server((request.host, request.port))
                 self.send_response(http.make_connect_response(request.data.http_version))
@@ -219,7 +224,7 @@ class HttpLayer(base.Layer):
             f.request.headers["Host"] = self.config.upstream_server.address.host
 
         # set upstream auth
-        if self.mode == "upstream" and self.config.upstream_auth is not None:
+        if self.mode is HTTPMode.upstream and self.config.upstream_auth is not None:
             f.request.headers["Proxy-Authorization"] = self.config.upstream_auth
 
         # Determine .scheme, .host and .port attributes for inline scripts.
@@ -227,11 +232,7 @@ class HttpLayer(base.Layer):
         # For authority-form requests, we only need to determine the request scheme.
         # For relative-form requests, we need to determine host and port as
         # well.
-        if self.mode == "regular":
-            pass  # only absolute-form at this point, nothing to do here.
-        elif self.mode == "upstream":
-            pass
-        else:
+        if self.mode is HTTPMode.transparent:
             # Setting request.host also updates the host header, which we want to preserve
             host_header = f.request.headers.get("host", None)
             f.request.host = self.__initial_server_conn.address.host
@@ -395,7 +396,7 @@ class HttpLayer(base.Layer):
         address = tcp.Address((host, port))
         tls = (scheme == "https")
 
-        if self.mode == "regular" or self.mode == "transparent":
+        if self.mode is HTTPMode.regular or self.mode is HTTPMode.transparent:
             # If there's an existing connection that doesn't match our expectations, kill it.
             if address != self.server_conn.address or tls != self.server_tls:
                 self.set_server(address)
@@ -414,7 +415,7 @@ class HttpLayer(base.Layer):
             if self.config.authenticator.authenticate(request.headers):
                 self.config.authenticator.clean(request.headers)
             else:
-                if self.mode == "transparent":
+                if self.mode == HTTPMode.transparent:
                     self.send_response(http.make_error_response(
                         401,
                         "Authentication Required",
diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py
index 4362347b..50dbe79e 100644
--- a/mitmproxy/proxy/root_context.py
+++ b/mitmproxy/proxy/root_context.py
@@ -2,6 +2,7 @@ from mitmproxy import log
 from mitmproxy import exceptions
 from mitmproxy.proxy import protocol
 from mitmproxy.proxy import modes
+from mitmproxy.proxy.protocol import http
 
 
 class RootContext:
@@ -70,9 +71,9 @@ class RootContext:
         # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed.
         if isinstance(top_layer, protocol.TlsLayer):
             if isinstance(top_layer.ctx, modes.HttpProxy):
-                return protocol.Http1Layer(top_layer, "regular")
+                return protocol.Http1Layer(top_layer, http.HTTPMode.regular)
             if isinstance(top_layer.ctx, modes.HttpUpstreamProxy):
-                return protocol.Http1Layer(top_layer, "upstream")
+                return protocol.Http1Layer(top_layer, http.HTTPMode.upstream)
 
         # 4. Check for other TLS cases (e.g. after CONNECT).
         if client_tls:
@@ -86,12 +87,12 @@ class RootContext:
         if isinstance(top_layer, protocol.TlsLayer):
             alpn = top_layer.client_conn.get_alpn_proto_negotiated()
             if alpn == b'h2':
-                return protocol.Http2Layer(top_layer, 'transparent')
+                return protocol.Http2Layer(top_layer, http.HTTPMode.transparent)
             if alpn == b'http/1.1':
-                return protocol.Http1Layer(top_layer, 'transparent')
+                return protocol.Http1Layer(top_layer, http.HTTPMode.transparent)
 
         # 6. Assume HTTP1 by default
-        return protocol.Http1Layer(top_layer, 'transparent')
+        return protocol.Http1Layer(top_layer, http.HTTPMode.transparent)
 
     def log(self, msg, level, subs=()):
         """
-- 
cgit v1.2.3


From 00492919e7fe47c504e363fe9d5e461cf8f8967b Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 12 Nov 2016 10:59:57 +1300
Subject: Add HTTPFlow.mode to record the HTTP proxy layer mode

---
 mitmproxy/http.py                | 9 ++++++---
 mitmproxy/io_compat.py           | 1 +
 mitmproxy/proxy/protocol/http.py | 7 ++++++-
 3 files changed, 13 insertions(+), 4 deletions(-)

diff --git a/mitmproxy/http.py b/mitmproxy/http.py
index 50174764..dafd4782 100644
--- a/mitmproxy/http.py
+++ b/mitmproxy/http.py
@@ -53,7 +53,7 @@ class HTTPRequest(http.Request):
     def get_state(self):
         state = super().get_state()
         state.update(
-            is_replay=self.is_replay,
+            is_replay=self.is_replay
         )
         return state
 
@@ -143,7 +143,7 @@ class HTTPFlow(flow.Flow):
     transaction.
     """
 
-    def __init__(self, client_conn, server_conn, live=None):
+    def __init__(self, client_conn, server_conn, live=None, mode="regular"):
         super().__init__("http", client_conn, server_conn, live)
 
         self.request = None  # type: HTTPRequest
@@ -163,11 +163,14 @@ class HTTPFlow(flow.Flow):
         """:py:class:`ClientConnection` object """
         self.intercepted = False  # type: bool
         """ Is this flow currently being intercepted? """
+        self.mode = mode
+        """ What mode was the proxy layer in when receiving this request? """
 
     _stateobject_attributes = flow.Flow._stateobject_attributes.copy()
     _stateobject_attributes.update(
         request=HTTPRequest,
-        response=HTTPResponse
+        response=HTTPResponse,
+        mode=str
     )
 
     def __repr__(self):
diff --git a/mitmproxy/io_compat.py b/mitmproxy/io_compat.py
index b1b5a296..20ee8824 100644
--- a/mitmproxy/io_compat.py
+++ b/mitmproxy/io_compat.py
@@ -69,6 +69,7 @@ def convert_018_019(data):
     data["client_conn"]["sni"] = None
     data["client_conn"]["cipher_name"] = None
     data["client_conn"]["tls_version"] = None
+    data["mode"] = "regular"
     data["metadata"] = dict()
     return data
 
diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index e974ffe2..5f4a9856 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -159,7 +159,12 @@ class HttpLayer(base.Layer):
             self.__initial_server_tls = self.server_tls
             self.__initial_server_conn = self.server_conn
         while True:
-            flow = http.HTTPFlow(self.client_conn, self.server_conn, live=self)
+            flow = http.HTTPFlow(
+                self.client_conn,
+                self.server_conn,
+                live=self,
+                mode=self.mode.name
+            )
             if not self._process_flow(flow):
                 return
 
-- 
cgit v1.2.3


From bc01a146b070ecccc4abb5d9382ac4745c430b3c Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 12 Nov 2016 11:39:16 +1300
Subject: Upstream proxy auth to addon

---
 mitmproxy/addons/__init__.py                      |  2 +
 mitmproxy/addons/upstream_proxy_auth.py           | 30 +++++++++++++
 mitmproxy/proxy/config.py                         | 15 -------
 mitmproxy/proxy/protocol/http.py                  |  4 --
 test/mitmproxy/addons/test_upstream_proxy_auth.py | 54 +++++++++++++++++++++++
 test/mitmproxy/test_proxy.py                      |  4 --
 test/mitmproxy/test_proxy_config.py               | 21 ---------
 7 files changed, 86 insertions(+), 44 deletions(-)
 create mode 100644 mitmproxy/addons/upstream_proxy_auth.py
 create mode 100644 test/mitmproxy/addons/test_upstream_proxy_auth.py

diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py
index d2b50c35..d25c231b 100644
--- a/mitmproxy/addons/__init__.py
+++ b/mitmproxy/addons/__init__.py
@@ -10,6 +10,7 @@ from mitmproxy.addons import serverplayback
 from mitmproxy.addons import stickyauth
 from mitmproxy.addons import stickycookie
 from mitmproxy.addons import streambodies
+from mitmproxy.addons import upstream_proxy_auth
 
 
 def default_addons():
@@ -26,4 +27,5 @@ def default_addons():
         setheaders.SetHeaders(),
         serverplayback.ServerPlayback(),
         clientplayback.ClientPlayback(),
+        upstream_proxy_auth.UpstreamProxyAuth(),
     ]
diff --git a/mitmproxy/addons/upstream_proxy_auth.py b/mitmproxy/addons/upstream_proxy_auth.py
new file mode 100644
index 00000000..2ee51fcb
--- /dev/null
+++ b/mitmproxy/addons/upstream_proxy_auth.py
@@ -0,0 +1,30 @@
+import re
+import base64
+
+from mitmproxy import exceptions
+from mitmproxy.utils import strutils
+
+
+def parse_upstream_auth(auth):
+    pattern = re.compile(".+:")
+    if pattern.search(auth) is None:
+        raise exceptions.OptionsError(
+            "Invalid upstream auth specification: %s" % auth
+        )
+    return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
+
+
+class UpstreamProxyAuth():
+    def __init__(self):
+        self.auth = None
+
+    def configure(self, options, updated):
+        if "upstream_auth" in updated:
+            if options.upstream_auth is None:
+                self.auth = None
+            else:
+                self.auth = parse_upstream_auth(options.upstream_auth)
+
+    def requestheaders(self, f):
+        if self.auth and f.mode == "upstream":
+            f.request.headers["Proxy-Authorization"] = self.auth
diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py
index 9c414b9c..e144c175 100644
--- a/mitmproxy/proxy/config.py
+++ b/mitmproxy/proxy/config.py
@@ -1,11 +1,8 @@
-import base64
 import collections
 import os
 import re
 from typing import Any
 
-from mitmproxy.utils import strutils
-
 from OpenSSL import SSL, crypto
 
 from mitmproxy import exceptions
@@ -56,15 +53,6 @@ def parse_server_spec(spec):
     return ServerSpec(scheme, address)
 
 
-def parse_upstream_auth(auth):
-    pattern = re.compile(".+:")
-    if pattern.search(auth) is None:
-        raise exceptions.OptionsError(
-            "Invalid upstream auth specification: %s" % auth
-        )
-    return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
-
-
 class ProxyConfig:
 
     def __init__(self, options: moptions.Options) -> None:
@@ -134,11 +122,8 @@ class ProxyConfig:
                 )
 
         self.upstream_server = None
-        self.upstream_auth = None
         if options.upstream_server:
             self.upstream_server = parse_server_spec(options.upstream_server)
-        if options.upstream_auth:
-            self.upstream_auth = parse_upstream_auth(options.upstream_auth)
 
         self.authenticator = authentication.NullProxyAuth(None)
         needsauth = any(
diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index 5f4a9856..ebe41ac3 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -228,10 +228,6 @@ class HttpLayer(base.Layer):
         if self.config.options.mode == "reverse":
             f.request.headers["Host"] = self.config.upstream_server.address.host
 
-        # set upstream auth
-        if self.mode is HTTPMode.upstream and self.config.upstream_auth is not None:
-            f.request.headers["Proxy-Authorization"] = self.config.upstream_auth
-
         # Determine .scheme, .host and .port attributes for inline scripts.
         # For absolute-form requests, they are directly given in the request.
         # For authority-form requests, we only need to determine the request scheme.
diff --git a/test/mitmproxy/addons/test_upstream_proxy_auth.py b/test/mitmproxy/addons/test_upstream_proxy_auth.py
new file mode 100644
index 00000000..e9a7f4ef
--- /dev/null
+++ b/test/mitmproxy/addons/test_upstream_proxy_auth.py
@@ -0,0 +1,54 @@
+import base64
+
+from mitmproxy import exceptions
+from mitmproxy.test import taddons
+from mitmproxy.test import tflow
+from mitmproxy.test import tutils
+from mitmproxy.addons import upstream_proxy_auth
+
+
+def test_configure():
+    up = upstream_proxy_auth.UpstreamProxyAuth()
+    with taddons.context() as tctx:
+        tctx.configure(up, upstream_auth="test:test")
+        assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:test")
+
+        tctx.configure(up, upstream_auth="test:")
+        assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:")
+
+        tctx.configure(up, upstream_auth=None)
+        assert not up.auth
+
+        tutils.raises(
+            exceptions.OptionsError,
+            tctx.configure,
+            up,
+            upstream_auth=""
+        )
+        tutils.raises(
+            exceptions.OptionsError,
+            tctx.configure,
+            up,
+            upstream_auth=":"
+        )
+        tutils.raises(
+            exceptions.OptionsError,
+            tctx.configure,
+            up,
+            upstream_auth=":test"
+        )
+
+
+def test_simple():
+    up = upstream_proxy_auth.UpstreamProxyAuth()
+    with taddons.context() as tctx:
+        tctx.configure(up, upstream_auth="foo:bar")
+
+        f = tflow.tflow()
+        f.mode = "upstream"
+        up.requestheaders(f)
+        assert "proxy-authorization" in f.request.headers
+
+        f = tflow.tflow()
+        up.requestheaders(f)
+        assert "proxy-authorization" not in f.request.headers
diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py
index 7cadb6c2..8847c088 100644
--- a/test/mitmproxy/test_proxy.py
+++ b/test/mitmproxy/test_proxy.py
@@ -107,14 +107,10 @@ class TestProcessProxyOptions:
         self.assert_noerr("-T")
 
         self.assert_noerr("-U", "http://localhost")
-        self.assert_err("expected one argument", "-U")
         self.assert_err("Invalid server specification", "-U", "upstream")
 
         self.assert_noerr("--upstream-auth", "test:test")
         self.assert_err("expected one argument", "--upstream-auth")
-        self.assert_err(
-            "Invalid upstream auth specification", "--upstream-auth", "test"
-        )
         self.assert_err("mutually exclusive", "-R", "http://localhost", "-T")
 
     def test_socks_auth(self):
diff --git a/test/mitmproxy/test_proxy_config.py b/test/mitmproxy/test_proxy_config.py
index e012cb5e..e2c39846 100644
--- a/test/mitmproxy/test_proxy_config.py
+++ b/test/mitmproxy/test_proxy_config.py
@@ -1,5 +1,4 @@
 from mitmproxy.test import tutils
-import base64
 from mitmproxy.proxy import config
 
 
@@ -26,23 +25,3 @@ def test_parse_server_spec():
         config.parse_server_spec,
         "http://"
     )
-
-
-def test_parse_upstream_auth():
-    tutils.raises(
-        "Invalid upstream auth specification",
-        config.parse_upstream_auth,
-        ""
-    )
-    tutils.raises(
-        "Invalid upstream auth specification",
-        config.parse_upstream_auth,
-        ":"
-    )
-    tutils.raises(
-        "Invalid upstream auth specification",
-        config.parse_upstream_auth,
-        ":test"
-    )
-    assert config.parse_upstream_auth("test:test") == b"Basic" + b" " + base64.b64encode(b"test:test")
-    assert config.parse_upstream_auth("test:") == b"Basic" + b" " + base64.b64encode(b"test:")
-- 
cgit v1.2.3


From 38f8d9e541f8d60cf3d829f6ac2c204ef493680f Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 12 Nov 2016 12:44:43 +1300
Subject: Add the http_connect event for HTTP CONNECT requests

---
 mitmproxy/events.py                  |  1 +
 mitmproxy/master.py                  |  4 ++++
 mitmproxy/proxy/protocol/http.py     | 41 +++++++++++++++++++++---------------
 test/mitmproxy/test_eventsequence.py | 18 +++++++++++++---
 4 files changed, 44 insertions(+), 20 deletions(-)

diff --git a/mitmproxy/events.py b/mitmproxy/events.py
index 56f1a45b..f9475768 100644
--- a/mitmproxy/events.py
+++ b/mitmproxy/events.py
@@ -13,6 +13,7 @@ Events = frozenset([
     "tcp_error",
     "tcp_end",
 
+    "http_connect",
     "request",
     "requestheaders",
     "response",
diff --git a/mitmproxy/master.py b/mitmproxy/master.py
index ffbfb0cb..55eb74e5 100644
--- a/mitmproxy/master.py
+++ b/mitmproxy/master.py
@@ -255,6 +255,10 @@ class Master:
     def next_layer(self, top_layer):
         pass
 
+    @controller.handler
+    def http_connect(self, f):
+        pass
+
     @controller.handler
     def error(self, f):
         pass
diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index ebe41ac3..d99812f8 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -176,6 +176,30 @@ class HttpLayer(base.Layer):
                 # don't throw an error for disconnects that happen before/between requests.
                 return False
 
+            # Regular Proxy Mode: Handle CONNECT
+            if self.mode is HTTPMode.regular and request.first_line_format == "authority":
+                self.connect_request = True
+                # The standards are silent on what we should do with a CONNECT
+                # request body, so although it's not common, it's allowed.
+                request.data.content = b"".join(self.read_request_body(request))
+                request.timestamp_end = time.time()
+
+                self.channel.ask("http_connect", f)
+
+                try:
+                    self.set_server((request.host, request.port))
+                except (exceptions.ProtocolException, exceptions.NetlibException) as e:
+                    # HTTPS tasting means that ordinary errors like resolution and
+                    # connection errors can happen here.
+                    self.send_error_response(502, repr(e))
+                    f.error = flow.Error(str(e))
+                    self.channel.ask("error", f)
+                    return False
+                self.send_response(http.make_connect_response(request.data.http_version))
+                layer = self.ctx.next_layer(self)
+                layer()
+                return False
+
             f.request = request
             self.channel.ask("requestheaders", f)
 
@@ -207,23 +231,6 @@ class HttpLayer(base.Layer):
 
         f.request = request
 
-        try:
-            # Regular Proxy Mode: Handle CONNECT
-            if self.mode is HTTPMode.regular and request.first_line_format == "authority":
-                self.connect_request = True
-                self.set_server((request.host, request.port))
-                self.send_response(http.make_connect_response(request.data.http_version))
-                layer = self.ctx.next_layer(self)
-                layer()
-                return False
-        except (exceptions.ProtocolException, exceptions.NetlibException) as e:
-            # HTTPS tasting means that ordinary errors like resolution and
-            # connection errors can happen here.
-            self.send_error_response(502, repr(e))
-            f.error = flow.Error(str(e))
-            self.channel.ask("error", f)
-            return False
-
         # update host header in reverse proxy mode
         if self.config.options.mode == "reverse":
             f.request.headers["Host"] = self.config.upstream_server.address.host
diff --git a/test/mitmproxy/test_eventsequence.py b/test/mitmproxy/test_eventsequence.py
index 31c57e82..e6eb6569 100644
--- a/test/mitmproxy/test_eventsequence.py
+++ b/test/mitmproxy/test_eventsequence.py
@@ -37,6 +37,8 @@ class SequenceTester:
 
 
 class TestBasic(tservers.HTTPProxyTest, SequenceTester):
+    ssl = True
+
     def test_requestheaders(self):
 
         def hdrs(f):
@@ -50,7 +52,7 @@ class TestBasic(tservers.HTTPProxyTest, SequenceTester):
         with self.addon(Eventer(requestheaders=hdrs, request=req)):
             p = self.pathoc()
             with p.connect():
-                assert p.request("get:'%s/p/200':b@10" % self.server.urlbase).status_code == 200
+                assert p.request("get:'/p/200':b@10").status_code == 200
 
     def test_100_continue_fail(self):
         e = Eventer()
@@ -59,10 +61,20 @@ class TestBasic(tservers.HTTPProxyTest, SequenceTester):
             with p.connect():
                 p.request(
                     """
-                        get:'%s/p/200'
+                        get:'/p/200'
                         h'expect'='100-continue'
                         h'content-length'='1000'
                         da
-                    """ % self.server.urlbase
+                    """
                 )
             assert e.called[-1] == "requestheaders"
+
+    def test_connect(self):
+        e = Eventer()
+        with self.addon(e):
+            p = self.pathoc()
+            with p.connect():
+                p.request("get:'/p/200:b@1'")
+            assert "http_connect" in e.called
+            assert e.called.count("requestheaders") == 1
+            assert e.called.count("request") == 1
-- 
cgit v1.2.3


From a9b4560187df02c0d69e89a4892587a65bb03ea7 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 12 Nov 2016 18:28:37 +1300
Subject: Refine handling of HTTP CONNECT

- CONNECT requests do not generate the usual http events. Instead, they
generate the http_connect event and handlers then have the option of setting an
error response to abort the connect.
- The connect handler is called for both upstream proxy and regular proxy CONNECTs.
---
 docs/scripting/events.rst             |  13 ++++
 mitmproxy/proxy/protocol/http.py      | 128 ++++++++++++++++++++++------------
 test/mitmproxy/protocol/test_http1.py |   2 +-
 test/mitmproxy/test_server.py         |  81 +++++++--------------
 4 files changed, 124 insertions(+), 100 deletions(-)

diff --git a/docs/scripting/events.rst b/docs/scripting/events.rst
index 62266485..a5721403 100644
--- a/docs/scripting/events.rst
+++ b/docs/scripting/events.rst
@@ -98,6 +98,19 @@ HTTP Events
     :widths: 40 60
     :header-rows: 0
 
+    *   - .. py:function:: http_connect(flow)
+
+        - Called when we receive an HTTP CONNECT request. Setting a non 2xx
+        response on the flow will return the response to the client abort the
+        connection. CONNECT requests and responses do not generate the usual
+        HTTP handler events. CONNECT requests are only valid in regular and
+        upstream proxy modes.
+
+          *flow*
+            A ``models.HTTPFlow`` object. The flow is guaranteed to have
+            non-None ``request``  and ``requestheaders`` attributes.
+
+
     *   - .. py:function:: request(flow)
         - Called when a client request has been received.
 
diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index d99812f8..15f3f7cf 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -114,6 +114,10 @@ class UpstreamConnectLayer(base.Layer):
         self.server_conn.address = address
 
 
+def is_ok(status):
+    return 200 <= status < 300
+
+
 class HTTPMode(enum.Enum):
     regular = 1
     transparent = 2
@@ -168,43 +172,85 @@ class HttpLayer(base.Layer):
             if not self._process_flow(flow):
                 return
 
+    def handle_regular_connect(self, f):
+        self.connect_request = True
+
+        try:
+            self.set_server((f.request.host, f.request.port))
+        except (
+            exceptions.ProtocolException, exceptions.NetlibException
+        ) as e:
+            # HTTPS tasting means that ordinary errors like resolution
+            # and connection errors can happen here.
+            self.send_error_response(502, repr(e))
+            f.error = flow.Error(str(e))
+            self.channel.ask("error", f)
+            return False
+
+        if f.response:
+            resp = f.response
+        else:
+            resp = http.make_connect_response(f.request.data.http_version)
+
+        self.send_response(resp)
+
+        if is_ok(resp.status_code):
+            layer = self.ctx.next_layer(self)
+            layer()
+
+        return False
+
+    def handle_upstream_connect(self, f):
+        self.establish_server_connection(
+            f.request.host,
+            f.request.port,
+            f.request.scheme
+        )
+        self.send_request(f.request)
+        f.response = self.read_response_headers()
+        f.response.data.content = b"".join(
+            self.read_response_body(f.request, f.response)
+        )
+        self.send_response(f.response)
+        if is_ok(f.response.status_code):
+            layer = UpstreamConnectLayer(self, f.request)
+            return layer()
+        return False
+
     def _process_flow(self, f):
         try:
             try:
                 request = self.read_request_headers(f)
             except exceptions.HttpReadDisconnect:
-                # don't throw an error for disconnects that happen before/between requests.
+                # don't throw an error for disconnects that happen
+                # before/between requests.
                 return False
 
-            # Regular Proxy Mode: Handle CONNECT
-            if self.mode is HTTPMode.regular and request.first_line_format == "authority":
-                self.connect_request = True
+            f.request = request
+
+            if request.first_line_format == "authority":
                 # The standards are silent on what we should do with a CONNECT
                 # request body, so although it's not common, it's allowed.
-                request.data.content = b"".join(self.read_request_body(request))
-                request.timestamp_end = time.time()
-
+                f.request.data.content = b"".join(
+                    self.read_request_body(f.request)
+                )
+                f.request.timestamp_end = time.time()
                 self.channel.ask("http_connect", f)
 
-                try:
-                    self.set_server((request.host, request.port))
-                except (exceptions.ProtocolException, exceptions.NetlibException) as e:
-                    # HTTPS tasting means that ordinary errors like resolution and
-                    # connection errors can happen here.
-                    self.send_error_response(502, repr(e))
-                    f.error = flow.Error(str(e))
-                    self.channel.ask("error", f)
-                    return False
-                self.send_response(http.make_connect_response(request.data.http_version))
-                layer = self.ctx.next_layer(self)
-                layer()
-                return False
+                if self.mode is HTTPMode.regular:
+                    return self.handle_regular_connect(f)
+                elif self.mode is HTTPMode.upstream:
+                    return self.handle_upstream_connect(f)
+                else:
+                    msg = "Unexpected CONNECT request."
+                    self.send_error_response(400, msg)
+                    raise exceptions.ProtocolException(msg)
 
-            f.request = request
             self.channel.ask("requestheaders", f)
 
             if request.headers.get("expect", "").lower() == "100-continue":
-                # TODO: We may have to use send_response_headers for HTTP2 here.
+                # TODO: We may have to use send_response_headers for HTTP2
+                # here.
                 self.send_response(http.expect_continue_response)
                 request.headers.pop("expect")
 
@@ -222,10 +268,10 @@ class HttpLayer(base.Layer):
 
         self.log("request", "debug", [repr(request)])
 
-        # Handle Proxy Authentication
-        # Proxy Authentication conceptually does not work in transparent mode.
-        # We catch this misconfiguration on startup. Here, we sort out requests
-        # after a successful CONNECT request (which do not need to be validated anymore)
+        # Handle Proxy Authentication Proxy Authentication conceptually does
+        # not work in transparent mode. We catch this misconfiguration on
+        # startup. Here, we sort out requests after a successful CONNECT
+        # request (which do not need to be validated anymore)
         if not self.connect_request and not self.authenticate(request):
             return False
 
@@ -235,13 +281,14 @@ class HttpLayer(base.Layer):
         if self.config.options.mode == "reverse":
             f.request.headers["Host"] = self.config.upstream_server.address.host
 
-        # Determine .scheme, .host and .port attributes for inline scripts.
-        # For absolute-form requests, they are directly given in the request.
-        # For authority-form requests, we only need to determine the request scheme.
-        # For relative-form requests, we need to determine host and port as
-        # well.
+        # Determine .scheme, .host and .port attributes for inline scripts. For
+        # absolute-form requests, they are directly given in the request. For
+        # authority-form requests, we only need to determine the request
+        # scheme. For relative-form requests, we need to determine host and
+        # port as well.
         if self.mode is HTTPMode.transparent:
-            # Setting request.host also updates the host header, which we want to preserve
+            # Setting request.host also updates the host header, which we want
+            # to preserve
             host_header = f.request.headers.get("host", None)
             f.request.host = self.__initial_server_conn.address.host
             f.request.port = self.__initial_server_conn.address.port
@@ -296,17 +343,16 @@ class HttpLayer(base.Layer):
                     self.connect()
                     get_response()
 
-                # call the appropriate script hook - this is an opportunity for an
-                # inline script to set f.stream = True
+                # call the appropriate script hook - this is an opportunity for
+                # an inline script to set f.stream = True
                 self.channel.ask("responseheaders", f)
 
                 if f.response.stream:
                     f.response.data.content = None
                 else:
-                    f.response.data.content = b"".join(self.read_response_body(
-                        f.request,
-                        f.response
-                    ))
+                    f.response.data.content = b"".join(
+                        self.read_response_body(f.request, f.response)
+                    )
                 f.response.timestamp_end = time.time()
 
                 # no further manipulation of self.server_conn beyond this point
@@ -365,12 +411,6 @@ class HttpLayer(base.Layer):
                 layer()
                 return False  # should never be reached
 
-            # Upstream Proxy Mode: Handle CONNECT
-            if f.request.first_line_format == "authority" and f.response.status_code == 200:
-                layer = UpstreamConnectLayer(self, f.request)
-                layer()
-                return False
-
         except (exceptions.ProtocolException, exceptions.NetlibException) as e:
             self.send_error_response(502, repr(e))
             if not f.response:
diff --git a/test/mitmproxy/protocol/test_http1.py b/test/mitmproxy/protocol/test_http1.py
index 5026bef1..cd937ada 100644
--- a/test/mitmproxy/protocol/test_http1.py
+++ b/test/mitmproxy/protocol/test_http1.py
@@ -20,7 +20,7 @@ class TestInvalidRequests(tservers.HTTPProxyTest):
         with p.connect():
             r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port))
         assert r.status_code == 400
-        assert b"Invalid HTTP request form" in r.content
+        assert b"Unexpected CONNECT" in r.content
 
     def test_relative_request(self):
         p = self.pathoc_raw()
diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py
index 9fa6ed06..dab47c9c 100644
--- a/test/mitmproxy/test_server.py
+++ b/test/mitmproxy/test_server.py
@@ -50,10 +50,7 @@ class CommonMixin:
 
     def test_replay(self):
         assert self.pathod("304").status_code == 304
-        if isinstance(self, tservers.HTTPUpstreamProxyTest) and self.ssl:
-            assert len(self.master.state.flows) == 2
-        else:
-            assert len(self.master.state.flows) == 1
+        assert len(self.master.state.flows) == 1
         l = self.master.state.flows[-1]
         assert l.response.status_code == 304
         l.request.path = "/p/305"
@@ -952,10 +949,10 @@ class TestUpstreamProxySSL(
         assert req.status_code == 418
 
         # CONNECT from pathoc to chain[0],
-        assert self.proxy.tmaster.state.flow_count() == 2
+        assert self.proxy.tmaster.state.flow_count() == 1
         # request from pathoc to chain[0]
         # CONNECT from proxy to chain[1],
-        assert self.chain[0].tmaster.state.flow_count() == 2
+        assert self.chain[0].tmaster.state.flow_count() == 1
         # request from proxy to chain[1]
         # request from chain[0] (regular proxy doesn't store CONNECTs)
         assert self.chain[1].tmaster.state.flow_count() == 1
@@ -978,21 +975,12 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest):
     def test_reconnect(self):
         """
         Tests proper functionality of ConnectionHandler.server_reconnect mock.
-        If we have a disconnect on a secure connection that's transparently proxified to
-        an upstream http proxy, we need to send the CONNECT request again.
+        If we have a disconnect on a secure connection that's transparently
+        proxified to an upstream http proxy, we need to send the CONNECT
+        request again.
         """
-        self.chain[1].tmaster.addons.add(
-            RequestKiller([2])
-        )
-        self.chain[0].tmaster.addons.add(
-            RequestKiller(
-                [
-                    1,  # CONNECT
-                    3,  # reCONNECT
-                    4   # request
-                ]
-            )
-        )
+        self.chain[0].tmaster.addons.add(RequestKiller([1, 2]))
+        self.chain[1].tmaster.addons.add(RequestKiller([1]))
 
         p = self.pathoc()
         with p.connect():
@@ -1000,44 +988,27 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest):
             assert req.content == b"content"
             assert req.status_code == 418
 
-            assert self.proxy.tmaster.state.flow_count() == 2  # CONNECT and request
-            # CONNECT, failing request,
-            assert self.chain[0].tmaster.state.flow_count() == 4
-            # reCONNECT, request
-            # failing request, request
-            assert self.chain[1].tmaster.state.flow_count() == 2
-            # (doesn't store (repeated) CONNECTs from chain[0]
-            #  as it is a regular proxy)
-
-            assert not self.chain[1].tmaster.state.flows[0].response  # killed
-            assert self.chain[1].tmaster.state.flows[1].response
-
-            assert self.proxy.tmaster.state.flows[0].request.first_line_format == "authority"
-            assert self.proxy.tmaster.state.flows[1].request.first_line_format == "relative"
-
-            assert self.chain[0].tmaster.state.flows[
-                0].request.first_line_format == "authority"
-            assert self.chain[0].tmaster.state.flows[
-                1].request.first_line_format == "relative"
-            assert self.chain[0].tmaster.state.flows[
-                2].request.first_line_format == "authority"
-            assert self.chain[0].tmaster.state.flows[
-                3].request.first_line_format == "relative"
-
-            assert self.chain[1].tmaster.state.flows[
-                0].request.first_line_format == "relative"
-            assert self.chain[1].tmaster.state.flows[
-                1].request.first_line_format == "relative"
+            # First request goes through all three proxies exactly once
+            assert self.proxy.tmaster.state.flow_count() == 1
+            assert self.chain[0].tmaster.state.flow_count() == 1
+            assert self.chain[1].tmaster.state.flow_count() == 1
 
             req = p.request("get:'/p/418:b\"content2\"'")
-
             assert req.status_code == 502
-            assert self.proxy.tmaster.state.flow_count() == 3  # + new request
-            # + new request, repeated CONNECT from chain[1]
-            assert self.chain[0].tmaster.state.flow_count() == 6
-            # (both terminated)
-            # nothing happened here
-            assert self.chain[1].tmaster.state.flow_count() == 2
+
+            assert self.proxy.tmaster.state.flow_count() == 2
+            assert self.chain[0].tmaster.state.flow_count() == 2
+            # Upstream sees two requests due to reconnection attempt
+            assert self.chain[1].tmaster.state.flow_count() == 3
+            assert not self.chain[1].tmaster.state.flows[-1].response
+            assert not self.chain[1].tmaster.state.flows[-2].response
+
+            # Reconnection failed, so we're now disconnected
+            tutils.raises(
+                exceptions.HttpException,
+                p.request,
+                "get:'/p/418:b\"content3\"'"
+            )
 
 
 class AddUpstreamCertsToClientChainMixin:
-- 
cgit v1.2.3


From 3b00bc339d1c65703431e92cfeb2b7436790d04e Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sun, 13 Nov 2016 11:43:27 +1300
Subject: Complete upstream authentication module

- Handles upstream CONNECT and regular requests, plus HTTP Basic for reverse
proxy
- Add some tests to make sure we can rely on the .via attribute on server
connections.
---
 mitmproxy/addons/upstream_proxy_auth.py           | 25 ++++++++++++++++++++++-
 mitmproxy/proxy/protocol/http.py                  | 20 ++++++++++--------
 mitmproxy/proxy/root_context.py                   |  7 ++++++-
 test/mitmproxy/addons/test_upstream_proxy_auth.py | 11 ++++++++++
 test/mitmproxy/test_server.py                     | 10 +++++++++
 5 files changed, 62 insertions(+), 11 deletions(-)

diff --git a/mitmproxy/addons/upstream_proxy_auth.py b/mitmproxy/addons/upstream_proxy_auth.py
index 2ee51fcb..8b31c10a 100644
--- a/mitmproxy/addons/upstream_proxy_auth.py
+++ b/mitmproxy/addons/upstream_proxy_auth.py
@@ -15,16 +15,39 @@ def parse_upstream_auth(auth):
 
 
 class UpstreamProxyAuth():
+    """
+        This addon handles authentication to systems upstream from us for the
+        upstream proxy and reverse proxy mode. There are 3 cases:
+
+        - Upstream proxy CONNECT requests should have authentication added, and
+          subsequent already connected requests should not.
+        - Upstream proxy regular requests
+        - Reverse proxy regular requests (CONNECT is invalid in this mode)
+    """
     def __init__(self):
         self.auth = None
+        self.root_mode = None
 
     def configure(self, options, updated):
+        # FIXME: We're doing this because our proxy core is terminally confused
+        # at the moment. Ideally, we should be able to check if we're in
+        # reverse proxy mode at the HTTP layer, so that scripts can put the
+        # proxy in reverse proxy mode for specific reuests.
+        if "mode" in updated:
+            self.root_mode = options.mode
         if "upstream_auth" in updated:
             if options.upstream_auth is None:
                 self.auth = None
             else:
                 self.auth = parse_upstream_auth(options.upstream_auth)
 
-    def requestheaders(self, f):
+    def http_connect(self, f):
         if self.auth and f.mode == "upstream":
             f.request.headers["Proxy-Authorization"] = self.auth
+
+    def requestheaders(self, f):
+        if self.auth:
+            if f.mode == "upstream" and not f.server_conn.via:
+                f.request.headers["Proxy-Authorization"] = self.auth
+            elif self.root_mode == "reverse":
+                f.request.headers["Proxy-Authorization"] = self.auth
diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index 15f3f7cf..5165018d 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -135,7 +135,9 @@ MODE_REQUEST_FORMS = {
 
 def validate_request_form(mode, request):
     if request.first_line_format == "absolute" and request.scheme != "http":
-        raise exceptions.HttpException("Invalid request scheme: %s" % request.scheme)
+        raise exceptions.HttpException(
+            "Invalid request scheme: %s" % request.scheme
+        )
     allowed_request_forms = MODE_REQUEST_FORMS[mode]
     if request.first_line_format not in allowed_request_forms:
         err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
@@ -275,8 +277,6 @@ class HttpLayer(base.Layer):
         if not self.connect_request and not self.authenticate(request):
             return False
 
-        f.request = request
-
         # update host header in reverse proxy mode
         if self.config.options.mode == "reverse":
             f.request.headers["Host"] = self.config.upstream_server.address.host
@@ -389,10 +389,8 @@ class HttpLayer(base.Layer):
 
             # Handle 101 Switching Protocols
             if f.response.status_code == 101:
-                """
-                    Handle a successful HTTP 101 Switching Protocols Response, received after
-                    e.g. a WebSocket upgrade request.
-                """
+                # Handle a successful HTTP 101 Switching Protocols Response,
+                # received after e.g. a WebSocket upgrade request.
                 # Check for WebSockets handshake
                 is_websockets = (
                     websockets.check_handshake(f.request.headers) and
@@ -467,13 +465,17 @@ class HttpLayer(base.Layer):
                     self.send_response(http.make_error_response(
                         401,
                         "Authentication Required",
-                        mitmproxy.net.http.Headers(**self.config.authenticator.auth_challenge_headers())
+                        mitmproxy.net.http.Headers(
+                            **self.config.authenticator.auth_challenge_headers()
+                        )
                     ))
                 else:
                     self.send_response(http.make_error_response(
                         407,
                         "Proxy Authentication Required",
-                        mitmproxy.net.http.Headers(**self.config.authenticator.auth_challenge_headers())
+                        mitmproxy.net.http.Headers(
+                            **self.config.authenticator.auth_challenge_headers()
+                        )
                     ))
                 return False
         return True
diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py
index 50dbe79e..f38f2a8c 100644
--- a/mitmproxy/proxy/root_context.py
+++ b/mitmproxy/proxy/root_context.py
@@ -64,7 +64,12 @@ class RootContext:
         # An inline script may upgrade from http to https,
         # in which case we need some form of TLS layer.
         if isinstance(top_layer, modes.ReverseProxy):
-            return protocol.TlsLayer(top_layer, client_tls, top_layer.server_tls, top_layer.server_conn.address.host)
+            return protocol.TlsLayer(
+                top_layer,
+                client_tls,
+                top_layer.server_tls,
+                top_layer.server_conn.address.host
+            )
         if isinstance(top_layer, protocol.ServerConnectionMixin) or isinstance(top_layer, protocol.UpstreamConnectLayer):
             return protocol.TlsLayer(top_layer, client_tls, client_tls)
 
diff --git a/test/mitmproxy/addons/test_upstream_proxy_auth.py b/test/mitmproxy/addons/test_upstream_proxy_auth.py
index e9a7f4ef..d5d6a3e3 100644
--- a/test/mitmproxy/addons/test_upstream_proxy_auth.py
+++ b/test/mitmproxy/addons/test_upstream_proxy_auth.py
@@ -52,3 +52,14 @@ def test_simple():
         f = tflow.tflow()
         up.requestheaders(f)
         assert "proxy-authorization" not in f.request.headers
+
+        tctx.configure(up, mode="reverse")
+        f = tflow.tflow()
+        f.mode = "transparent"
+        up.requestheaders(f)
+        assert "proxy-authorization" in f.request.headers
+
+        f = tflow.tflow()
+        f.mode = "upstream"
+        up.http_connect(f)
+        assert "proxy-authorization" in f.request.headers
diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py
index dab47c9c..5a5b6817 100644
--- a/test/mitmproxy/test_server.py
+++ b/test/mitmproxy/test_server.py
@@ -669,6 +669,13 @@ class TestProxySSL(tservers.HTTPProxyTest):
         first_flow = self.master.state.flows[0]
         assert first_flow.server_conn.timestamp_ssl_setup
 
+    def test_via(self):
+        # tests that the ssl timestamp is present when ssl is used
+        f = self.pathod("200:b@10")
+        assert f.status_code == 200
+        first_flow = self.master.state.flows[0]
+        assert not first_flow.server_conn.via
+
 
 class MasterRedirectRequest(tservers.TestMaster):
     redirect_port = None  # Set by TestRedirectRequest
@@ -950,11 +957,14 @@ class TestUpstreamProxySSL(
 
         # CONNECT from pathoc to chain[0],
         assert self.proxy.tmaster.state.flow_count() == 1
+        assert self.proxy.tmaster.state.flows[0].server_conn.via
         # request from pathoc to chain[0]
         # CONNECT from proxy to chain[1],
         assert self.chain[0].tmaster.state.flow_count() == 1
+        assert self.chain[0].tmaster.state.flows[0].server_conn.via
         # request from proxy to chain[1]
         # request from chain[0] (regular proxy doesn't store CONNECTs)
+        assert not self.chain[1].tmaster.state.flows[0].server_conn.via
         assert self.chain[1].tmaster.state.flow_count() == 1
 
 
-- 
cgit v1.2.3


From fe01b1435a2acc9896b24a814e535558884a6143 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sun, 13 Nov 2016 11:50:28 +1300
Subject: upstream_proxy_auth -> upstream_auth

Also clarify what this does in commandline help.
---
 mitmproxy/addons/__init__.py                      |  4 +-
 mitmproxy/addons/upstream_auth.py                 | 53 ++++++++++++++++++
 mitmproxy/addons/upstream_proxy_auth.py           | 53 ------------------
 mitmproxy/tools/cmdline.py                        |  4 +-
 test/mitmproxy/addons/test_upstream_auth.py       | 65 +++++++++++++++++++++++
 test/mitmproxy/addons/test_upstream_proxy_auth.py | 65 -----------------------
 6 files changed, 122 insertions(+), 122 deletions(-)
 create mode 100644 mitmproxy/addons/upstream_auth.py
 delete mode 100644 mitmproxy/addons/upstream_proxy_auth.py
 create mode 100644 test/mitmproxy/addons/test_upstream_auth.py
 delete mode 100644 test/mitmproxy/addons/test_upstream_proxy_auth.py

diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py
index d25c231b..d71b8912 100644
--- a/mitmproxy/addons/__init__.py
+++ b/mitmproxy/addons/__init__.py
@@ -10,7 +10,7 @@ from mitmproxy.addons import serverplayback
 from mitmproxy.addons import stickyauth
 from mitmproxy.addons import stickycookie
 from mitmproxy.addons import streambodies
-from mitmproxy.addons import upstream_proxy_auth
+from mitmproxy.addons import upstream_auth
 
 
 def default_addons():
@@ -27,5 +27,5 @@ def default_addons():
         setheaders.SetHeaders(),
         serverplayback.ServerPlayback(),
         clientplayback.ClientPlayback(),
-        upstream_proxy_auth.UpstreamProxyAuth(),
+        upstream_auth.UpstreamAuth(),
     ]
diff --git a/mitmproxy/addons/upstream_auth.py b/mitmproxy/addons/upstream_auth.py
new file mode 100644
index 00000000..9beecfc0
--- /dev/null
+++ b/mitmproxy/addons/upstream_auth.py
@@ -0,0 +1,53 @@
+import re
+import base64
+
+from mitmproxy import exceptions
+from mitmproxy.utils import strutils
+
+
+def parse_upstream_auth(auth):
+    pattern = re.compile(".+:")
+    if pattern.search(auth) is None:
+        raise exceptions.OptionsError(
+            "Invalid upstream auth specification: %s" % auth
+        )
+    return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
+
+
+class UpstreamAuth():
+    """
+        This addon handles authentication to systems upstream from us for the
+        upstream proxy and reverse proxy mode. There are 3 cases:
+
+        - Upstream proxy CONNECT requests should have authentication added, and
+          subsequent already connected requests should not.
+        - Upstream proxy regular requests
+        - Reverse proxy regular requests (CONNECT is invalid in this mode)
+    """
+    def __init__(self):
+        self.auth = None
+        self.root_mode = None
+
+    def configure(self, options, updated):
+        # FIXME: We're doing this because our proxy core is terminally confused
+        # at the moment. Ideally, we should be able to check if we're in
+        # reverse proxy mode at the HTTP layer, so that scripts can put the
+        # proxy in reverse proxy mode for specific reuests.
+        if "mode" in updated:
+            self.root_mode = options.mode
+        if "upstream_auth" in updated:
+            if options.upstream_auth is None:
+                self.auth = None
+            else:
+                self.auth = parse_upstream_auth(options.upstream_auth)
+
+    def http_connect(self, f):
+        if self.auth and f.mode == "upstream":
+            f.request.headers["Proxy-Authorization"] = self.auth
+
+    def requestheaders(self, f):
+        if self.auth:
+            if f.mode == "upstream" and not f.server_conn.via:
+                f.request.headers["Proxy-Authorization"] = self.auth
+            elif self.root_mode == "reverse":
+                f.request.headers["Proxy-Authorization"] = self.auth
diff --git a/mitmproxy/addons/upstream_proxy_auth.py b/mitmproxy/addons/upstream_proxy_auth.py
deleted file mode 100644
index 8b31c10a..00000000
--- a/mitmproxy/addons/upstream_proxy_auth.py
+++ /dev/null
@@ -1,53 +0,0 @@
-import re
-import base64
-
-from mitmproxy import exceptions
-from mitmproxy.utils import strutils
-
-
-def parse_upstream_auth(auth):
-    pattern = re.compile(".+:")
-    if pattern.search(auth) is None:
-        raise exceptions.OptionsError(
-            "Invalid upstream auth specification: %s" % auth
-        )
-    return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
-
-
-class UpstreamProxyAuth():
-    """
-        This addon handles authentication to systems upstream from us for the
-        upstream proxy and reverse proxy mode. There are 3 cases:
-
-        - Upstream proxy CONNECT requests should have authentication added, and
-          subsequent already connected requests should not.
-        - Upstream proxy regular requests
-        - Reverse proxy regular requests (CONNECT is invalid in this mode)
-    """
-    def __init__(self):
-        self.auth = None
-        self.root_mode = None
-
-    def configure(self, options, updated):
-        # FIXME: We're doing this because our proxy core is terminally confused
-        # at the moment. Ideally, we should be able to check if we're in
-        # reverse proxy mode at the HTTP layer, so that scripts can put the
-        # proxy in reverse proxy mode for specific reuests.
-        if "mode" in updated:
-            self.root_mode = options.mode
-        if "upstream_auth" in updated:
-            if options.upstream_auth is None:
-                self.auth = None
-            else:
-                self.auth = parse_upstream_auth(options.upstream_auth)
-
-    def http_connect(self, f):
-        if self.auth and f.mode == "upstream":
-            f.request.headers["Proxy-Authorization"] = self.auth
-
-    def requestheaders(self, f):
-        if self.auth:
-            if f.mode == "upstream" and not f.server_conn.via:
-                f.request.headers["Proxy-Authorization"] = self.auth
-            elif self.root_mode == "reverse":
-                f.request.headers["Proxy-Authorization"] = self.auth
diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py
index debe6db9..8b579952 100644
--- a/mitmproxy/tools/cmdline.py
+++ b/mitmproxy/tools/cmdline.py
@@ -463,8 +463,8 @@ def proxy_options(parser):
         action="store", dest="upstream_auth", default=None,
         type=str,
         help="""
-            Proxy Authentication:
-            username:password
+            Add HTTP Basic authentcation to upstream proxy and reverse proxy
+            requests. Format: username:password
         """
     )
     rawtcp = group.add_mutually_exclusive_group()
diff --git a/test/mitmproxy/addons/test_upstream_auth.py b/test/mitmproxy/addons/test_upstream_auth.py
new file mode 100644
index 00000000..985b13a7
--- /dev/null
+++ b/test/mitmproxy/addons/test_upstream_auth.py
@@ -0,0 +1,65 @@
+import base64
+
+from mitmproxy import exceptions
+from mitmproxy.test import taddons
+from mitmproxy.test import tflow
+from mitmproxy.test import tutils
+from mitmproxy.addons import upstream_auth
+
+
+def test_configure():
+    up = upstream_auth.UpstreamAuth()
+    with taddons.context() as tctx:
+        tctx.configure(up, upstream_auth="test:test")
+        assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:test")
+
+        tctx.configure(up, upstream_auth="test:")
+        assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:")
+
+        tctx.configure(up, upstream_auth=None)
+        assert not up.auth
+
+        tutils.raises(
+            exceptions.OptionsError,
+            tctx.configure,
+            up,
+            upstream_auth=""
+        )
+        tutils.raises(
+            exceptions.OptionsError,
+            tctx.configure,
+            up,
+            upstream_auth=":"
+        )
+        tutils.raises(
+            exceptions.OptionsError,
+            tctx.configure,
+            up,
+            upstream_auth=":test"
+        )
+
+
+def test_simple():
+    up = upstream_auth.UpstreamAuth()
+    with taddons.context() as tctx:
+        tctx.configure(up, upstream_auth="foo:bar")
+
+        f = tflow.tflow()
+        f.mode = "upstream"
+        up.requestheaders(f)
+        assert "proxy-authorization" in f.request.headers
+
+        f = tflow.tflow()
+        up.requestheaders(f)
+        assert "proxy-authorization" not in f.request.headers
+
+        tctx.configure(up, mode="reverse")
+        f = tflow.tflow()
+        f.mode = "transparent"
+        up.requestheaders(f)
+        assert "proxy-authorization" in f.request.headers
+
+        f = tflow.tflow()
+        f.mode = "upstream"
+        up.http_connect(f)
+        assert "proxy-authorization" in f.request.headers
diff --git a/test/mitmproxy/addons/test_upstream_proxy_auth.py b/test/mitmproxy/addons/test_upstream_proxy_auth.py
deleted file mode 100644
index d5d6a3e3..00000000
--- a/test/mitmproxy/addons/test_upstream_proxy_auth.py
+++ /dev/null
@@ -1,65 +0,0 @@
-import base64
-
-from mitmproxy import exceptions
-from mitmproxy.test import taddons
-from mitmproxy.test import tflow
-from mitmproxy.test import tutils
-from mitmproxy.addons import upstream_proxy_auth
-
-
-def test_configure():
-    up = upstream_proxy_auth.UpstreamProxyAuth()
-    with taddons.context() as tctx:
-        tctx.configure(up, upstream_auth="test:test")
-        assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:test")
-
-        tctx.configure(up, upstream_auth="test:")
-        assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:")
-
-        tctx.configure(up, upstream_auth=None)
-        assert not up.auth
-
-        tutils.raises(
-            exceptions.OptionsError,
-            tctx.configure,
-            up,
-            upstream_auth=""
-        )
-        tutils.raises(
-            exceptions.OptionsError,
-            tctx.configure,
-            up,
-            upstream_auth=":"
-        )
-        tutils.raises(
-            exceptions.OptionsError,
-            tctx.configure,
-            up,
-            upstream_auth=":test"
-        )
-
-
-def test_simple():
-    up = upstream_proxy_auth.UpstreamProxyAuth()
-    with taddons.context() as tctx:
-        tctx.configure(up, upstream_auth="foo:bar")
-
-        f = tflow.tflow()
-        f.mode = "upstream"
-        up.requestheaders(f)
-        assert "proxy-authorization" in f.request.headers
-
-        f = tflow.tflow()
-        up.requestheaders(f)
-        assert "proxy-authorization" not in f.request.headers
-
-        tctx.configure(up, mode="reverse")
-        f = tflow.tflow()
-        f.mode = "transparent"
-        up.requestheaders(f)
-        assert "proxy-authorization" in f.request.headers
-
-        f = tflow.tflow()
-        f.mode = "upstream"
-        up.http_connect(f)
-        assert "proxy-authorization" in f.request.headers
-- 
cgit v1.2.3


From e644d2167c5fe591d475ca1a371690f89a6f0878 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sun, 13 Nov 2016 12:32:20 +1300
Subject: stub out proxyauth addon

Stub out basic workings, add and test configure event.
---
 mitmproxy/addons/proxyauth.py           | 68 +++++++++++++++++++++++++++++++++
 test/mitmproxy/addons/test_proxyauth.py | 53 +++++++++++++++++++++++++
 2 files changed, 121 insertions(+)
 create mode 100644 mitmproxy/addons/proxyauth.py
 create mode 100644 test/mitmproxy/addons/test_proxyauth.py

diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py
new file mode 100644
index 00000000..fc68de71
--- /dev/null
+++ b/mitmproxy/addons/proxyauth.py
@@ -0,0 +1,68 @@
+import binascii
+
+import passlib.apache
+
+from mitmproxy import exceptions
+
+
+def parse_http_basic_auth(s):
+    words = s.split()
+    if len(words) != 2:
+        return None
+    scheme = words[0]
+    try:
+        user = binascii.a2b_base64(words[1]).decode("utf8", "replace")
+    except binascii.Error:
+        return None
+    parts = user.split(':')
+    if len(parts) != 2:
+        return None
+    return scheme, parts[0], parts[1]
+
+
+def assemble_http_basic_auth(scheme, username, password):
+    v = binascii.b2a_base64(
+        (username + ":" + password).encode("utf8")
+    ).decode("ascii")
+    return scheme + " " + v
+
+
+class ProxyAuth:
+    def __init__(self):
+        self.nonanonymous = False
+        self.htpasswd = None
+        self.singleuser = None
+
+    def configure(self, options, updated):
+        if "auth_nonanonymous" in updated:
+            self.nonanonymous = options.auth_nonanonymous
+        if "auth_singleuser" in updated:
+            if options.auth_singleuser:
+                parts = options.auth_singleuser.split(':')
+                if len(parts) != 2:
+                    raise exceptions.OptionsError(
+                        "Invalid single-user auth specification."
+                    )
+                self.singleuser = parts
+            else:
+                self.singleuser = None
+        if "auth_htpasswd" in updated:
+            if options.auth_htpasswd:
+                try:
+                    self.htpasswd = passlib.apache.HtpasswdFile(
+                        options.auth_htpasswd
+                    )
+                except (ValueError, OSError) as v:
+                    raise exceptions.OptionsError(
+                        "Could not open htpasswd file: %s" % v
+                    )
+            else:
+                self.auth_htpasswd = None
+
+    def http_connect(self, f):
+        # mode = regular
+        pass
+
+    def http_request(self, f):
+        # mode = regular, no via
+        pass
diff --git a/test/mitmproxy/addons/test_proxyauth.py b/test/mitmproxy/addons/test_proxyauth.py
new file mode 100644
index 00000000..e9dcf7bf
--- /dev/null
+++ b/test/mitmproxy/addons/test_proxyauth.py
@@ -0,0 +1,53 @@
+import binascii
+
+from mitmproxy import exceptions
+from mitmproxy.test import taddons
+from mitmproxy.test import tflow
+from mitmproxy.test import tutils
+from mitmproxy.addons import proxyauth
+
+
+def test_parse_http_basic_auth():
+    vals = ("basic", "foo", "bar")
+    assert proxyauth.parse_http_basic_auth(
+        proxyauth.assemble_http_basic_auth(*vals)
+    ) == vals
+    assert not proxyauth.parse_http_basic_auth("")
+    assert not proxyauth.parse_http_basic_auth("foo bar")
+    v = "basic " + binascii.b2a_base64(b"foo").decode("ascii")
+    assert not proxyauth.parse_http_basic_auth(v)
+
+
+def test_configure():
+    up = proxyauth.ProxyAuth()
+    with taddons.context() as ctx:
+        tutils.raises(
+            exceptions.OptionsError,
+            ctx.configure, up, auth_singleuser="foo"
+        )
+
+        ctx.configure(up, auth_singleuser="foo:bar")
+        assert up.singleuser == ["foo", "bar"]
+
+        ctx.configure(up, auth_singleuser=None)
+        assert up.singleuser is None
+
+        ctx.configure(up, auth_nonanonymous=True)
+        assert up.nonanonymous
+        ctx.configure(up, auth_nonanonymous=False)
+        assert not up.nonanonymous
+
+        tutils.raises(
+            exceptions.OptionsError,
+            ctx.configure,
+            up,
+            auth_htpasswd = tutils.test_data.path(
+                "mitmproxy/net/data/server.crt"
+            )
+        )
+        tutils.raises(
+            exceptions.OptionsError,
+            ctx.configure,
+            up,
+            auth_htpasswd = "nonexistent"
+        )
-- 
cgit v1.2.3


From dc88b7d1102e0bf2d0634fe22682ce4e66ebf772 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sun, 13 Nov 2016 18:14:23 +1300
Subject: addons.proxyauth: complete and test

---
 mitmproxy/addons/proxyauth.py           |  99 ++++++++++++++++++++++----
 test/mitmproxy/addons/test_proxyauth.py | 122 +++++++++++++++++++++++++++++++-
 2 files changed, 205 insertions(+), 16 deletions(-)

diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py
index fc68de71..aeeb04f3 100644
--- a/mitmproxy/addons/proxyauth.py
+++ b/mitmproxy/addons/proxyauth.py
@@ -3,6 +3,11 @@ import binascii
 import passlib.apache
 
 from mitmproxy import exceptions
+from mitmproxy import http
+import mitmproxy.net.http
+
+
+REALM = "mitmproxy"
 
 
 def parse_http_basic_auth(s):
@@ -20,19 +25,74 @@ def parse_http_basic_auth(s):
     return scheme, parts[0], parts[1]
 
 
-def assemble_http_basic_auth(scheme, username, password):
-    v = binascii.b2a_base64(
-        (username + ":" + password).encode("utf8")
-    ).decode("ascii")
-    return scheme + " " + v
-
-
 class ProxyAuth:
     def __init__(self):
         self.nonanonymous = False
         self.htpasswd = None
         self.singleuser = None
 
+    def enabled(self):
+        return any([self.nonanonymous, self.htpasswd, self.singleuser])
+
+    def which_auth_header(self, f):
+        if f.mode == "regular":
+            return 'Proxy-Authorization'
+        else:
+            return 'Authorization'
+
+    def auth_required_response(self, f):
+        if f.mode == "regular":
+            hdrname = 'Proxy-Authenticate'
+        else:
+            hdrname = 'WWW-Authenticate'
+
+        headers = mitmproxy.net.http.Headers()
+        headers[hdrname] = 'Basic realm="%s"' % REALM
+
+        if f.mode == "transparent":
+            return http.make_error_response(
+                401,
+                "Authentication Required",
+                headers
+            )
+        else:
+            return http.make_error_response(
+                407,
+                "Proxy Authentication Required",
+                headers,
+            )
+
+    def check(self, f):
+        auth_value = f.request.headers.get(self.which_auth_header(f), None)
+        if not auth_value:
+            return False
+        parts = parse_http_basic_auth(auth_value)
+        if not parts:
+            return False
+        scheme, username, password = parts
+        if scheme.lower() != 'basic':
+            return False
+
+        if self.nonanonymous:
+            pass
+        elif self.singleuser:
+            if [username, password] != self.singleuser:
+                return False
+        elif self.htpasswd:
+            if not self.htpasswd.check_password(username, password):
+                return False
+        else:
+            raise NotImplementedError("Should never happen.")
+
+        return True
+
+    def authenticate(self, f):
+        if self.check(f):
+            del f.request.headers[self.which_auth_header(f)]
+        else:
+            f.response = self.auth_required_response(f)
+
+    # Handlers
     def configure(self, options, updated):
         if "auth_nonanonymous" in updated:
             self.nonanonymous = options.auth_nonanonymous
@@ -57,12 +117,25 @@ class ProxyAuth:
                         "Could not open htpasswd file: %s" % v
                     )
             else:
-                self.auth_htpasswd = None
+                self.htpasswd = None
+        if self.enabled():
+            if options.mode == "transparent":
+                raise exceptions.OptionsError(
+                    "Proxy Authentication not supported in transparent mode."
+                )
+            elif options.mode == "socks5":
+                raise exceptions.OptionsError(
+                    "Proxy Authentication not supported in SOCKS mode. "
+                    "https://github.com/mitmproxy/mitmproxy/issues/738"
+                )
+            # TODO: check for multiple auth options
 
     def http_connect(self, f):
-        # mode = regular
-        pass
+        if self.enabled() and f.mode == "regular":
+            self.authenticate(f)
 
-    def http_request(self, f):
-        # mode = regular, no via
-        pass
+    def requestheaders(self, f):
+        if self.enabled():
+            # Are we already authenticated in CONNECT?
+            if not (f.mode == "regular" and f.server_conn.via):
+                self.authenticate(f)
diff --git a/test/mitmproxy/addons/test_proxyauth.py b/test/mitmproxy/addons/test_proxyauth.py
index e9dcf7bf..73d87cbf 100644
--- a/test/mitmproxy/addons/test_proxyauth.py
+++ b/test/mitmproxy/addons/test_proxyauth.py
@@ -7,11 +7,17 @@ from mitmproxy.test import tutils
 from mitmproxy.addons import proxyauth
 
 
+def mkauth(username, password, scheme="basic"):
+    v = binascii.b2a_base64(
+        (username + ":" + password).encode("utf8")
+    ).decode("ascii")
+    return scheme + " " + v
+
+
 def test_parse_http_basic_auth():
-    vals = ("basic", "foo", "bar")
     assert proxyauth.parse_http_basic_auth(
-        proxyauth.assemble_http_basic_auth(*vals)
-    ) == vals
+        mkauth("test", "test")
+    ) == ("basic", "test", "test")
     assert not proxyauth.parse_http_basic_auth("")
     assert not proxyauth.parse_http_basic_auth("foo bar")
     v = "basic " + binascii.b2a_base64(b"foo").decode("ascii")
@@ -51,3 +57,113 @@ def test_configure():
             up,
             auth_htpasswd = "nonexistent"
         )
+
+        ctx.configure(
+            up,
+            auth_htpasswd = tutils.test_data.path(
+                "mitmproxy/net/data/htpasswd"
+            )
+        )
+        assert up.htpasswd
+        assert up.htpasswd.check_password("test", "test")
+        assert not up.htpasswd.check_password("test", "foo")
+        ctx.configure(up, auth_htpasswd = None)
+        assert not up.htpasswd
+
+        tutils.raises(
+            exceptions.OptionsError,
+            ctx.configure,
+            up,
+            auth_nonanonymous = True,
+            mode = "transparent"
+        )
+        tutils.raises(
+            exceptions.OptionsError,
+            ctx.configure,
+            up,
+            auth_nonanonymous = True,
+            mode = "socks5"
+        )
+
+
+def test_check():
+    up = proxyauth.ProxyAuth()
+    with taddons.context() as ctx:
+        ctx.configure(up, auth_nonanonymous=True)
+        f = tflow.tflow()
+        assert not up.check(f)
+        f.request.headers["Proxy-Authorization"] = mkauth("test", "test")
+        assert up.check(f)
+
+        f.request.headers["Proxy-Authorization"] = "invalid"
+        assert not up.check(f)
+
+        f.request.headers["Proxy-Authorization"] = mkauth(
+            "test", "test", scheme = "unknown"
+        )
+        assert not up.check(f)
+
+        ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:test")
+        f.request.headers["Proxy-Authorization"] = mkauth("test", "test")
+        assert up.check(f)
+        ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:foo")
+        assert not up.check(f)
+
+        ctx.configure(
+            up,
+            auth_singleuser = None,
+            auth_htpasswd = tutils.test_data.path(
+                "mitmproxy/net/data/htpasswd"
+            )
+        )
+        f.request.headers["Proxy-Authorization"] = mkauth("test", "test")
+        assert up.check(f)
+        f.request.headers["Proxy-Authorization"] = mkauth("test", "foo")
+        assert not up.check(f)
+
+
+def test_authenticate():
+    up = proxyauth.ProxyAuth()
+    with taddons.context() as ctx:
+        ctx.configure(up, auth_nonanonymous=True)
+
+        f = tflow.tflow()
+        assert not f.response
+        up.authenticate(f)
+        assert f.response.status_code == 407
+
+        f = tflow.tflow()
+        f.request.headers["Proxy-Authorization"] = mkauth("test", "test")
+        up.authenticate(f)
+        assert not f.response
+        assert not f.request.headers.get("Proxy-Authorization")
+
+        f = tflow.tflow()
+        f.mode = "transparent"
+        assert not f.response
+        up.authenticate(f)
+        assert f.response.status_code == 401
+
+        f = tflow.tflow()
+        f.mode = "transparent"
+        f.request.headers["Authorization"] = mkauth("test", "test")
+        up.authenticate(f)
+        assert not f.response
+        assert not f.request.headers.get("Authorization")
+
+
+def test_handlers():
+    up = proxyauth.ProxyAuth()
+    with taddons.context() as ctx:
+        ctx.configure(up, auth_nonanonymous=True)
+
+        f = tflow.tflow()
+        assert not f.response
+        up.requestheaders(f)
+        assert f.response.status_code == 407
+
+        f = tflow.tflow()
+        f.request.method = "CONNECT"
+        assert not f.response
+        up.http_connect(f)
+        assert f.response.status_code == 407
-- 
cgit v1.2.3


From 9b08279c7c3384f716b66329fefbe97a368189a2 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sun, 13 Nov 2016 18:45:27 +1300
Subject: addons.proxyauth: out with the old, in with the new

- Strip out old auth mechanisms, and enable addon
- Disable web app auth for now - this should just use the Tornado auth stuff
---
 docs/scripting/events.rst                      |   9 +-
 mitmproxy/addons/__init__.py                   |   2 +
 mitmproxy/addons/proxyauth.py                  |   7 +
 mitmproxy/net/http/authentication.py           | 176 -------------------------
 mitmproxy/proxy/config.py                      |  48 -------
 mitmproxy/proxy/protocol/http.py               |  33 -----
 mitmproxy/tools/web/master.py                  |  42 +++---
 test/mitmproxy/addons/test_proxyauth.py        |  35 ++---
 test/mitmproxy/net/http/test_authentication.py | 122 -----------------
 test/mitmproxy/test_eventsequence.py           |   3 +-
 test/mitmproxy/test_proxy.py                   |  27 ----
 test/mitmproxy/test_server.py                  |  12 +-
 12 files changed, 63 insertions(+), 453 deletions(-)
 delete mode 100644 mitmproxy/net/http/authentication.py
 delete mode 100644 test/mitmproxy/net/http/test_authentication.py

diff --git a/docs/scripting/events.rst b/docs/scripting/events.rst
index a5721403..5f560e58 100644
--- a/docs/scripting/events.rst
+++ b/docs/scripting/events.rst
@@ -99,12 +99,11 @@ HTTP Events
     :header-rows: 0
 
     *   - .. py:function:: http_connect(flow)
-
         - Called when we receive an HTTP CONNECT request. Setting a non 2xx
-        response on the flow will return the response to the client abort the
-        connection. CONNECT requests and responses do not generate the usual
-        HTTP handler events. CONNECT requests are only valid in regular and
-        upstream proxy modes.
+          response on the flow will return the response to the client abort the
+          connection. CONNECT requests and responses do not generate the usual
+          HTTP handler events. CONNECT requests are only valid in regular and
+          upstream proxy modes.
 
           *flow*
             A ``models.HTTPFlow`` object. The flow is guaranteed to have
diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py
index d71b8912..71d83dad 100644
--- a/mitmproxy/addons/__init__.py
+++ b/mitmproxy/addons/__init__.py
@@ -3,6 +3,7 @@ from mitmproxy.addons import anticomp
 from mitmproxy.addons import clientplayback
 from mitmproxy.addons import streamfile
 from mitmproxy.addons import onboarding
+from mitmproxy.addons import proxyauth
 from mitmproxy.addons import replace
 from mitmproxy.addons import script
 from mitmproxy.addons import setheaders
@@ -16,6 +17,7 @@ from mitmproxy.addons import upstream_auth
 def default_addons():
     return [
         onboarding.Onboarding(),
+        proxyauth.ProxyAuth(),
         anticache.AntiCache(),
         anticomp.AntiComp(),
         stickyauth.StickyAuth(),
diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py
index aeeb04f3..69d45029 100644
--- a/mitmproxy/addons/proxyauth.py
+++ b/mitmproxy/addons/proxyauth.py
@@ -10,6 +10,13 @@ import mitmproxy.net.http
 REALM = "mitmproxy"
 
 
+def mkauth(username, password, scheme="basic"):
+    v = binascii.b2a_base64(
+        (username + ":" + password).encode("utf8")
+    ).decode("ascii")
+    return scheme + " " + v
+
+
 def parse_http_basic_auth(s):
     words = s.split()
     if len(words) != 2:
diff --git a/mitmproxy/net/http/authentication.py b/mitmproxy/net/http/authentication.py
deleted file mode 100644
index a65279e4..00000000
--- a/mitmproxy/net/http/authentication.py
+++ /dev/null
@@ -1,176 +0,0 @@
-import argparse
-import binascii
-
-
-def parse_http_basic_auth(s):
-    words = s.split()
-    if len(words) != 2:
-        return None
-    scheme = words[0]
-    try:
-        user = binascii.a2b_base64(words[1]).decode("utf8", "replace")
-    except binascii.Error:
-        return None
-    parts = user.split(':')
-    if len(parts) != 2:
-        return None
-    return scheme, parts[0], parts[1]
-
-
-def assemble_http_basic_auth(scheme, username, password):
-    v = binascii.b2a_base64((username + ":" + password).encode("utf8")).decode("ascii")
-    return scheme + " " + v
-
-
-class NullProxyAuth:
-
-    """
-        No proxy auth at all (returns empty challange headers)
-    """
-
-    def __init__(self, password_manager):
-        self.password_manager = password_manager
-
-    def clean(self, headers_):
-        """
-            Clean up authentication headers, so they're not passed upstream.
-        """
-
-    def authenticate(self, headers_):
-        """
-            Tests that the user is allowed to use the proxy
-        """
-        return True
-
-    def auth_challenge_headers(self):
-        """
-            Returns a dictionary containing the headers require to challenge the user
-        """
-        return {}
-
-
-class BasicAuth(NullProxyAuth):
-    CHALLENGE_HEADER = None
-    AUTH_HEADER = None
-
-    def __init__(self, password_manager, realm):
-        NullProxyAuth.__init__(self, password_manager)
-        self.realm = realm
-
-    def clean(self, headers):
-        del headers[self.AUTH_HEADER]
-
-    def authenticate(self, headers):
-        auth_value = headers.get(self.AUTH_HEADER)
-        if not auth_value:
-            return False
-        parts = parse_http_basic_auth(auth_value)
-        if not parts:
-            return False
-        scheme, username, password = parts
-        if scheme.lower() != 'basic':
-            return False
-        if not self.password_manager.test(username, password):
-            return False
-        self.username = username
-        return True
-
-    def auth_challenge_headers(self):
-        return {self.CHALLENGE_HEADER: 'Basic realm="%s"' % self.realm}
-
-
-class BasicWebsiteAuth(BasicAuth):
-    CHALLENGE_HEADER = 'WWW-Authenticate'
-    AUTH_HEADER = 'Authorization'
-
-
-class BasicProxyAuth(BasicAuth):
-    CHALLENGE_HEADER = 'Proxy-Authenticate'
-    AUTH_HEADER = 'Proxy-Authorization'
-
-
-class PassMan:
-
-    def test(self, username_, password_token_):
-        return False
-
-
-class PassManNonAnon(PassMan):
-
-    """
-        Ensure the user specifies a username, accept any password.
-    """
-
-    def test(self, username, password_token_):
-        if username:
-            return True
-        return False
-
-
-class PassManHtpasswd(PassMan):
-
-    """
-        Read usernames and passwords from an htpasswd file
-    """
-
-    def __init__(self, path):
-        """
-            Raises ValueError if htpasswd file is invalid.
-        """
-        import passlib.apache
-        self.htpasswd = passlib.apache.HtpasswdFile(path)
-
-    def test(self, username, password_token):
-        return bool(self.htpasswd.check_password(username, password_token))
-
-
-class PassManSingleUser(PassMan):
-
-    def __init__(self, username, password):
-        self.username, self.password = username, password
-
-    def test(self, username, password_token):
-        return self.username == username and self.password == password_token
-
-
-class AuthAction(argparse.Action):
-
-    """
-    Helper class to allow seamless integration int argparse. Example usage:
-    parser.add_argument(
-        "--nonanonymous",
-        action=NonanonymousAuthAction, nargs=0,
-        help="Allow access to any user long as a credentials are specified."
-    )
-    """
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        passman = self.getPasswordManager(values)
-        authenticator = BasicProxyAuth(passman, "mitmproxy")
-        setattr(namespace, self.dest, authenticator)
-
-    def getPasswordManager(self, s):  # pragma: no cover
-        raise NotImplementedError()
-
-
-class SingleuserAuthAction(AuthAction):
-
-    def getPasswordManager(self, s):
-        if len(s.split(':')) != 2:
-            raise argparse.ArgumentTypeError(
-                "Invalid single-user specification. Please use the format username:password"
-            )
-        username, password = s.split(':')
-        return PassManSingleUser(username, password)
-
-
-class NonanonymousAuthAction(AuthAction):
-
-    def getPasswordManager(self, s):
-        return PassManNonAnon()
-
-
-class HtpasswdAuthAction(AuthAction):
-
-    def getPasswordManager(self, s):
-        return PassManHtpasswd(s)
diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py
index e144c175..513c0b5b 100644
--- a/mitmproxy/proxy/config.py
+++ b/mitmproxy/proxy/config.py
@@ -9,7 +9,6 @@ from mitmproxy import exceptions
 from mitmproxy import options as moptions
 from mitmproxy import certs
 from mitmproxy.net import tcp
-from mitmproxy.net.http import authentication
 from mitmproxy.net.http import url
 
 CONF_BASENAME = "mitmproxy"
@@ -58,7 +57,6 @@ class ProxyConfig:
     def __init__(self, options: moptions.Options) -> None:
         self.options = options
 
-        self.authenticator = None
         self.check_ignore = None
         self.check_tcp = None
         self.certstore = None
@@ -124,49 +122,3 @@ class ProxyConfig:
         self.upstream_server = None
         if options.upstream_server:
             self.upstream_server = parse_server_spec(options.upstream_server)
-
-        self.authenticator = authentication.NullProxyAuth(None)
-        needsauth = any(
-            [
-                options.auth_nonanonymous,
-                options.auth_singleuser,
-                options.auth_htpasswd
-            ]
-        )
-        if needsauth:
-            if options.mode == "transparent":
-                raise exceptions.OptionsError(
-                    "Proxy Authentication not supported in transparent mode."
-                )
-            elif options.mode == "socks5":
-                raise exceptions.OptionsError(
-                    "Proxy Authentication not supported in SOCKS mode. "
-                    "https://github.com/mitmproxy/mitmproxy/issues/738"
-                )
-            elif options.auth_singleuser:
-                parts = options.auth_singleuser.split(':')
-                if len(parts) != 2:
-                    raise exceptions.OptionsError(
-                        "Invalid single-user specification. "
-                        "Please use the format username:password"
-                    )
-                password_manager = authentication.PassManSingleUser(*parts)
-            elif options.auth_nonanonymous:
-                password_manager = authentication.PassManNonAnon()
-            elif options.auth_htpasswd:
-                try:
-                    password_manager = authentication.PassManHtpasswd(
-                        options.auth_htpasswd
-                    )
-                except ValueError as v:
-                    raise exceptions.OptionsError(str(v))
-            if options.mode == "reverse":
-                self.authenticator = authentication.BasicWebsiteAuth(
-                    password_manager,
-                    self.upstream_server.address
-                )
-            else:
-                self.authenticator = authentication.BasicProxyAuth(
-                    password_manager,
-                    "mitmproxy"
-                )
diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index 5165018d..5f9dafab 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -8,7 +8,6 @@ from mitmproxy import http
 from mitmproxy import flow
 from mitmproxy.proxy.protocol import base
 from mitmproxy.proxy.protocol import websockets as pwebsockets
-import mitmproxy.net.http
 from mitmproxy.net import tcp
 from mitmproxy.net import websockets
 
@@ -124,7 +123,6 @@ class HTTPMode(enum.Enum):
     upstream = 3
 
 
-FIRSTLINES = set(["absolute", "relative", "authority"])
 # At this point, we see only a subset of the proxy modes
 MODE_REQUEST_FORMS = {
     HTTPMode.regular: ("authority", "absolute"),
@@ -270,13 +268,6 @@ class HttpLayer(base.Layer):
 
         self.log("request", "debug", [repr(request)])
 
-        # Handle Proxy Authentication Proxy Authentication conceptually does
-        # not work in transparent mode. We catch this misconfiguration on
-        # startup. Here, we sort out requests after a successful CONNECT
-        # request (which do not need to be validated anymore)
-        if not self.connect_request and not self.authenticate(request):
-            return False
-
         # update host header in reverse proxy mode
         if self.config.options.mode == "reverse":
             f.request.headers["Host"] = self.config.upstream_server.address.host
@@ -455,27 +446,3 @@ class HttpLayer(base.Layer):
                 self.connect()
             if tls:
                 raise exceptions.HttpProtocolException("Cannot change scheme in upstream proxy mode.")
-
-    def authenticate(self, request) -> bool:
-        if self.config.authenticator:
-            if self.config.authenticator.authenticate(request.headers):
-                self.config.authenticator.clean(request.headers)
-            else:
-                if self.mode == HTTPMode.transparent:
-                    self.send_response(http.make_error_response(
-                        401,
-                        "Authentication Required",
-                        mitmproxy.net.http.Headers(
-                            **self.config.authenticator.auth_challenge_headers()
-                        )
-                    ))
-                else:
-                    self.send_response(http.make_error_response(
-                        407,
-                        "Proxy Authentication Required",
-                        mitmproxy.net.http.Headers(
-                            **self.config.authenticator.auth_challenge_headers()
-                        )
-                    ))
-                return False
-        return True
diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py
index e35815ad..2cb5953f 100644
--- a/mitmproxy/tools/web/master.py
+++ b/mitmproxy/tools/web/master.py
@@ -12,7 +12,6 @@ from mitmproxy.addons import intercept
 from mitmproxy import options
 from mitmproxy import master
 from mitmproxy.tools.web import app
-from mitmproxy.net.http import authentication
 
 
 class Stop(Exception):
@@ -52,7 +51,7 @@ class Options(options.Options):
             wdebug: bool = False,
             wport: int = 8081,
             wiface: str = "127.0.0.1",
-            wauthenticator: Optional[authentication.PassMan] = None,
+            # wauthenticator: Optional[authentication.PassMan] = None,
             wsingleuser: Optional[str] = None,
             whtpasswd: Optional[str] = None,
             **kwargs
@@ -60,29 +59,30 @@ class Options(options.Options):
         self.wdebug = wdebug
         self.wport = wport
         self.wiface = wiface
-        self.wauthenticator = wauthenticator
-        self.wsingleuser = wsingleuser
-        self.whtpasswd = whtpasswd
+        # self.wauthenticator = wauthenticator
+        # self.wsingleuser = wsingleuser
+        # self.whtpasswd = whtpasswd
         self.intercept = intercept
         super().__init__(**kwargs)
 
     # TODO: This doesn't belong here.
     def process_web_options(self, parser):
-        if self.wsingleuser or self.whtpasswd:
-            if self.wsingleuser:
-                if len(self.wsingleuser.split(':')) != 2:
-                    return parser.error(
-                        "Invalid single-user specification. Please use the format username:password"
-                    )
-                username, password = self.wsingleuser.split(':')
-                self.wauthenticator = authentication.PassManSingleUser(username, password)
-            elif self.whtpasswd:
-                try:
-                    self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd)
-                except ValueError as v:
-                    return parser.error(v.message)
-        else:
-            self.wauthenticator = None
+        # if self.wsingleuser or self.whtpasswd:
+        #     if self.wsingleuser:
+        #         if len(self.wsingleuser.split(':')) != 2:
+        #             return parser.error(
+        #                 "Invalid single-user specification. Please use the format username:password"
+        #             )
+        #         username, password = self.wsingleuser.split(':')
+        #         # self.wauthenticator = authentication.PassManSingleUser(username, password)
+        #     elif self.whtpasswd:
+        #         try:
+        #             self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd)
+        #         except ValueError as v:
+        #             return parser.error(v.message)
+        # else:
+        #     self.wauthenticator = None
+        pass
 
 
 class WebMaster(master.Master):
@@ -98,7 +98,7 @@ class WebMaster(master.Master):
         self.addons.add(*addons.default_addons())
         self.addons.add(self.view, intercept.Intercept())
         self.app = app.Application(
-            self, self.options.wdebug, self.options.wauthenticator
+            self, self.options.wdebug, False
         )
         # This line is just for type hinting
         self.options = self.options  # type: Options
diff --git a/test/mitmproxy/addons/test_proxyauth.py b/test/mitmproxy/addons/test_proxyauth.py
index 73d87cbf..494a992f 100644
--- a/test/mitmproxy/addons/test_proxyauth.py
+++ b/test/mitmproxy/addons/test_proxyauth.py
@@ -7,16 +7,9 @@ from mitmproxy.test import tutils
 from mitmproxy.addons import proxyauth
 
 
-def mkauth(username, password, scheme="basic"):
-    v = binascii.b2a_base64(
-        (username + ":" + password).encode("utf8")
-    ).decode("ascii")
-    return scheme + " " + v
-
-
 def test_parse_http_basic_auth():
     assert proxyauth.parse_http_basic_auth(
-        mkauth("test", "test")
+        proxyauth.mkauth("test", "test")
     ) == ("basic", "test", "test")
     assert not proxyauth.parse_http_basic_auth("")
     assert not proxyauth.parse_http_basic_auth("foo bar")
@@ -92,19 +85,23 @@ def test_check():
         ctx.configure(up, auth_nonanonymous=True)
         f = tflow.tflow()
         assert not up.check(f)
-        f.request.headers["Proxy-Authorization"] = mkauth("test", "test")
+        f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
+            "test", "test"
+        )
         assert up.check(f)
 
         f.request.headers["Proxy-Authorization"] = "invalid"
         assert not up.check(f)
 
-        f.request.headers["Proxy-Authorization"] = mkauth(
+        f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
             "test", "test", scheme = "unknown"
         )
         assert not up.check(f)
 
         ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:test")
-        f.request.headers["Proxy-Authorization"] = mkauth("test", "test")
+        f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
+            "test", "test"
+        )
         assert up.check(f)
         ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:foo")
         assert not up.check(f)
@@ -116,9 +113,13 @@ def test_check():
                 "mitmproxy/net/data/htpasswd"
             )
         )
-        f.request.headers["Proxy-Authorization"] = mkauth("test", "test")
+        f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
+            "test", "test"
+        )
         assert up.check(f)
-        f.request.headers["Proxy-Authorization"] = mkauth("test", "foo")
+        f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
+            "test", "foo"
+        )
         assert not up.check(f)
 
 
@@ -133,7 +134,9 @@ def test_authenticate():
         assert f.response.status_code == 407
 
         f = tflow.tflow()
-        f.request.headers["Proxy-Authorization"] = mkauth("test", "test")
+        f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
+            "test", "test"
+        )
         up.authenticate(f)
         assert not f.response
         assert not f.request.headers.get("Proxy-Authorization")
@@ -146,7 +149,9 @@ def test_authenticate():
 
         f = tflow.tflow()
         f.mode = "transparent"
-        f.request.headers["Authorization"] = mkauth("test", "test")
+        f.request.headers["Authorization"] = proxyauth.mkauth(
+            "test", "test"
+        )
         up.authenticate(f)
         assert not f.response
         assert not f.request.headers.get("Authorization")
diff --git a/test/mitmproxy/net/http/test_authentication.py b/test/mitmproxy/net/http/test_authentication.py
deleted file mode 100644
index 01eae52d..00000000
--- a/test/mitmproxy/net/http/test_authentication.py
+++ /dev/null
@@ -1,122 +0,0 @@
-import binascii
-
-from mitmproxy.test import tutils
-from mitmproxy.net.http import authentication, Headers
-
-
-def test_parse_http_basic_auth():
-    vals = ("basic", "foo", "bar")
-    assert authentication.parse_http_basic_auth(
-        authentication.assemble_http_basic_auth(*vals)
-    ) == vals
-    assert not authentication.parse_http_basic_auth("")
-    assert not authentication.parse_http_basic_auth("foo bar")
-    v = "basic " + binascii.b2a_base64(b"foo").decode("ascii")
-    assert not authentication.parse_http_basic_auth(v)
-
-
-class TestPassManNonAnon:
-
-    def test_simple(self):
-        p = authentication.PassManNonAnon()
-        assert not p.test("", "")
-        assert p.test("user", "")
-
-
-class TestPassManHtpasswd:
-
-    def test_file_errors(self):
-        tutils.raises(
-            "malformed htpasswd file",
-            authentication.PassManHtpasswd,
-            tutils.test_data.path("mitmproxy/net/data/server.crt"))
-
-    def test_simple(self):
-        pm = authentication.PassManHtpasswd(tutils.test_data.path("mitmproxy/net/data/htpasswd"))
-
-        vals = ("basic", "test", "test")
-        authentication.assemble_http_basic_auth(*vals)
-        assert pm.test("test", "test")
-        assert not pm.test("test", "foo")
-        assert not pm.test("foo", "test")
-        assert not pm.test("test", "")
-        assert not pm.test("", "")
-
-
-class TestPassManSingleUser:
-
-    def test_simple(self):
-        pm = authentication.PassManSingleUser("test", "test")
-        assert pm.test("test", "test")
-        assert not pm.test("test", "foo")
-        assert not pm.test("foo", "test")
-
-
-class TestNullProxyAuth:
-
-    def test_simple(self):
-        na = authentication.NullProxyAuth(authentication.PassManNonAnon())
-        assert not na.auth_challenge_headers()
-        assert na.authenticate("foo")
-        na.clean({})
-
-
-class TestBasicProxyAuth:
-
-    def test_simple(self):
-        ba = authentication.BasicProxyAuth(authentication.PassManNonAnon(), "test")
-        headers = Headers()
-        assert ba.auth_challenge_headers()
-        assert not ba.authenticate(headers)
-
-    def test_authenticate_clean(self):
-        ba = authentication.BasicProxyAuth(authentication.PassManNonAnon(), "test")
-
-        headers = Headers()
-        vals = ("basic", "foo", "bar")
-        headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(*vals)
-        assert ba.authenticate(headers)
-
-        ba.clean(headers)
-        assert ba.AUTH_HEADER not in headers
-
-        headers[ba.AUTH_HEADER] = ""
-        assert not ba.authenticate(headers)
-
-        headers[ba.AUTH_HEADER] = "foo"
-        assert not ba.authenticate(headers)
-
-        vals = ("foo", "foo", "bar")
-        headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(*vals)
-        assert not ba.authenticate(headers)
-
-        ba = authentication.BasicProxyAuth(authentication.PassMan(), "test")
-        vals = ("basic", "foo", "bar")
-        headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(*vals)
-        assert not ba.authenticate(headers)
-
-
-class Bunch:
-    pass
-
-
-class TestAuthAction:
-
-    def test_nonanonymous(self):
-        m = Bunch()
-        aa = authentication.NonanonymousAuthAction(None, "authenticator")
-        aa(None, m, None, None)
-        assert m.authenticator
-
-    def test_singleuser(self):
-        m = Bunch()
-        aa = authentication.SingleuserAuthAction(None, "authenticator")
-        aa(None, m, "foo:bar", None)
-        assert m.authenticator
-        tutils.raises("invalid", aa, None, m, "foo", None)
-
-    def test_httppasswd(self):
-        m = Bunch()
-        aa = authentication.HtpasswdAuthAction(None, "authenticator")
-        aa(None, m, tutils.test_data.path("mitmproxy/net/data/htpasswd"), None)
-        assert m.authenticator
diff --git a/test/mitmproxy/test_eventsequence.py b/test/mitmproxy/test_eventsequence.py
index e6eb6569..262df4b0 100644
--- a/test/mitmproxy/test_eventsequence.py
+++ b/test/mitmproxy/test_eventsequence.py
@@ -67,7 +67,8 @@ class TestBasic(tservers.HTTPProxyTest, SequenceTester):
                         da
                     """
                 )
-            assert e.called[-1] == "requestheaders"
+        assert "requestheaders" in e.called
+        assert "responseheaders" not in e.called
 
     def test_connect(self):
         e = Eventer()
diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py
index 8847c088..aa3b8979 100644
--- a/test/mitmproxy/test_proxy.py
+++ b/test/mitmproxy/test_proxy.py
@@ -113,13 +113,6 @@ class TestProcessProxyOptions:
         self.assert_err("expected one argument", "--upstream-auth")
         self.assert_err("mutually exclusive", "-R", "http://localhost", "-T")
 
-    def test_socks_auth(self):
-        self.assert_err(
-            "Proxy Authentication not supported in SOCKS mode.",
-            "--socks",
-            "--nonanonymous"
-        )
-
     def test_client_certs(self):
         with tutils.tmpdir() as cadir:
             self.assert_noerr("--client-certs", cadir)
@@ -137,26 +130,6 @@ class TestProcessProxyOptions:
             tutils.test_data.path("mitmproxy/data/testkey.pem"))
         self.assert_err("does not exist", "--cert", "nonexistent")
 
-    def test_auth(self):
-        p = self.assert_noerr("--nonanonymous")
-        assert p.authenticator
-
-        p = self.assert_noerr(
-            "--htpasswd",
-            tutils.test_data.path("mitmproxy/data/htpasswd"))
-        assert p.authenticator
-        self.assert_err(
-            "malformed htpasswd file",
-            "--htpasswd",
-            tutils.test_data.path("mitmproxy/data/htpasswd.invalid"))
-
-        p = self.assert_noerr("--singleuser", "test:test")
-        assert p.authenticator
-        self.assert_err(
-            "invalid single-user specification",
-            "--singleuser",
-            "test")
-
     def test_insecure(self):
         p = self.assert_noerr("--insecure")
         assert p.openssl_verification_mode_server == SSL.VERIFY_NONE
diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py
index 5a5b6817..9429ab0f 100644
--- a/test/mitmproxy/test_server.py
+++ b/test/mitmproxy/test_server.py
@@ -6,6 +6,7 @@ from mitmproxy.test import tutils
 from mitmproxy import controller
 from mitmproxy import options
 from mitmproxy.addons import script
+from mitmproxy.addons import proxyauth
 from mitmproxy import http
 from mitmproxy.proxy.config import HostMatcher, parse_server_spec
 import mitmproxy.net.http
@@ -13,7 +14,6 @@ from mitmproxy.net import tcp
 from mitmproxy.net import socks
 from mitmproxy import certs
 from mitmproxy import exceptions
-from mitmproxy.net.http import authentication
 from mitmproxy.net.http import http1
 from mitmproxy.net.tcp import Address
 from pathod import pathoc
@@ -285,6 +285,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin):
 
 class TestHTTPAuth(tservers.HTTPProxyTest):
     def test_auth(self):
+        self.master.addons.add(proxyauth.ProxyAuth())
         self.master.options.auth_singleuser = "test:test"
         assert self.pathod("202").status_code == 407
         p = self.pathoc()
@@ -295,14 +296,15 @@ class TestHTTPAuth(tservers.HTTPProxyTest):
                 h'%s'='%s'
             """ % (
                 self.server.port,
-                mitmproxy.net.http.authentication.BasicProxyAuth.AUTH_HEADER,
-                authentication.assemble_http_basic_auth("basic", "test", "test")
+                "Proxy-Authorization",
+                proxyauth.mkauth("test", "test")
             ))
         assert ret.status_code == 202
 
 
 class TestHTTPReverseAuth(tservers.ReverseProxyTest):
     def test_auth(self):
+        self.master.addons.add(proxyauth.ProxyAuth())
         self.master.options.auth_singleuser = "test:test"
         assert self.pathod("202").status_code == 401
         p = self.pathoc()
@@ -312,8 +314,8 @@ class TestHTTPReverseAuth(tservers.ReverseProxyTest):
                 '/p/202'
                 h'%s'='%s'
             """ % (
-                mitmproxy.net.http.authentication.BasicWebsiteAuth.AUTH_HEADER,
-                authentication.assemble_http_basic_auth("basic", "test", "test")
+                "Authorization",
+                proxyauth.mkauth("test", "test")
             ))
         assert ret.status_code == 202
 
-- 
cgit v1.2.3