Merge pull request #1720 from cortesi/proxyrefactor

proxy.protocol.http-related refactoring

24 files changed, 860 insertions, 768 deletions

diff --git a/docs/scripting/events.rst b/docs/scripting/events.rst
index 62266485..5f560e58 100644
--- a/docs/scripting/events.rst
+++ b/docs/scripting/events.rst
@@ -98,6 +98,18 @@ HTTP Events
     :widths: 40 60
     :header-rows: 0
 
+    *   - .. py:function:: http_connect(flow)
+        - Called when we receive an HTTP CONNECT request. Setting a non 2xx
+          response on the flow will return the response to the client abort the
+          connection. CONNECT requests and responses do not generate the usual
+          HTTP handler events. CONNECT requests are only valid in regular and
+          upstream proxy modes.
+
+          *flow*
+            A ``models.HTTPFlow`` object. The flow is guaranteed to have
+            non-None ``request``  and ``requestheaders`` attributes.
+
+
     *   - .. py:function:: request(flow)
         - Called when a client request has been received.
 
diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py
index d2b50c35..71d83dad 100644
--- a/mitmproxy/addons/__init__.py
+++ b/mitmproxy/addons/__init__.py
@@ -3,6 +3,7 @@ from mitmproxy.addons import anticomp
 from mitmproxy.addons import clientplayback
 from mitmproxy.addons import streamfile
 from mitmproxy.addons import onboarding
+from mitmproxy.addons import proxyauth
 from mitmproxy.addons import replace
 from mitmproxy.addons import script
 from mitmproxy.addons import setheaders
@@ -10,11 +11,13 @@ from mitmproxy.addons import serverplayback
 from mitmproxy.addons import stickyauth
 from mitmproxy.addons import stickycookie
 from mitmproxy.addons import streambodies
+from mitmproxy.addons import upstream_auth
 
 
 def default_addons():
     return [
         onboarding.Onboarding(),
+        proxyauth.ProxyAuth(),
         anticache.AntiCache(),
         anticomp.AntiComp(),
         stickyauth.StickyAuth(),
@@ -26,4 +29,5 @@ def default_addons():
         setheaders.SetHeaders(),
         serverplayback.ServerPlayback(),
         clientplayback.ClientPlayback(),
+        upstream_auth.UpstreamAuth(),
     ]
diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py
new file mode 100644
index 00000000..69d45029
--- /dev/null
+++ b/mitmproxy/addons/proxyauth.py
@@ -0,0 +1,148 @@
+import binascii
+
+import passlib.apache
+
+from mitmproxy import exceptions
+from mitmproxy import http
+import mitmproxy.net.http
+
+
+REALM = "mitmproxy"
+
+
+def mkauth(username, password, scheme="basic"):
+    v = binascii.b2a_base64(
+        (username + ":" + password).encode("utf8")
+    ).decode("ascii")
+    return scheme + " " + v
+
+
+def parse_http_basic_auth(s):
+    words = s.split()
+    if len(words) != 2:
+        return None
+    scheme = words[0]
+    try:
+        user = binascii.a2b_base64(words[1]).decode("utf8", "replace")
+    except binascii.Error:
+        return None
+    parts = user.split(':')
+    if len(parts) != 2:
+        return None
+    return scheme, parts[0], parts[1]
+
+
+class ProxyAuth:
+    def __init__(self):
+        self.nonanonymous = False
+        self.htpasswd = None
+        self.singleuser = None
+
+    def enabled(self):
+        return any([self.nonanonymous, self.htpasswd, self.singleuser])
+
+    def which_auth_header(self, f):
+        if f.mode == "regular":
+            return 'Proxy-Authorization'
+        else:
+            return 'Authorization'
+
+    def auth_required_response(self, f):
+        if f.mode == "regular":
+            hdrname = 'Proxy-Authenticate'
+        else:
+            hdrname = 'WWW-Authenticate'
+
+        headers = mitmproxy.net.http.Headers()
+        headers[hdrname] = 'Basic realm="%s"' % REALM
+
+        if f.mode == "transparent":
+            return http.make_error_response(
+                401,
+                "Authentication Required",
+                headers
+            )
+        else:
+            return http.make_error_response(
+                407,
+                "Proxy Authentication Required",
+                headers,
+            )
+
+    def check(self, f):
+        auth_value = f.request.headers.get(self.which_auth_header(f), None)
+        if not auth_value:
+            return False
+        parts = parse_http_basic_auth(auth_value)
+        if not parts:
+            return False
+        scheme, username, password = parts
+        if scheme.lower() != 'basic':
+            return False
+
+        if self.nonanonymous:
+            pass
+        elif self.singleuser:
+            if [username, password] != self.singleuser:
+                return False
+        elif self.htpasswd:
+            if not self.htpasswd.check_password(username, password):
+                return False
+        else:
+            raise NotImplementedError("Should never happen.")
+
+        return True
+
+    def authenticate(self, f):
+        if self.check(f):
+            del f.request.headers[self.which_auth_header(f)]
+        else:
+            f.response = self.auth_required_response(f)
+
+    # Handlers
+    def configure(self, options, updated):
+        if "auth_nonanonymous" in updated:
+            self.nonanonymous = options.auth_nonanonymous
+        if "auth_singleuser" in updated:
+            if options.auth_singleuser:
+                parts = options.auth_singleuser.split(':')
+                if len(parts) != 2:
+                    raise exceptions.OptionsError(
+                        "Invalid single-user auth specification."
+                    )
+                self.singleuser = parts
+            else:
+                self.singleuser = None
+        if "auth_htpasswd" in updated:
+            if options.auth_htpasswd:
+                try:
+                    self.htpasswd = passlib.apache.HtpasswdFile(
+                        options.auth_htpasswd
+                    )
+                except (ValueError, OSError) as v:
+                    raise exceptions.OptionsError(
+                        "Could not open htpasswd file: %s" % v
+                    )
+            else:
+                self.htpasswd = None
+        if self.enabled():
+            if options.mode == "transparent":
+                raise exceptions.OptionsError(
+                    "Proxy Authentication not supported in transparent mode."
+                )
+            elif options.mode == "socks5":
+                raise exceptions.OptionsError(
+                    "Proxy Authentication not supported in SOCKS mode. "
+                    "https://github.com/mitmproxy/mitmproxy/issues/738"
+                )
+            # TODO: check for multiple auth options
+
+    def http_connect(self, f):
+        if self.enabled() and f.mode == "regular":
+            self.authenticate(f)
+
+    def requestheaders(self, f):
+        if self.enabled():
+            # Are we already authenticated in CONNECT?
+            if not (f.mode == "regular" and f.server_conn.via):
+                self.authenticate(f)
diff --git a/mitmproxy/addons/upstream_auth.py b/mitmproxy/addons/upstream_auth.py
new file mode 100644
index 00000000..9beecfc0
--- /dev/null
+++ b/mitmproxy/addons/upstream_auth.py
@@ -0,0 +1,53 @@
+import re
+import base64
+
+from mitmproxy import exceptions
+from mitmproxy.utils import strutils
+
+
+def parse_upstream_auth(auth):
+    pattern = re.compile(".+:")
+    if pattern.search(auth) is None:
+        raise exceptions.OptionsError(
+            "Invalid upstream auth specification: %s" % auth
+        )
+    return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
+
+
+class UpstreamAuth():
+    """
+        This addon handles authentication to systems upstream from us for the
+        upstream proxy and reverse proxy mode. There are 3 cases:
+
+        - Upstream proxy CONNECT requests should have authentication added, and
+          subsequent already connected requests should not.
+        - Upstream proxy regular requests
+        - Reverse proxy regular requests (CONNECT is invalid in this mode)
+    """
+    def __init__(self):
+        self.auth = None
+        self.root_mode = None
+
+    def configure(self, options, updated):
+        # FIXME: We're doing this because our proxy core is terminally confused
+        # at the moment. Ideally, we should be able to check if we're in
+        # reverse proxy mode at the HTTP layer, so that scripts can put the
+        # proxy in reverse proxy mode for specific reuests.
+        if "mode" in updated:
+            self.root_mode = options.mode
+        if "upstream_auth" in updated:
+            if options.upstream_auth is None:
+                self.auth = None
+            else:
+                self.auth = parse_upstream_auth(options.upstream_auth)
+
+    def http_connect(self, f):
+        if self.auth and f.mode == "upstream":
+            f.request.headers["Proxy-Authorization"] = self.auth
+
+    def requestheaders(self, f):
+        if self.auth:
+            if f.mode == "upstream" and not f.server_conn.via:
+                f.request.headers["Proxy-Authorization"] = self.auth
+            elif self.root_mode == "reverse":
+                f.request.headers["Proxy-Authorization"] = self.auth
diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py
index f3a75222..c7941ad9 100644
--- a/mitmproxy/connections.py
+++ b/mitmproxy/connections.py
@@ -42,7 +42,6 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject):
         self.timestamp_start = time.time()
         self.timestamp_end = None
         self.timestamp_ssl_setup = None
-        self.protocol = None
         self.sni = None
         self.cipher_name = None
         self.tls_version = None
@@ -144,7 +143,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject):
         self.timestamp_end = None
         self.timestamp_tcp_setup = None
         self.timestamp_ssl_setup = None
-        self.protocol = None
 
     def connected(self):
         return bool(self.connection) and not self.finished
diff --git a/mitmproxy/events.py b/mitmproxy/events.py
index 56f1a45b..f9475768 100644
--- a/mitmproxy/events.py
+++ b/mitmproxy/events.py
@@ -13,6 +13,7 @@ Events = frozenset([
     "tcp_error",
     "tcp_end",
 
+    "http_connect",
     "request",
     "requestheaders",
     "response",
diff --git a/mitmproxy/http.py b/mitmproxy/http.py
index 50174764..dafd4782 100644
--- a/mitmproxy/http.py
+++ b/mitmproxy/http.py
@@ -53,7 +53,7 @@ class HTTPRequest(http.Request):
     def get_state(self):
         state = super().get_state()
         state.update(
-            is_replay=self.is_replay,
+            is_replay=self.is_replay
         )
         return state
 
@@ -143,7 +143,7 @@ class HTTPFlow(flow.Flow):
     transaction.
     """
 
-    def __init__(self, client_conn, server_conn, live=None):
+    def __init__(self, client_conn, server_conn, live=None, mode="regular"):
         super().__init__("http", client_conn, server_conn, live)
 
         self.request = None  # type: HTTPRequest
@@ -163,11 +163,14 @@ class HTTPFlow(flow.Flow):
         """:py:class:`ClientConnection` object """
         self.intercepted = False  # type: bool
         """ Is this flow currently being intercepted? """
+        self.mode = mode
+        """ What mode was the proxy layer in when receiving this request? """
 
     _stateobject_attributes = flow.Flow._stateobject_attributes.copy()
     _stateobject_attributes.update(
         request=HTTPRequest,
-        response=HTTPResponse
+        response=HTTPResponse,
+        mode=str
     )
 
     def __repr__(self):
diff --git a/mitmproxy/io_compat.py b/mitmproxy/io_compat.py
index b1b5a296..20ee8824 100644
--- a/mitmproxy/io_compat.py
+++ b/mitmproxy/io_compat.py
@@ -69,6 +69,7 @@ def convert_018_019(data):
     data["client_conn"]["sni"] = None
     data["client_conn"]["cipher_name"] = None
     data["client_conn"]["tls_version"] = None
+    data["mode"] = "regular"
     data["metadata"] = dict()
     return data
 
diff --git a/mitmproxy/master.py b/mitmproxy/master.py
index ffbfb0cb..55eb74e5 100644
--- a/mitmproxy/master.py
+++ b/mitmproxy/master.py
@@ -256,6 +256,10 @@ class Master:
         pass
 
     @controller.handler
+    def http_connect(self, f):
+        pass
+
+    @controller.handler
     def error(self, f):
         pass
 
diff --git a/mitmproxy/net/http/authentication.py b/mitmproxy/net/http/authentication.py
deleted file mode 100644
index a65279e4..00000000
--- a/mitmproxy/net/http/authentication.py
+++ /dev/null
@@ -1,176 +0,0 @@
-import argparse
-import binascii
-
-
-def parse_http_basic_auth(s):
-    words = s.split()
-    if len(words) != 2:
-        return None
-    scheme = words[0]
-    try:
-        user = binascii.a2b_base64(words[1]).decode("utf8", "replace")
-    except binascii.Error:
-        return None
-    parts = user.split(':')
-    if len(parts) != 2:
-        return None
-    return scheme, parts[0], parts[1]
-
-
-def assemble_http_basic_auth(scheme, username, password):
-    v = binascii.b2a_base64((username + ":" + password).encode("utf8")).decode("ascii")
-    return scheme + " " + v
-
-
-class NullProxyAuth:
-
-    """
-        No proxy auth at all (returns empty challange headers)
-    """
-
-    def __init__(self, password_manager):
-        self.password_manager = password_manager
-
-    def clean(self, headers_):
-        """
-            Clean up authentication headers, so they're not passed upstream.
-        """
-
-    def authenticate(self, headers_):
-        """
-            Tests that the user is allowed to use the proxy
-        """
-        return True
-
-    def auth_challenge_headers(self):
-        """
-            Returns a dictionary containing the headers require to challenge the user
-        """
-        return {}
-
-
-class BasicAuth(NullProxyAuth):
-    CHALLENGE_HEADER = None
-    AUTH_HEADER = None
-
-    def __init__(self, password_manager, realm):
-        NullProxyAuth.__init__(self, password_manager)
-        self.realm = realm
-
-    def clean(self, headers):
-        del headers[self.AUTH_HEADER]
-
-    def authenticate(self, headers):
-        auth_value = headers.get(self.AUTH_HEADER)
-        if not auth_value:
-            return False
-        parts = parse_http_basic_auth(auth_value)
-        if not parts:
-            return False
-        scheme, username, password = parts
-        if scheme.lower() != 'basic':
-            return False
-        if not self.password_manager.test(username, password):
-            return False
-        self.username = username
-        return True
-
-    def auth_challenge_headers(self):
-        return {self.CHALLENGE_HEADER: 'Basic realm="%s"' % self.realm}
-
-
-class BasicWebsiteAuth(BasicAuth):
-    CHALLENGE_HEADER = 'WWW-Authenticate'
-    AUTH_HEADER = 'Authorization'
-
-
-class BasicProxyAuth(BasicAuth):
-    CHALLENGE_HEADER = 'Proxy-Authenticate'
-    AUTH_HEADER = 'Proxy-Authorization'
-
-
-class PassMan:
-
-    def test(self, username_, password_token_):
-        return False
-
-
-class PassManNonAnon(PassMan):
-
-    """
-        Ensure the user specifies a username, accept any password.
-    """
-
-    def test(self, username, password_token_):
-        if username:
-            return True
-        return False
-
-
-class PassManHtpasswd(PassMan):
-
-    """
-        Read usernames and passwords from an htpasswd file
-    """
-
-    def __init__(self, path):
-        """
-            Raises ValueError if htpasswd file is invalid.
-        """
-        import passlib.apache
-        self.htpasswd = passlib.apache.HtpasswdFile(path)
-
-    def test(self, username, password_token):
-        return bool(self.htpasswd.check_password(username, password_token))
-
-
-class PassManSingleUser(PassMan):
-
-    def __init__(self, username, password):
-        self.username, self.password = username, password
-
-    def test(self, username, password_token):
-        return self.username == username and self.password == password_token
-
-
-class AuthAction(argparse.Action):
-
-    """
-    Helper class to allow seamless integration int argparse. Example usage:
-    parser.add_argument(
-        "--nonanonymous",
-        action=NonanonymousAuthAction, nargs=0,
-        help="Allow access to any user long as a credentials are specified."
-    )
-    """
-
-    def __call__(self, parser, namespace, values, option_string=None):
-        passman = self.getPasswordManager(values)
-        authenticator = BasicProxyAuth(passman, "mitmproxy")
-        setattr(namespace, self.dest, authenticator)
-
-    def getPasswordManager(self, s):  # pragma: no cover
-        raise NotImplementedError()
-
-
-class SingleuserAuthAction(AuthAction):
-
-    def getPasswordManager(self, s):
-        if len(s.split(':')) != 2:
-            raise argparse.ArgumentTypeError(
-                "Invalid single-user specification. Please use the format username:password"
-            )
-        username, password = s.split(':')
-        return PassManSingleUser(username, password)
-
-
-class NonanonymousAuthAction(AuthAction):
-
-    def getPasswordManager(self, s):
-        return PassManNonAnon()
-
-
-class HtpasswdAuthAction(AuthAction):
-
-    def getPasswordManager(self, s):
-        return PassManHtpasswd(s)
diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py
index 9c414b9c..513c0b5b 100644
--- a/mitmproxy/proxy/config.py
+++ b/mitmproxy/proxy/config.py
@@ -1,18 +1,14 @@
-import base64
 import collections
 import os
 import re
 from typing import Any
 
-from mitmproxy.utils import strutils
-
 from OpenSSL import SSL, crypto
 
 from mitmproxy import exceptions
 from mitmproxy import options as moptions
 from mitmproxy import certs
 from mitmproxy.net import tcp
-from mitmproxy.net.http import authentication
 from mitmproxy.net.http import url
 
 CONF_BASENAME = "mitmproxy"
@@ -56,21 +52,11 @@ def parse_server_spec(spec):
     return ServerSpec(scheme, address)
 
 
-def parse_upstream_auth(auth):
-    pattern = re.compile(".+:")
-    if pattern.search(auth) is None:
-        raise exceptions.OptionsError(
-            "Invalid upstream auth specification: %s" % auth
-        )
-    return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth))
-
-
 class ProxyConfig:
 
     def __init__(self, options: moptions.Options) -> None:
         self.options = options
 
-        self.authenticator = None
         self.check_ignore = None
         self.check_tcp = None
         self.certstore = None
@@ -134,54 +120,5 @@ class ProxyConfig:
                 )
 
         self.upstream_server = None
-        self.upstream_auth = None
         if options.upstream_server:
             self.upstream_server = parse_server_spec(options.upstream_server)
-        if options.upstream_auth:
-            self.upstream_auth = parse_upstream_auth(options.upstream_auth)
-
-        self.authenticator = authentication.NullProxyAuth(None)
-        needsauth = any(
-            [
-                options.auth_nonanonymous,
-                options.auth_singleuser,
-                options.auth_htpasswd
-            ]
-        )
-        if needsauth:
-            if options.mode == "transparent":
-                raise exceptions.OptionsError(
-                    "Proxy Authentication not supported in transparent mode."
-                )
-            elif options.mode == "socks5":
-                raise exceptions.OptionsError(
-                    "Proxy Authentication not supported in SOCKS mode. "
-                    "https://github.com/mitmproxy/mitmproxy/issues/738"
-                )
-            elif options.auth_singleuser:
-                parts = options.auth_singleuser.split(':')
-                if len(parts) != 2:
-                    raise exceptions.OptionsError(
-                        "Invalid single-user specification. "
-                        "Please use the format username:password"
-                    )
-                password_manager = authentication.PassManSingleUser(*parts)
-            elif options.auth_nonanonymous:
-                password_manager = authentication.PassManNonAnon()
-            elif options.auth_htpasswd:
-                try:
-                    password_manager = authentication.PassManHtpasswd(
-                        options.auth_htpasswd
-                    )
-                except ValueError as v:
-                    raise exceptions.OptionsError(str(v))
-            if options.mode == "reverse":
-                self.authenticator = authentication.BasicWebsiteAuth(
-                    password_manager,
-                    self.upstream_server.address
-                )
-            else:
-                self.authenticator = authentication.BasicProxyAuth(
-                    password_manager,
-                    "mitmproxy"
-                )
diff --git a/mitmproxy/proxy/modes/socks_proxy.py b/mitmproxy/proxy/modes/socks_proxy.py
index 04258037..3121b731 100644
--- a/mitmproxy/proxy/modes/socks_proxy.py
+++ b/mitmproxy/proxy/modes/socks_proxy.py
@@ -5,9 +5,6 @@ from mitmproxy.net import socks
 
 class Socks5Proxy(protocol.Layer, protocol.ServerConnectionMixin):
 
-    def __init__(self, ctx):
-        super().__init__(ctx)
-
     def __call__(self):
         try:
             # Parse Client Greeting
diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py
index 5412827f..5f9dafab 100644
--- a/mitmproxy/proxy/protocol/http.py
+++ b/mitmproxy/proxy/protocol/http.py
@@ -1,12 +1,13 @@
 import h2.exceptions
 import time
 import traceback
+import enum
+
 from mitmproxy import exceptions
 from mitmproxy import http
 from mitmproxy import flow
 from mitmproxy.proxy.protocol import base
 from mitmproxy.proxy.protocol import websockets as pwebsockets
-import mitmproxy.net.http
 from mitmproxy.net import tcp
 from mitmproxy.net import websockets
 
@@ -18,14 +19,6 @@ class _HttpTransmissionLayer(base.Layer):
     def read_request_body(self, request):
         raise NotImplementedError()
 
-    def read_request(self, f):
-        request = self.read_request_headers(f)
-        request.data.content = b"".join(
-            self.read_request_body(request)
-        )
-        request.timestamp_end = time.time()
-        return request
-
     def send_request(self, request):
         raise NotImplementedError()
 
@@ -120,12 +113,42 @@ class UpstreamConnectLayer(base.Layer):
         self.server_conn.address = address
 
 
+def is_ok(status):
+    return 200 <= status < 300
+
+
+class HTTPMode(enum.Enum):
+    regular = 1
+    transparent = 2
+    upstream = 3
+
+
+# At this point, we see only a subset of the proxy modes
+MODE_REQUEST_FORMS = {
+    HTTPMode.regular: ("authority", "absolute"),
+    HTTPMode.transparent: ("relative"),
+    HTTPMode.upstream: ("authority", "absolute"),
+}
+
+
+def validate_request_form(mode, request):
+    if request.first_line_format == "absolute" and request.scheme != "http":
+        raise exceptions.HttpException(
+            "Invalid request scheme: %s" % request.scheme
+        )
+    allowed_request_forms = MODE_REQUEST_FORMS[mode]
+    if request.first_line_format not in allowed_request_forms:
+        err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
+            " or ".join(allowed_request_forms), request.first_line_format
+        )
+        raise exceptions.HttpException(err_message)
+
+
 class HttpLayer(base.Layer):
 
     def __init__(self, ctx, mode):
         super().__init__(ctx)
         self.mode = mode
-        self.flow = None  # type: http.HTTPFlow
         self.__initial_server_conn = None
         "Contains the original destination in transparent mode, which needs to be restored"
         "if an inline script modified the target server for a single http request"
@@ -133,25 +156,108 @@ class HttpLayer(base.Layer):
         # see https://github.com/mitmproxy/mitmproxy/issues/925
         self.__initial_server_tls = None
         # Requests happening after CONNECT do not need Proxy-Authorization headers.
-        self.http_authenticated = False
+        self.connect_request = False
 
     def __call__(self):
-        if self.mode == "transparent":
+        if self.mode == HTTPMode.transparent:
             self.__initial_server_tls = self.server_tls
             self.__initial_server_conn = self.server_conn
         while True:
-            self.flow = http.HTTPFlow(self.client_conn, self.server_conn, live=self)
-            if not self._process_flow(self.flow):
+            flow = http.HTTPFlow(
+                self.client_conn,
+                self.server_conn,
+                live=self,
+                mode=self.mode.name
+            )
+            if not self._process_flow(flow):
                 return
 
-    def _process_flow(self, f):
+    def handle_regular_connect(self, f):
+        self.connect_request = True
+
         try:
-            request = self.get_request_from_client(f)
-            # Make sure that the incoming request matches our expectations
-            self.validate_request(request)
-        except exceptions.HttpReadDisconnect:
-            # don't throw an error for disconnects that happen before/between requests.
+            self.set_server((f.request.host, f.request.port))
+        except (
+            exceptions.ProtocolException, exceptions.NetlibException
+        ) as e:
+            # HTTPS tasting means that ordinary errors like resolution
+            # and connection errors can happen here.
+            self.send_error_response(502, repr(e))
+            f.error = flow.Error(str(e))
+            self.channel.ask("error", f)
             return False
+
+        if f.response:
+            resp = f.response
+        else:
+            resp = http.make_connect_response(f.request.data.http_version)
+
+        self.send_response(resp)
+
+        if is_ok(resp.status_code):
+            layer = self.ctx.next_layer(self)
+            layer()
+
+        return False
+
+    def handle_upstream_connect(self, f):
+        self.establish_server_connection(
+            f.request.host,
+            f.request.port,
+            f.request.scheme
+        )
+        self.send_request(f.request)
+        f.response = self.read_response_headers()
+        f.response.data.content = b"".join(
+            self.read_response_body(f.request, f.response)
+        )
+        self.send_response(f.response)
+        if is_ok(f.response.status_code):
+            layer = UpstreamConnectLayer(self, f.request)
+            return layer()
+        return False
+
+    def _process_flow(self, f):
+        try:
+            try:
+                request = self.read_request_headers(f)
+            except exceptions.HttpReadDisconnect:
+                # don't throw an error for disconnects that happen
+                # before/between requests.
+                return False
+
+            f.request = request
+
+            if request.first_line_format == "authority":
+                # The standards are silent on what we should do with a CONNECT
+                # request body, so although it's not common, it's allowed.
+                f.request.data.content = b"".join(
+                    self.read_request_body(f.request)
+                )
+                f.request.timestamp_end = time.time()
+                self.channel.ask("http_connect", f)
+
+                if self.mode is HTTPMode.regular:
+                    return self.handle_regular_connect(f)
+                elif self.mode is HTTPMode.upstream:
+                    return self.handle_upstream_connect(f)
+                else:
+                    msg = "Unexpected CONNECT request."
+                    self.send_error_response(400, msg)
+                    raise exceptions.ProtocolException(msg)
+
+            self.channel.ask("requestheaders", f)
+
+            if request.headers.get("expect", "").lower() == "100-continue":
+                # TODO: We may have to use send_response_headers for HTTP2
+                # here.
+                self.send_response(http.expect_continue_response)
+                request.headers.pop("expect")
+
+            request.data.content = b"".join(self.read_request_body(request))
+            request.timestamp_end = time.time()
+
+            validate_request_form(self.mode, request)
         except exceptions.HttpException as e:
             # We optimistically guess there might be an HTTP client on the
             # other end
@@ -162,36 +268,25 @@ class HttpLayer(base.Layer):
 
         self.log("request", "debug", [repr(request)])
 
-        # Handle Proxy Authentication
-        # Proxy Authentication conceptually does not work in transparent mode.
-        # We catch this misconfiguration on startup. Here, we sort out requests
-        # after a successful CONNECT request (which do not need to be validated anymore)
-        if not (self.http_authenticated or self.authenticate(request)):
-            return False
-
-        f.request = request
-
-        try:
-            # Regular Proxy Mode: Handle CONNECT
-            if self.mode == "regular" and request.first_line_format == "authority":
-                self.handle_regular_mode_connect(request)
-                return False
-        except (exceptions.ProtocolException, exceptions.NetlibException) as e:
-            # HTTPS tasting means that ordinary errors like resolution and
-            # connection errors can happen here.
-            self.send_error_response(502, repr(e))
-            f.error = flow.Error(str(e))
-            self.channel.ask("error", f)
-            return False
-
         # update host header in reverse proxy mode
         if self.config.options.mode == "reverse":
             f.request.headers["Host"] = self.config.upstream_server.address.host
 
-        # set upstream auth
-        if self.mode == "upstream" and self.config.upstream_auth is not None:
-            f.request.headers["Proxy-Authorization"] = self.config.upstream_auth
-        self.process_request_hook(f)
+        # Determine .scheme, .host and .port attributes for inline scripts. For
+        # absolute-form requests, they are directly given in the request. For
+        # authority-form requests, we only need to determine the request
+        # scheme. For relative-form requests, we need to determine host and
+        # port as well.
+        if self.mode is HTTPMode.transparent:
+            # Setting request.host also updates the host header, which we want
+            # to preserve
+            host_header = f.request.headers.get("host", None)
+            f.request.host = self.__initial_server_conn.address.host
+            f.request.port = self.__initial_server_conn.address.port
+            if host_header:
+                f.request.headers["host"] = host_header
+            f.request.scheme = "https" if self.__initial_server_tls else "http"
+        self.channel.ask("request", f)
 
         try:
             if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers):
@@ -205,7 +300,55 @@ class HttpLayer(base.Layer):
                     f.request.port,
                     f.request.scheme
                 )
-                self.get_response_from_server(f)
+
+                def get_response():
+                    self.send_request(f.request)
+                    f.response = self.read_response_headers()
+
+                try:
+                    get_response()
+                except exceptions.NetlibException as e:
+                    self.log(
+                        "server communication error: %s" % repr(e),
+                        level="debug"
+                    )
+                    # In any case, we try to reconnect at least once. This is
+                    # necessary because it might be possible that we already
+                    # initiated an upstream connection after clientconnect that
+                    # has already been expired, e.g consider the following event
+                    # log:
+                    # > clientconnect (transparent mode destination known)
+                    # > serverconnect (required for client tls handshake)
+                    # > read n% of large request
+                    # > server detects timeout, disconnects
+                    # > read (100-n)% of large request
+                    # > send large request upstream
+
+                    if isinstance(e, exceptions.Http2ProtocolException):
+                        # do not try to reconnect for HTTP2
+                        raise exceptions.ProtocolException(
+                            "First and only attempt to get response via HTTP2 failed."
+                        )
+
+                    self.disconnect()
+                    self.connect()
+                    get_response()
+
+                # call the appropriate script hook - this is an opportunity for
+                # an inline script to set f.stream = True
+                self.channel.ask("responseheaders", f)
+
+                if f.response.stream:
+                    f.response.data.content = None
+                else:
+                    f.response.data.content = b"".join(
+                        self.read_response_body(f.request, f.response)
+                    )
+                f.response.timestamp_end = time.time()
+
+                # no further manipulation of self.server_conn beyond this point
+                # we can safely set it as the final attribute value here.
+                f.server_conn = self.server_conn
             else:
                 # response was set by an inline script.
                 # we now need to emulate the responseheaders hook.
@@ -213,21 +356,50 @@ class HttpLayer(base.Layer):
 
             self.log("response", "debug", [repr(f.response)])
             self.channel.ask("response", f)
-            self.send_response_to_client(f)
+
+            if not f.response.stream:
+                # no streaming:
+                # we already received the full response from the server and can
+                # send it to the client straight away.
+                self.send_response(f.response)
+            else:
+                # streaming:
+                # First send the headers and then transfer the response incrementally
+                self.send_response_headers(f.response)
+                chunks = self.read_response_body(
+                    f.request,
+                    f.response
+                )
+                if callable(f.response.stream):
+                    chunks = f.response.stream(chunks)
+                self.send_response_body(f.response, chunks)
+                f.response.timestamp_end = time.time()
 
             if self.check_close_connection(f):
                 return False
 
             # Handle 101 Switching Protocols
             if f.response.status_code == 101:
-                self.handle_101_switching_protocols(f)
+                # Handle a successful HTTP 101 Switching Protocols Response,
+                # received after e.g. a WebSocket upgrade request.
+                # Check for WebSockets handshake
+                is_websockets = (
+                    websockets.check_handshake(f.request.headers) and
+                    websockets.check_handshake(f.response.headers)
+                )
+                if is_websockets and not self.config.options.websockets:
+                    self.log(
+                        "Client requested WebSocket connection, but the protocol is disabled.",
+                        "info"
+                    )
+
+                if is_websockets and self.config.options.websockets:
+                    layer = pwebsockets.WebSocketsLayer(self, f)
+                else:
+                    layer = self.ctx.next_layer(self)
+                layer()
                 return False  # should never be reached
 
-            # Upstream Proxy Mode: Handle CONNECT
-            if f.request.first_line_format == "authority" and f.response.status_code == 200:
-                self.handle_upstream_mode_connect(f.request.copy())
-                return False
-
         except (exceptions.ProtocolException, exceptions.NetlibException) as e:
             self.send_error_response(502, repr(e))
             if not f.response:
@@ -244,135 +416,24 @@ class HttpLayer(base.Layer):
 
         return True
 
-    def get_request_from_client(self, f):
-        request = self.read_request(f)
-        f.request = request
-        self.channel.ask("requestheaders", f)
-        if request.headers.get("expect", "").lower() == "100-continue":
-            # TODO: We may have to use send_response_headers for HTTP2 here.
-            self.send_response(http.expect_continue_response)
-            request.headers.pop("expect")
-            request.content = b"".join(self.read_request_body(request))
-            request.timestamp_end = time.time()
-        return request
-
-    def send_error_response(self, code, message, headers=None):
+    def send_error_response(self, code, message, headers=None) -> None:
         try:
             response = http.make_error_response(code, message, headers)
             self.send_response(response)
         except (exceptions.NetlibException, h2.exceptions.H2Error, exceptions.Http2ProtocolException):
             self.log(traceback.format_exc(), "debug")
 
-    def change_upstream_proxy_server(self, address):
+    def change_upstream_proxy_server(self, address) -> None:
         # Make set_upstream_proxy_server always available,
         # even if there's no UpstreamConnectLayer
         if address != self.server_conn.address:
-            return self.set_server(address)
+            self.set_server(address)
 
-    def handle_regular_mode_connect(self, request):
-        self.http_authenticated = True
-        self.set_server((request.host, request.port))
-        self.send_response(http.make_connect_response(request.data.http_version))
-        layer = self.ctx.next_layer(self)
-        layer()
-
-    def handle_upstream_mode_connect(self, connect_request):
-        layer = UpstreamConnectLayer(self, connect_request)
-        layer()
-
-    def send_response_to_client(self, f):
-        if not f.response.stream:
-            # no streaming:
-            # we already received the full response from the server and can
-            # send it to the client straight away.
-            self.send_response(f.response)
-        else:
-            # streaming:
-            # First send the headers and then transfer the response incrementally
-            self.send_response_headers(f.response)
-            chunks = self.read_response_body(
-                f.request,
-                f.response
-            )
-            if callable(f.response.stream):
-                chunks = f.response.stream(chunks)
-            self.send_response_body(f.response, chunks)
-            f.response.timestamp_end = time.time()
-
-    def get_response_from_server(self, f):
-        def get_response():
-            self.send_request(f.request)
-            f.response = self.read_response_headers()
-
-        try:
-            get_response()
-        except exceptions.NetlibException as e:
-            self.log(
-                "server communication error: %s" % repr(e),
-                level="debug"
-            )
-            # In any case, we try to reconnect at least once. This is
-            # necessary because it might be possible that we already
-            # initiated an upstream connection after clientconnect that
-            # has already been expired, e.g consider the following event
-            # log:
-            # > clientconnect (transparent mode destination known)
-            # > serverconnect (required for client tls handshake)
-            # > read n% of large request
-            # > server detects timeout, disconnects
-            # > read (100-n)% of large request
-            # > send large request upstream
-
-            if isinstance(e, exceptions.Http2ProtocolException):
-                # do not try to reconnect for HTTP2
-                raise exceptions.ProtocolException("First and only attempt to get response via HTTP2 failed.")
-
-            self.disconnect()
-            self.connect()
-            get_response()
-
-        # call the appropriate script hook - this is an opportunity for an
-        # inline script to set f.stream = True
-        self.channel.ask("responseheaders", f)
-
-        if f.response.stream:
-            f.response.data.content = None
-        else:
-            f.response.data.content = b"".join(self.read_response_body(
-                f.request,
-                f.response
-            ))
-        f.response.timestamp_end = time.time()
-
-        # no further manipulation of self.server_conn beyond this point
-        # we can safely set it as the final attribute value here.
-        f.server_conn = self.server_conn
-
-    def process_request_hook(self, f):
-        # Determine .scheme, .host and .port attributes for inline scripts.
-        # For absolute-form requests, they are directly given in the request.
-        # For authority-form requests, we only need to determine the request scheme.
-        # For relative-form requests, we need to determine host and port as
-        # well.
-        if self.mode == "regular":
-            pass  # only absolute-form at this point, nothing to do here.
-        elif self.mode == "upstream":
-            pass
-        else:
-            # Setting request.host also updates the host header, which we want to preserve
-            host_header = f.request.headers.get("host", None)
-            f.request.host = self.__initial_server_conn.address.host
-            f.request.port = self.__initial_server_conn.address.port
-            if host_header:
-                f.request.headers["host"] = host_header
-            f.request.scheme = "https" if self.__initial_server_tls else "http"
-        self.channel.ask("request", f)
-
-    def establish_server_connection(self, host, port, scheme):
+    def establish_server_connection(self, host: str, port: int, scheme: str):
         address = tcp.Address((host, port))
         tls = (scheme == "https")
 
-        if self.mode == "regular" or self.mode == "transparent":
+        if self.mode is HTTPMode.regular or self.mode is HTTPMode.transparent:
             # If there's an existing connection that doesn't match our expectations, kill it.
             if address != self.server_conn.address or tls != self.server_tls:
                 self.set_server(address)
@@ -385,80 +446,3 @@ class HttpLayer(base.Layer):
                 self.connect()
             if tls:
                 raise exceptions.HttpProtocolException("Cannot change scheme in upstream proxy mode.")
-            """
-            # This is a very ugly (untested) workaround to solve a very ugly problem.
-            if self.server_conn and self.server_conn.tls_established and not ssl:
-                self.disconnect()
-                self.connect()
-            elif ssl and not hasattr(self, "connected_to") or self.connected_to != address:
-                if self.server_conn.tls_established:
-                    self.disconnect()
-                    self.connect()
-
-                self.send_request(make_connect_request(address))
-                tls_layer = TlsLayer(self, False, True)
-                tls_layer._establish_tls_with_server()
-            """
-
-    def validate_request(self, request):
-        if request.first_line_format == "absolute" and request.scheme != "http":
-            raise exceptions.HttpException("Invalid request scheme: %s" % request.scheme)
-
-        expected_request_forms = {
-            "regular": ("authority", "absolute",),
-            "upstream": ("authority", "absolute"),
-            "transparent": ("relative",)
-        }
-
-        allowed_request_forms = expected_request_forms[self.mode]
-        if request.first_line_format not in allowed_request_forms:
-            err_message = "Invalid HTTP request form (expected: %s, got: %s)" % (
-                " or ".join(allowed_request_forms), request.first_line_format
-            )
-            raise exceptions.HttpException(err_message)
-
-        if self.mode == "regular" and request.first_line_format == "absolute":
-            request.first_line_format = "relative"
-
-    def authenticate(self, request):
-        if self.config.authenticator:
-            if self.config.authenticator.authenticate(request.headers):
-                self.config.authenticator.clean(request.headers)
-            else:
-                if self.mode == "transparent":
-                    self.send_response(http.make_error_response(
-                        401,
-                        "Authentication Required",
-                        mitmproxy.net.http.Headers(**self.config.authenticator.auth_challenge_headers())
-                    ))
-                else:
-                    self.send_response(http.make_error_response(
-                        407,
-                        "Proxy Authentication Required",
-                        mitmproxy.net.http.Headers(**self.config.authenticator.auth_challenge_headers())
-                    ))
-                return False
-        return True
-
-    def handle_101_switching_protocols(self, f):
-        """
-        Handle a successful HTTP 101 Switching Protocols Response, received after e.g. a WebSocket upgrade request.
-        """
-        # Check for WebSockets handshake
-        is_websockets = (
-            f and
-            websockets.check_handshake(f.request.headers) and
-            websockets.check_handshake(f.response.headers)
-        )
-        if is_websockets and not self.config.options.websockets:
-            self.log(
-                "Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.",
-                "info"
-            )
-
-        if is_websockets and self.config.options.websockets:
-            layer = pwebsockets.WebSocketsLayer(self, f)
-        else:
-            layer = self.ctx.next_layer(self)
-
-        layer()
diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py
index eacf7e0b..f38f2a8c 100644
--- a/mitmproxy/proxy/root_context.py
+++ b/mitmproxy/proxy/root_context.py
@@ -2,6 +2,7 @@ from mitmproxy import log
 from mitmproxy import exceptions
 from mitmproxy.proxy import protocol
 from mitmproxy.proxy import modes
+from mitmproxy.proxy.protocol import http
 
 
 class RootContext:
@@ -63,16 +64,21 @@ class RootContext:
         # An inline script may upgrade from http to https,
         # in which case we need some form of TLS layer.
         if isinstance(top_layer, modes.ReverseProxy):
-            return protocol.TlsLayer(top_layer, client_tls, top_layer.server_tls, top_layer.server_conn.address.host)
+            return protocol.TlsLayer(
+                top_layer,
+                client_tls,
+                top_layer.server_tls,
+                top_layer.server_conn.address.host
+            )
         if isinstance(top_layer, protocol.ServerConnectionMixin) or isinstance(top_layer, protocol.UpstreamConnectLayer):
             return protocol.TlsLayer(top_layer, client_tls, client_tls)
 
         # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed.
         if isinstance(top_layer, protocol.TlsLayer):
             if isinstance(top_layer.ctx, modes.HttpProxy):
-                return protocol.Http1Layer(top_layer, "regular")
+                return protocol.Http1Layer(top_layer, http.HTTPMode.regular)
             if isinstance(top_layer.ctx, modes.HttpUpstreamProxy):
-                return protocol.Http1Layer(top_layer, "upstream")
+                return protocol.Http1Layer(top_layer, http.HTTPMode.upstream)
 
         # 4. Check for other TLS cases (e.g. after CONNECT).
         if client_tls:
@@ -86,21 +92,12 @@ class RootContext:
         if isinstance(top_layer, protocol.TlsLayer):
             alpn = top_layer.client_conn.get_alpn_proto_negotiated()
             if alpn == b'h2':
-                return protocol.Http2Layer(top_layer, 'transparent')
+                return protocol.Http2Layer(top_layer, http.HTTPMode.transparent)
             if alpn == b'http/1.1':
-                return protocol.Http1Layer(top_layer, 'transparent')
-
-        # 6. Check for raw tcp mode
-        is_ascii = (
-            len(d) == 3 and
-            # expect A-Za-z
-            all(65 <= x <= 90 or 97 <= x <= 122 for x in d)
-        )
-        if self.config.options.rawtcp and not is_ascii:
-            return protocol.RawTCPLayer(top_layer)
+                return protocol.Http1Layer(top_layer, http.HTTPMode.transparent)
 
-        # 7. Assume HTTP1 by default
-        return protocol.Http1Layer(top_layer, 'transparent')
+        # 6. Assume HTTP1 by default
+        return protocol.Http1Layer(top_layer, http.HTTPMode.transparent)
 
     def log(self, msg, level, subs=()):
         """
diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py
index debe6db9..8b579952 100644
--- a/mitmproxy/tools/cmdline.py
+++ b/mitmproxy/tools/cmdline.py
@@ -463,8 +463,8 @@ def proxy_options(parser):
         action="store", dest="upstream_auth", default=None,
         type=str,
         help="""
-            Proxy Authentication:
-            username:password
+            Add HTTP Basic authentcation to upstream proxy and reverse proxy
+            requests. Format: username:password
         """
     )
     rawtcp = group.add_mutually_exclusive_group()
diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py
index e35815ad..2cb5953f 100644
--- a/mitmproxy/tools/web/master.py
+++ b/mitmproxy/tools/web/master.py
@@ -12,7 +12,6 @@ from mitmproxy.addons import intercept
 from mitmproxy import options
 from mitmproxy import master
 from mitmproxy.tools.web import app
-from mitmproxy.net.http import authentication
 
 
 class Stop(Exception):
@@ -52,7 +51,7 @@ class Options(options.Options):
             wdebug: bool = False,
             wport: int = 8081,
             wiface: str = "127.0.0.1",
-            wauthenticator: Optional[authentication.PassMan] = None,
+            # wauthenticator: Optional[authentication.PassMan] = None,
             wsingleuser: Optional[str] = None,
             whtpasswd: Optional[str] = None,
             **kwargs
@@ -60,29 +59,30 @@ class Options(options.Options):
         self.wdebug = wdebug
         self.wport = wport
         self.wiface = wiface
-        self.wauthenticator = wauthenticator
-        self.wsingleuser = wsingleuser
-        self.whtpasswd = whtpasswd
+        # self.wauthenticator = wauthenticator
+        # self.wsingleuser = wsingleuser
+        # self.whtpasswd = whtpasswd
         self.intercept = intercept
         super().__init__(**kwargs)
 
     # TODO: This doesn't belong here.
     def process_web_options(self, parser):
-        if self.wsingleuser or self.whtpasswd:
-            if self.wsingleuser:
-                if len(self.wsingleuser.split(':')) != 2:
-                    return parser.error(
-                        "Invalid single-user specification. Please use the format username:password"
-                    )
-                username, password = self.wsingleuser.split(':')
-                self.wauthenticator = authentication.PassManSingleUser(username, password)
-            elif self.whtpasswd:
-                try:
-                    self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd)
-                except ValueError as v:
-                    return parser.error(v.message)
-        else:
-            self.wauthenticator = None
+        # if self.wsingleuser or self.whtpasswd:
+        #     if self.wsingleuser:
+        #         if len(self.wsingleuser.split(':')) != 2:
+        #             return parser.error(
+        #                 "Invalid single-user specification. Please use the format username:password"
+        #             )
+        #         username, password = self.wsingleuser.split(':')
+        #         # self.wauthenticator = authentication.PassManSingleUser(username, password)
+        #     elif self.whtpasswd:
+        #         try:
+        #             self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd)
+        #         except ValueError as v:
+        #             return parser.error(v.message)
+        # else:
+        #     self.wauthenticator = None
+        pass
 
 
 class WebMaster(master.Master):
@@ -98,7 +98,7 @@ class WebMaster(master.Master):
         self.addons.add(*addons.default_addons())
         self.addons.add(self.view, intercept.Intercept())
         self.app = app.Application(
-            self, self.options.wdebug, self.options.wauthenticator
+            self, self.options.wdebug, False
         )
         # This line is just for type hinting
         self.options = self.options  # type: Options
diff --git a/test/mitmproxy/addons/test_proxyauth.py b/test/mitmproxy/addons/test_proxyauth.py
new file mode 100644
index 00000000..494a992f
--- /dev/null
+++ b/test/mitmproxy/addons/test_proxyauth.py
@@ -0,0 +1,174 @@
+import binascii
+
+from mitmproxy import exceptions
+from mitmproxy.test import taddons
+from mitmproxy.test import tflow
+from mitmproxy.test import tutils
+from mitmproxy.addons import proxyauth
+
+
+def test_parse_http_basic_auth():
+    assert proxyauth.parse_http_basic_auth(
+        proxyauth.mkauth("test", "test")
+    ) == ("basic", "test", "test")
+    assert not proxyauth.parse_http_basic_auth("")
+    assert not proxyauth.parse_http_basic_auth("foo bar")
+    v = "basic " + binascii.b2a_base64(b"foo").decode("ascii")
+    assert not proxyauth.parse_http_basic_auth(v)
+
+
+def test_configure():
+    up = proxyauth.ProxyAuth()
+    with taddons.context() as ctx:
+        tutils.raises(
+            exceptions.OptionsError,
+            ctx.configure, up, auth_singleuser="foo"
+        )
+
+        ctx.configure(up, auth_singleuser="foo:bar")
+        assert up.singleuser == ["foo", "bar"]
+
+        ctx.configure(up, auth_singleuser=None)
+        assert up.singleuser is None
+
+        ctx.configure(up, auth_nonanonymous=True)
+        assert up.nonanonymous
+        ctx.configure(up, auth_nonanonymous=False)
+        assert not up.nonanonymous
+
+        tutils.raises(
+            exceptions.OptionsError,
+            ctx.configure,
+            up,
+            auth_htpasswd = tutils.test_data.path(
+                "mitmproxy/net/data/server.crt"
+            )
+        )
+        tutils.raises(
+            exceptions.OptionsError,
+            ctx.configure,
+            up,
+            auth_htpasswd = "nonexistent"
+        )
+
+        ctx.configure(
+            up,
+            auth_htpasswd = tutils.test_data.path(
+                "mitmproxy/net/data/htpasswd"
+            )
+        )
+        assert up.htpasswd
+        assert up.htpasswd.check_password("test", "test")
+        assert not up.htpasswd.check_password("test", "foo")
+        ctx.configure(up, auth_htpasswd = None)
+        assert not up.htpasswd
+
+        tutils.raises(
+            exceptions.OptionsError,
+            ctx.configure,
+            up,
+            auth_nonanonymous = True,
+            mode = "transparent"
+        )
+        tutils.raises(
+            exceptions.OptionsError,
+            ctx.configure,
+            up,
+            auth_nonanonymous = True,
+            mode = "socks5"
+        )
+
+
+def test_check():
+    up = proxyauth.ProxyAuth()
+    with taddons.context() as ctx:
+        ctx.configure(up, auth_nonanonymous=True)
+        f = tflow.tflow()
+        assert not up.check(f)
+        f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
+            "test", "test"
+        )
+        assert up.check(f)
+
+        f.request.headers["Proxy-Authorization"] = "invalid"
+        assert not up.check(f)
+
+        f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
+            "test", "test", scheme = "unknown"
+        )
+        assert not up.check(f)
+
+        ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:test")
+        f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
+            "test", "test"
+        )
+        assert up.check(f)
+        ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:foo")
+        assert not up.check(f)
+
+        ctx.configure(
+            up,
+            auth_singleuser = None,
+            auth_htpasswd = tutils.test_data.path(
+                "mitmproxy/net/data/htpasswd"
+            )
+        )
+        f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
+            "test", "test"
+        )
+        assert up.check(f)
+        f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
+            "test", "foo"
+        )
+        assert not up.check(f)
+
+
+def test_authenticate():
+    up = proxyauth.ProxyAuth()
+    with taddons.context() as ctx:
+        ctx.configure(up, auth_nonanonymous=True)
+
+        f = tflow.tflow()
+        assert not f.response
+        up.authenticate(f)
+        assert f.response.status_code == 407
+
+        f = tflow.tflow()
+        f.request.headers["Proxy-Authorization"] = proxyauth.mkauth(
+            "test", "test"
+        )
+        up.authenticate(f)
+        assert not f.response
+        assert not f.request.headers.get("Proxy-Authorization")
+
+        f = tflow.tflow()
+        f.mode = "transparent"
+        assert not f.response
+        up.authenticate(f)
+        assert f.response.status_code == 401
+
+        f = tflow.tflow()
+        f.mode = "transparent"
+        f.request.headers["Authorization"] = proxyauth.mkauth(
+            "test", "test"
+        )
+        up.authenticate(f)
+        assert not f.response
+        assert not f.request.headers.get("Authorization")
+
+
+def test_handlers():
+    up = proxyauth.ProxyAuth()
+    with taddons.context() as ctx:
+        ctx.configure(up, auth_nonanonymous=True)
+
+        f = tflow.tflow()
+        assert not f.response
+        up.requestheaders(f)
+        assert f.response.status_code == 407
+
+        f = tflow.tflow()
+        f.request.method = "CONNECT"
+        assert not f.response
+        up.http_connect(f)
+        assert f.response.status_code == 407
diff --git a/test/mitmproxy/addons/test_upstream_auth.py b/test/mitmproxy/addons/test_upstream_auth.py
new file mode 100644
index 00000000..985b13a7
--- /dev/null
+++ b/test/mitmproxy/addons/test_upstream_auth.py
@@ -0,0 +1,65 @@
+import base64
+
+from mitmproxy import exceptions
+from mitmproxy.test import taddons
+from mitmproxy.test import tflow
+from mitmproxy.test import tutils
+from mitmproxy.addons import upstream_auth
+
+
+def test_configure():
+    up = upstream_auth.UpstreamAuth()
+    with taddons.context() as tctx:
+        tctx.configure(up, upstream_auth="test:test")
+        assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:test")
+
+        tctx.configure(up, upstream_auth="test:")
+        assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:")
+
+        tctx.configure(up, upstream_auth=None)
+        assert not up.auth
+
+        tutils.raises(
+            exceptions.OptionsError,
+            tctx.configure,
+            up,
+            upstream_auth=""
+        )
+        tutils.raises(
+            exceptions.OptionsError,
+            tctx.configure,
+            up,
+            upstream_auth=":"
+        )
+        tutils.raises(
+            exceptions.OptionsError,
+            tctx.configure,
+            up,
+            upstream_auth=":test"
+        )
+
+
+def test_simple():
+    up = upstream_auth.UpstreamAuth()
+    with taddons.context() as tctx:
+        tctx.configure(up, upstream_auth="foo:bar")
+
+        f = tflow.tflow()
+        f.mode = "upstream"
+        up.requestheaders(f)
+        assert "proxy-authorization" in f.request.headers
+
+        f = tflow.tflow()
+        up.requestheaders(f)
+        assert "proxy-authorization" not in f.request.headers
+
+        tctx.configure(up, mode="reverse")
+        f = tflow.tflow()
+        f.mode = "transparent"
+        up.requestheaders(f)
+        assert "proxy-authorization" in f.request.headers
+
+        f = tflow.tflow()
+        f.mode = "upstream"
+        up.http_connect(f)
+        assert "proxy-authorization" in f.request.headers
diff --git a/test/mitmproxy/net/http/test_authentication.py b/test/mitmproxy/net/http/test_authentication.py
deleted file mode 100644
index 01eae52d..00000000
--- a/test/mitmproxy/net/http/test_authentication.py
+++ /dev/null
@@ -1,122 +0,0 @@
-import binascii
-
-from mitmproxy.test import tutils
-from mitmproxy.net.http import authentication, Headers
-
-
-def test_parse_http_basic_auth():
-    vals = ("basic", "foo", "bar")
-    assert authentication.parse_http_basic_auth(
-        authentication.assemble_http_basic_auth(*vals)
-    ) == vals
-    assert not authentication.parse_http_basic_auth("")
-    assert not authentication.parse_http_basic_auth("foo bar")
-    v = "basic " + binascii.b2a_base64(b"foo").decode("ascii")
-    assert not authentication.parse_http_basic_auth(v)
-
-
-class TestPassManNonAnon:
-
-    def test_simple(self):
-        p = authentication.PassManNonAnon()
-        assert not p.test("", "")
-        assert p.test("user", "")
-
-
-class TestPassManHtpasswd:
-
-    def test_file_errors(self):
-        tutils.raises(
-            "malformed htpasswd file",
-            authentication.PassManHtpasswd,
-            tutils.test_data.path("mitmproxy/net/data/server.crt"))
-
-    def test_simple(self):
-        pm = authentication.PassManHtpasswd(tutils.test_data.path("mitmproxy/net/data/htpasswd"))
-
-        vals = ("basic", "test", "test")
-        authentication.assemble_http_basic_auth(*vals)
-        assert pm.test("test", "test")
-        assert not pm.test("test", "foo")
-        assert not pm.test("foo", "test")
-        assert not pm.test("test", "")
-        assert not pm.test("", "")
-
-
-class TestPassManSingleUser:
-
-    def test_simple(self):
-        pm = authentication.PassManSingleUser("test", "test")
-        assert pm.test("test", "test")
-        assert not pm.test("test", "foo")
-        assert not pm.test("foo", "test")
-
-
-class TestNullProxyAuth:
-
-    def test_simple(self):
-        na = authentication.NullProxyAuth(authentication.PassManNonAnon())
-        assert not na.auth_challenge_headers()
-        assert na.authenticate("foo")
-        na.clean({})
-
-
-class TestBasicProxyAuth:
-
-    def test_simple(self):
-        ba = authentication.BasicProxyAuth(authentication.PassManNonAnon(), "test")
-        headers = Headers()
-        assert ba.auth_challenge_headers()
-        assert not ba.authenticate(headers)
-
-    def test_authenticate_clean(self):
-        ba = authentication.BasicProxyAuth(authentication.PassManNonAnon(), "test")
-
-        headers = Headers()
-        vals = ("basic", "foo", "bar")
-        headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(*vals)
-        assert ba.authenticate(headers)
-
-        ba.clean(headers)
-        assert ba.AUTH_HEADER not in headers
-
-        headers[ba.AUTH_HEADER] = ""
-        assert not ba.authenticate(headers)
-
-        headers[ba.AUTH_HEADER] = "foo"
-        assert not ba.authenticate(headers)
-
-        vals = ("foo", "foo", "bar")
-        headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(*vals)
-        assert not ba.authenticate(headers)
-
-        ba = authentication.BasicProxyAuth(authentication.PassMan(), "test")
-        vals = ("basic", "foo", "bar")
-        headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(*vals)
-        assert not ba.authenticate(headers)
-
-
-class Bunch:
-    pass
-
-
-class TestAuthAction:
-
-    def test_nonanonymous(self):
-        m = Bunch()
-        aa = authentication.NonanonymousAuthAction(None, "authenticator")
-        aa(None, m, None, None)
-        assert m.authenticator
-
-    def test_singleuser(self):
-        m = Bunch()
-        aa = authentication.SingleuserAuthAction(None, "authenticator")
-        aa(None, m, "foo:bar", None)
-        assert m.authenticator
-        tutils.raises("invalid", aa, None, m, "foo", None)
-
-    def test_httppasswd(self):
-        m = Bunch()
-        aa = authentication.HtpasswdAuthAction(None, "authenticator")
-        aa(None, m, tutils.test_data.path("mitmproxy/net/data/htpasswd"), None)
-        assert m.authenticator
diff --git a/test/mitmproxy/protocol/test_http1.py b/test/mitmproxy/protocol/test_http1.py
index 5026bef1..cd937ada 100644
--- a/test/mitmproxy/protocol/test_http1.py
+++ b/test/mitmproxy/protocol/test_http1.py
@@ -20,7 +20,7 @@ class TestInvalidRequests(tservers.HTTPProxyTest):
         with p.connect():
             r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port))
         assert r.status_code == 400
-        assert b"Invalid HTTP request form" in r.content
+        assert b"Unexpected CONNECT" in r.content
 
     def test_relative_request(self):
         p = self.pathoc_raw()
diff --git a/test/mitmproxy/test_eventsequence.py b/test/mitmproxy/test_eventsequence.py
new file mode 100644
index 00000000..262df4b0
--- /dev/null
+++ b/test/mitmproxy/test_eventsequence.py
@@ -0,0 +1,81 @@
+from mitmproxy import events
+import contextlib
+from . import tservers
+
+
+class Eventer:
+    def __init__(self, **handlers):
+        self.failure = None
+        self.called = []
+        self.handlers = handlers
+        for i in events.Events - {"tick"}:
+            def mkprox():
+                evt = i
+
+                def prox(*args, **kwargs):
+                    self.called.append(evt)
+                    if evt in self.handlers:
+                        try:
+                            handlers[evt](*args, **kwargs)
+                        except AssertionError as e:
+                            self.failure = e
+                return prox
+            setattr(self, i, mkprox())
+
+    def fail(self):
+        pass
+
+
+class SequenceTester:
+    @contextlib.contextmanager
+    def addon(self, addon):
+        self.master.addons.add(addon)
+        yield
+        self.master.addons.remove(addon)
+        if addon.failure:
+            raise addon.failure
+
+
+class TestBasic(tservers.HTTPProxyTest, SequenceTester):
+    ssl = True
+
+    def test_requestheaders(self):
+
+        def hdrs(f):
+            assert f.request.headers
+            assert not f.request.content
+
+        def req(f):
+            assert f.request.headers
+            assert f.request.content
+
+        with self.addon(Eventer(requestheaders=hdrs, request=req)):
+            p = self.pathoc()
+            with p.connect():
+                assert p.request("get:'/p/200':b@10").status_code == 200
+
+    def test_100_continue_fail(self):
+        e = Eventer()
+        with self.addon(e):
+            p = self.pathoc()
+            with p.connect():
+                p.request(
+                    """
+                        get:'/p/200'
+                        h'expect'='100-continue'
+                        h'content-length'='1000'
+                        da
+                    """
+                )
+        assert "requestheaders" in e.called
+        assert "responseheaders" not in e.called
+
+    def test_connect(self):
+        e = Eventer()
+        with self.addon(e):
+            p = self.pathoc()
+            with p.connect():
+                p.request("get:'/p/200:b@1'")
+            assert "http_connect" in e.called
+            assert e.called.count("requestheaders") == 1
+            assert e.called.count("request") == 1
diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py
index 7cadb6c2..aa3b8979 100644
--- a/test/mitmproxy/test_proxy.py
+++ b/test/mitmproxy/test_proxy.py
@@ -107,23 +107,12 @@ class TestProcessProxyOptions:
         self.assert_noerr("-T")
 
         self.assert_noerr("-U", "http://localhost")
-        self.assert_err("expected one argument", "-U")
         self.assert_err("Invalid server specification", "-U", "upstream")
 
         self.assert_noerr("--upstream-auth", "test:test")
         self.assert_err("expected one argument", "--upstream-auth")
-        self.assert_err(
-            "Invalid upstream auth specification", "--upstream-auth", "test"
-        )
         self.assert_err("mutually exclusive", "-R", "http://localhost", "-T")
 
-    def test_socks_auth(self):
-        self.assert_err(
-            "Proxy Authentication not supported in SOCKS mode.",
-            "--socks",
-            "--nonanonymous"
-        )
-
     def test_client_certs(self):
         with tutils.tmpdir() as cadir:
             self.assert_noerr("--client-certs", cadir)
@@ -141,26 +130,6 @@ class TestProcessProxyOptions:
             tutils.test_data.path("mitmproxy/data/testkey.pem"))
         self.assert_err("does not exist", "--cert", "nonexistent")
 
-    def test_auth(self):
-        p = self.assert_noerr("--nonanonymous")
-        assert p.authenticator
-
-        p = self.assert_noerr(
-            "--htpasswd",
-            tutils.test_data.path("mitmproxy/data/htpasswd"))
-        assert p.authenticator
-        self.assert_err(
-            "malformed htpasswd file",
-            "--htpasswd",
-            tutils.test_data.path("mitmproxy/data/htpasswd.invalid"))
-
-        p = self.assert_noerr("--singleuser", "test:test")
-        assert p.authenticator
-        self.assert_err(
-            "invalid single-user specification",
-            "--singleuser",
-            "test")
-
     def test_insecure(self):
         p = self.assert_noerr("--insecure")
         assert p.openssl_verification_mode_server == SSL.VERIFY_NONE
diff --git a/test/mitmproxy/test_proxy_config.py b/test/mitmproxy/test_proxy_config.py
index e012cb5e..e2c39846 100644
--- a/test/mitmproxy/test_proxy_config.py
+++ b/test/mitmproxy/test_proxy_config.py
@@ -1,5 +1,4 @@
 from mitmproxy.test import tutils
-import base64
 from mitmproxy.proxy import config
 
 
@@ -26,23 +25,3 @@ def test_parse_server_spec():
         config.parse_server_spec,
         "http://"
     )
-
-
-def test_parse_upstream_auth():
-    tutils.raises(
-        "Invalid upstream auth specification",
-        config.parse_upstream_auth,
-        ""
-    )
-    tutils.raises(
-        "Invalid upstream auth specification",
-        config.parse_upstream_auth,
-        ":"
-    )
-    tutils.raises(
-        "Invalid upstream auth specification",
-        config.parse_upstream_auth,
-        ":test"
-    )
-    assert config.parse_upstream_auth("test:test") == b"Basic" + b" " + base64.b64encode(b"test:test")
-    assert config.parse_upstream_auth("test:") == b"Basic" + b" " + base64.b64encode(b"test:")
diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py
index 9fa6ed06..9429ab0f 100644
--- a/test/mitmproxy/test_server.py
+++ b/test/mitmproxy/test_server.py
@@ -6,6 +6,7 @@ from mitmproxy.test import tutils
 from mitmproxy import controller
 from mitmproxy import options
 from mitmproxy.addons import script
+from mitmproxy.addons import proxyauth
 from mitmproxy import http
 from mitmproxy.proxy.config import HostMatcher, parse_server_spec
 import mitmproxy.net.http
@@ -13,7 +14,6 @@ from mitmproxy.net import tcp
 from mitmproxy.net import socks
 from mitmproxy import certs
 from mitmproxy import exceptions
-from mitmproxy.net.http import authentication
 from mitmproxy.net.http import http1
 from mitmproxy.net.tcp import Address
 from pathod import pathoc
@@ -50,10 +50,7 @@ class CommonMixin:
 
     def test_replay(self):
         assert self.pathod("304").status_code == 304
-        if isinstance(self, tservers.HTTPUpstreamProxyTest) and self.ssl:
-            assert len(self.master.state.flows) == 2
-        else:
-            assert len(self.master.state.flows) == 1
+        assert len(self.master.state.flows) == 1
         l = self.master.state.flows[-1]
         assert l.response.status_code == 304
         l.request.path = "/p/305"
@@ -288,6 +285,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin):
 
 class TestHTTPAuth(tservers.HTTPProxyTest):
     def test_auth(self):
+        self.master.addons.add(proxyauth.ProxyAuth())
         self.master.options.auth_singleuser = "test:test"
         assert self.pathod("202").status_code == 407
         p = self.pathoc()
@@ -298,14 +296,15 @@ class TestHTTPAuth(tservers.HTTPProxyTest):
                 h'%s'='%s'
             """ % (
                 self.server.port,
-                mitmproxy.net.http.authentication.BasicProxyAuth.AUTH_HEADER,
-                authentication.assemble_http_basic_auth("basic", "test", "test")
+                "Proxy-Authorization",
+                proxyauth.mkauth("test", "test")
             ))
         assert ret.status_code == 202
 
 
 class TestHTTPReverseAuth(tservers.ReverseProxyTest):
     def test_auth(self):
+        self.master.addons.add(proxyauth.ProxyAuth())
         self.master.options.auth_singleuser = "test:test"
         assert self.pathod("202").status_code == 401
         p = self.pathoc()
@@ -315,8 +314,8 @@ class TestHTTPReverseAuth(tservers.ReverseProxyTest):
                 '/p/202'
                 h'%s'='%s'
             """ % (
-                mitmproxy.net.http.authentication.BasicWebsiteAuth.AUTH_HEADER,
-                authentication.assemble_http_basic_auth("basic", "test", "test")
+                "Authorization",
+                proxyauth.mkauth("test", "test")
             ))
         assert ret.status_code == 202
 
@@ -672,6 +671,13 @@ class TestProxySSL(tservers.HTTPProxyTest):
         first_flow = self.master.state.flows[0]
         assert first_flow.server_conn.timestamp_ssl_setup
 
+    def test_via(self):
+        # tests that the ssl timestamp is present when ssl is used
+        f = self.pathod("200:b@10")
+        assert f.status_code == 200
+        first_flow = self.master.state.flows[0]
+        assert not first_flow.server_conn.via
+
 
 class MasterRedirectRequest(tservers.TestMaster):
     redirect_port = None  # Set by TestRedirectRequest
@@ -952,12 +958,15 @@ class TestUpstreamProxySSL(
         assert req.status_code == 418
 
         # CONNECT from pathoc to chain[0],
-        assert self.proxy.tmaster.state.flow_count() == 2
+        assert self.proxy.tmaster.state.flow_count() == 1
+        assert self.proxy.tmaster.state.flows[0].server_conn.via
         # request from pathoc to chain[0]
         # CONNECT from proxy to chain[1],
-        assert self.chain[0].tmaster.state.flow_count() == 2
+        assert self.chain[0].tmaster.state.flow_count() == 1
+        assert self.chain[0].tmaster.state.flows[0].server_conn.via
         # request from proxy to chain[1]
         # request from chain[0] (regular proxy doesn't store CONNECTs)
+        assert not self.chain[1].tmaster.state.flows[0].server_conn.via
         assert self.chain[1].tmaster.state.flow_count() == 1
 
 
@@ -978,21 +987,12 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest):
     def test_reconnect(self):
         """
         Tests proper functionality of ConnectionHandler.server_reconnect mock.
-        If we have a disconnect on a secure connection that's transparently proxified to
-        an upstream http proxy, we need to send the CONNECT request again.
+        If we have a disconnect on a secure connection that's transparently
+        proxified to an upstream http proxy, we need to send the CONNECT
+        request again.
         """
-        self.chain[1].tmaster.addons.add(
-            RequestKiller([2])
-        )
-        self.chain[0].tmaster.addons.add(
-            RequestKiller(
-                [
-                    1,  # CONNECT
-                    3,  # reCONNECT
-                    4   # request
-                ]
-            )
-        )
+        self.chain[0].tmaster.addons.add(RequestKiller([1, 2]))
+        self.chain[1].tmaster.addons.add(RequestKiller([1]))
 
         p = self.pathoc()
         with p.connect():
@@ -1000,44 +1000,27 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest):
             assert req.content == b"content"
             assert req.status_code == 418
 
-            assert self.proxy.tmaster.state.flow_count() == 2  # CONNECT and request
-            # CONNECT, failing request,
-            assert self.chain[0].tmaster.state.flow_count() == 4
-            # reCONNECT, request
-            # failing request, request
-            assert self.chain[1].tmaster.state.flow_count() == 2
-            # (doesn't store (repeated) CONNECTs from chain[0]
-            #  as it is a regular proxy)
-
-            assert not self.chain[1].tmaster.state.flows[0].response  # killed
-            assert self.chain[1].tmaster.state.flows[1].response
-
-            assert self.proxy.tmaster.state.flows[0].request.first_line_format == "authority"
-            assert self.proxy.tmaster.state.flows[1].request.first_line_format == "relative"
-
-            assert self.chain[0].tmaster.state.flows[
-                0].request.first_line_format == "authority"
-            assert self.chain[0].tmaster.state.flows[
-                1].request.first_line_format == "relative"
-            assert self.chain[0].tmaster.state.flows[
-                2].request.first_line_format == "authority"
-            assert self.chain[0].tmaster.state.flows[
-                3].request.first_line_format == "relative"
-
-            assert self.chain[1].tmaster.state.flows[
-                0].request.first_line_format == "relative"
-            assert self.chain[1].tmaster.state.flows[
-                1].request.first_line_format == "relative"
+            # First request goes through all three proxies exactly once
+            assert self.proxy.tmaster.state.flow_count() == 1
+            assert self.chain[0].tmaster.state.flow_count() == 1
+            assert self.chain[1].tmaster.state.flow_count() == 1
 
             req = p.request("get:'/p/418:b\"content2\"'")
-
             assert req.status_code == 502
-            assert self.proxy.tmaster.state.flow_count() == 3  # + new request
-            # + new request, repeated CONNECT from chain[1]
-            assert self.chain[0].tmaster.state.flow_count() == 6
-            # (both terminated)
-            # nothing happened here
-            assert self.chain[1].tmaster.state.flow_count() == 2
+
+            assert self.proxy.tmaster.state.flow_count() == 2
+            assert self.chain[0].tmaster.state.flow_count() == 2
+            # Upstream sees two requests due to reconnection attempt
+            assert self.chain[1].tmaster.state.flow_count() == 3
+            assert not self.chain[1].tmaster.state.flows[-1].response
+            assert not self.chain[1].tmaster.state.flows[-2].response
+
+            # Reconnection failed, so we're now disconnected
+            tutils.raises(
+                exceptions.HttpException,
+                p.request,
+                "get:'/p/418:b\"content3\"'"
+            )
 
 
 class AddUpstreamCertsToClientChainMixin: