diff options
24 files changed, 860 insertions, 768 deletions
diff --git a/docs/scripting/events.rst b/docs/scripting/events.rst index 62266485..5f560e58 100644 --- a/docs/scripting/events.rst +++ b/docs/scripting/events.rst @@ -98,6 +98,18 @@ HTTP Events :widths: 40 60 :header-rows: 0 + * - .. py:function:: http_connect(flow) + - Called when we receive an HTTP CONNECT request. Setting a non 2xx + response on the flow will return the response to the client abort the + connection. CONNECT requests and responses do not generate the usual + HTTP handler events. CONNECT requests are only valid in regular and + upstream proxy modes. + + *flow* + A ``models.HTTPFlow`` object. The flow is guaranteed to have + non-None ``request`` and ``requestheaders`` attributes. + + * - .. py:function:: request(flow) - Called when a client request has been received. diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py index d2b50c35..71d83dad 100644 --- a/mitmproxy/addons/__init__.py +++ b/mitmproxy/addons/__init__.py @@ -3,6 +3,7 @@ from mitmproxy.addons import anticomp from mitmproxy.addons import clientplayback from mitmproxy.addons import streamfile from mitmproxy.addons import onboarding +from mitmproxy.addons import proxyauth from mitmproxy.addons import replace from mitmproxy.addons import script from mitmproxy.addons import setheaders @@ -10,11 +11,13 @@ from mitmproxy.addons import serverplayback from mitmproxy.addons import stickyauth from mitmproxy.addons import stickycookie from mitmproxy.addons import streambodies +from mitmproxy.addons import upstream_auth def default_addons(): return [ onboarding.Onboarding(), + proxyauth.ProxyAuth(), anticache.AntiCache(), anticomp.AntiComp(), stickyauth.StickyAuth(), @@ -26,4 +29,5 @@ def default_addons(): setheaders.SetHeaders(), serverplayback.ServerPlayback(), clientplayback.ClientPlayback(), + upstream_auth.UpstreamAuth(), ] diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py new file mode 100644 index 00000000..69d45029 --- /dev/null +++ b/mitmproxy/addons/proxyauth.py @@ -0,0 +1,148 @@ +import binascii + +import passlib.apache + +from mitmproxy import exceptions +from mitmproxy import http +import mitmproxy.net.http + + +REALM = "mitmproxy" + + +def mkauth(username, password, scheme="basic"): + v = binascii.b2a_base64( + (username + ":" + password).encode("utf8") + ).decode("ascii") + return scheme + " " + v + + +def parse_http_basic_auth(s): + words = s.split() + if len(words) != 2: + return None + scheme = words[0] + try: + user = binascii.a2b_base64(words[1]).decode("utf8", "replace") + except binascii.Error: + return None + parts = user.split(':') + if len(parts) != 2: + return None + return scheme, parts[0], parts[1] + + +class ProxyAuth: + def __init__(self): + self.nonanonymous = False + self.htpasswd = None + self.singleuser = None + + def enabled(self): + return any([self.nonanonymous, self.htpasswd, self.singleuser]) + + def which_auth_header(self, f): + if f.mode == "regular": + return 'Proxy-Authorization' + else: + return 'Authorization' + + def auth_required_response(self, f): + if f.mode == "regular": + hdrname = 'Proxy-Authenticate' + else: + hdrname = 'WWW-Authenticate' + + headers = mitmproxy.net.http.Headers() + headers[hdrname] = 'Basic realm="%s"' % REALM + + if f.mode == "transparent": + return http.make_error_response( + 401, + "Authentication Required", + headers + ) + else: + return http.make_error_response( + 407, + "Proxy Authentication Required", + headers, + ) + + def check(self, f): + auth_value = f.request.headers.get(self.which_auth_header(f), None) + if not auth_value: + return False + parts = parse_http_basic_auth(auth_value) + if not parts: + return False + scheme, username, password = parts + if scheme.lower() != 'basic': + return False + + if self.nonanonymous: + pass + elif self.singleuser: + if [username, password] != self.singleuser: + return False + elif self.htpasswd: + if not self.htpasswd.check_password(username, password): + return False + else: + raise NotImplementedError("Should never happen.") + + return True + + def authenticate(self, f): + if self.check(f): + del f.request.headers[self.which_auth_header(f)] + else: + f.response = self.auth_required_response(f) + + # Handlers + def configure(self, options, updated): + if "auth_nonanonymous" in updated: + self.nonanonymous = options.auth_nonanonymous + if "auth_singleuser" in updated: + if options.auth_singleuser: + parts = options.auth_singleuser.split(':') + if len(parts) != 2: + raise exceptions.OptionsError( + "Invalid single-user auth specification." + ) + self.singleuser = parts + else: + self.singleuser = None + if "auth_htpasswd" in updated: + if options.auth_htpasswd: + try: + self.htpasswd = passlib.apache.HtpasswdFile( + options.auth_htpasswd + ) + except (ValueError, OSError) as v: + raise exceptions.OptionsError( + "Could not open htpasswd file: %s" % v + ) + else: + self.htpasswd = None + if self.enabled(): + if options.mode == "transparent": + raise exceptions.OptionsError( + "Proxy Authentication not supported in transparent mode." + ) + elif options.mode == "socks5": + raise exceptions.OptionsError( + "Proxy Authentication not supported in SOCKS mode. " + "https://github.com/mitmproxy/mitmproxy/issues/738" + ) + # TODO: check for multiple auth options + + def http_connect(self, f): + if self.enabled() and f.mode == "regular": + self.authenticate(f) + + def requestheaders(self, f): + if self.enabled(): + # Are we already authenticated in CONNECT? + if not (f.mode == "regular" and f.server_conn.via): + self.authenticate(f) diff --git a/mitmproxy/addons/upstream_auth.py b/mitmproxy/addons/upstream_auth.py new file mode 100644 index 00000000..9beecfc0 --- /dev/null +++ b/mitmproxy/addons/upstream_auth.py @@ -0,0 +1,53 @@ +import re +import base64 + +from mitmproxy import exceptions +from mitmproxy.utils import strutils + + +def parse_upstream_auth(auth): + pattern = re.compile(".+:") + if pattern.search(auth) is None: + raise exceptions.OptionsError( + "Invalid upstream auth specification: %s" % auth + ) + return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth)) + + +class UpstreamAuth(): + """ + This addon handles authentication to systems upstream from us for the + upstream proxy and reverse proxy mode. There are 3 cases: + + - Upstream proxy CONNECT requests should have authentication added, and + subsequent already connected requests should not. + - Upstream proxy regular requests + - Reverse proxy regular requests (CONNECT is invalid in this mode) + """ + def __init__(self): + self.auth = None + self.root_mode = None + + def configure(self, options, updated): + # FIXME: We're doing this because our proxy core is terminally confused + # at the moment. Ideally, we should be able to check if we're in + # reverse proxy mode at the HTTP layer, so that scripts can put the + # proxy in reverse proxy mode for specific reuests. + if "mode" in updated: + self.root_mode = options.mode + if "upstream_auth" in updated: + if options.upstream_auth is None: + self.auth = None + else: + self.auth = parse_upstream_auth(options.upstream_auth) + + def http_connect(self, f): + if self.auth and f.mode == "upstream": + f.request.headers["Proxy-Authorization"] = self.auth + + def requestheaders(self, f): + if self.auth: + if f.mode == "upstream" and not f.server_conn.via: + f.request.headers["Proxy-Authorization"] = self.auth + elif self.root_mode == "reverse": + f.request.headers["Proxy-Authorization"] = self.auth diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py index f3a75222..c7941ad9 100644 --- a/mitmproxy/connections.py +++ b/mitmproxy/connections.py @@ -42,7 +42,6 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): self.timestamp_start = time.time() self.timestamp_end = None self.timestamp_ssl_setup = None - self.protocol = None self.sni = None self.cipher_name = None self.tls_version = None @@ -144,7 +143,6 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.timestamp_end = None self.timestamp_tcp_setup = None self.timestamp_ssl_setup = None - self.protocol = None def connected(self): return bool(self.connection) and not self.finished diff --git a/mitmproxy/events.py b/mitmproxy/events.py index 56f1a45b..f9475768 100644 --- a/mitmproxy/events.py +++ b/mitmproxy/events.py @@ -13,6 +13,7 @@ Events = frozenset([ "tcp_error", "tcp_end", + "http_connect", "request", "requestheaders", "response", diff --git a/mitmproxy/http.py b/mitmproxy/http.py index 50174764..dafd4782 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -53,7 +53,7 @@ class HTTPRequest(http.Request): def get_state(self): state = super().get_state() state.update( - is_replay=self.is_replay, + is_replay=self.is_replay ) return state @@ -143,7 +143,7 @@ class HTTPFlow(flow.Flow): transaction. """ - def __init__(self, client_conn, server_conn, live=None): + def __init__(self, client_conn, server_conn, live=None, mode="regular"): super().__init__("http", client_conn, server_conn, live) self.request = None # type: HTTPRequest @@ -163,11 +163,14 @@ class HTTPFlow(flow.Flow): """:py:class:`ClientConnection` object """ self.intercepted = False # type: bool """ Is this flow currently being intercepted? """ + self.mode = mode + """ What mode was the proxy layer in when receiving this request? """ _stateobject_attributes = flow.Flow._stateobject_attributes.copy() _stateobject_attributes.update( request=HTTPRequest, - response=HTTPResponse + response=HTTPResponse, + mode=str ) def __repr__(self): diff --git a/mitmproxy/io_compat.py b/mitmproxy/io_compat.py index b1b5a296..20ee8824 100644 --- a/mitmproxy/io_compat.py +++ b/mitmproxy/io_compat.py @@ -69,6 +69,7 @@ def convert_018_019(data): data["client_conn"]["sni"] = None data["client_conn"]["cipher_name"] = None data["client_conn"]["tls_version"] = None + data["mode"] = "regular" data["metadata"] = dict() return data diff --git a/mitmproxy/master.py b/mitmproxy/master.py index ffbfb0cb..55eb74e5 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -256,6 +256,10 @@ class Master: pass @controller.handler + def http_connect(self, f): + pass + + @controller.handler def error(self, f): pass diff --git a/mitmproxy/net/http/authentication.py b/mitmproxy/net/http/authentication.py deleted file mode 100644 index a65279e4..00000000 --- a/mitmproxy/net/http/authentication.py +++ /dev/null @@ -1,176 +0,0 @@ -import argparse -import binascii - - -def parse_http_basic_auth(s): - words = s.split() - if len(words) != 2: - return None - scheme = words[0] - try: - user = binascii.a2b_base64(words[1]).decode("utf8", "replace") - except binascii.Error: - return None - parts = user.split(':') - if len(parts) != 2: - return None - return scheme, parts[0], parts[1] - - -def assemble_http_basic_auth(scheme, username, password): - v = binascii.b2a_base64((username + ":" + password).encode("utf8")).decode("ascii") - return scheme + " " + v - - -class NullProxyAuth: - - """ - No proxy auth at all (returns empty challange headers) - """ - - def __init__(self, password_manager): - self.password_manager = password_manager - - def clean(self, headers_): - """ - Clean up authentication headers, so they're not passed upstream. - """ - - def authenticate(self, headers_): - """ - Tests that the user is allowed to use the proxy - """ - return True - - def auth_challenge_headers(self): - """ - Returns a dictionary containing the headers require to challenge the user - """ - return {} - - -class BasicAuth(NullProxyAuth): - CHALLENGE_HEADER = None - AUTH_HEADER = None - - def __init__(self, password_manager, realm): - NullProxyAuth.__init__(self, password_manager) - self.realm = realm - - def clean(self, headers): - del headers[self.AUTH_HEADER] - - def authenticate(self, headers): - auth_value = headers.get(self.AUTH_HEADER) - if not auth_value: - return False - parts = parse_http_basic_auth(auth_value) - if not parts: - return False - scheme, username, password = parts - if scheme.lower() != 'basic': - return False - if not self.password_manager.test(username, password): - return False - self.username = username - return True - - def auth_challenge_headers(self): - return {self.CHALLENGE_HEADER: 'Basic realm="%s"' % self.realm} - - -class BasicWebsiteAuth(BasicAuth): - CHALLENGE_HEADER = 'WWW-Authenticate' - AUTH_HEADER = 'Authorization' - - -class BasicProxyAuth(BasicAuth): - CHALLENGE_HEADER = 'Proxy-Authenticate' - AUTH_HEADER = 'Proxy-Authorization' - - -class PassMan: - - def test(self, username_, password_token_): - return False - - -class PassManNonAnon(PassMan): - - """ - Ensure the user specifies a username, accept any password. - """ - - def test(self, username, password_token_): - if username: - return True - return False - - -class PassManHtpasswd(PassMan): - - """ - Read usernames and passwords from an htpasswd file - """ - - def __init__(self, path): - """ - Raises ValueError if htpasswd file is invalid. - """ - import passlib.apache - self.htpasswd = passlib.apache.HtpasswdFile(path) - - def test(self, username, password_token): - return bool(self.htpasswd.check_password(username, password_token)) - - -class PassManSingleUser(PassMan): - - def __init__(self, username, password): - self.username, self.password = username, password - - def test(self, username, password_token): - return self.username == username and self.password == password_token - - -class AuthAction(argparse.Action): - - """ - Helper class to allow seamless integration int argparse. Example usage: - parser.add_argument( - "--nonanonymous", - action=NonanonymousAuthAction, nargs=0, - help="Allow access to any user long as a credentials are specified." - ) - """ - - def __call__(self, parser, namespace, values, option_string=None): - passman = self.getPasswordManager(values) - authenticator = BasicProxyAuth(passman, "mitmproxy") - setattr(namespace, self.dest, authenticator) - - def getPasswordManager(self, s): # pragma: no cover - raise NotImplementedError() - - -class SingleuserAuthAction(AuthAction): - - def getPasswordManager(self, s): - if len(s.split(':')) != 2: - raise argparse.ArgumentTypeError( - "Invalid single-user specification. Please use the format username:password" - ) - username, password = s.split(':') - return PassManSingleUser(username, password) - - -class NonanonymousAuthAction(AuthAction): - - def getPasswordManager(self, s): - return PassManNonAnon() - - -class HtpasswdAuthAction(AuthAction): - - def getPasswordManager(self, s): - return PassManHtpasswd(s) diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 9c414b9c..513c0b5b 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -1,18 +1,14 @@ -import base64 import collections import os import re from typing import Any -from mitmproxy.utils import strutils - from OpenSSL import SSL, crypto from mitmproxy import exceptions from mitmproxy import options as moptions from mitmproxy import certs from mitmproxy.net import tcp -from mitmproxy.net.http import authentication from mitmproxy.net.http import url CONF_BASENAME = "mitmproxy" @@ -56,21 +52,11 @@ def parse_server_spec(spec): return ServerSpec(scheme, address) -def parse_upstream_auth(auth): - pattern = re.compile(".+:") - if pattern.search(auth) is None: - raise exceptions.OptionsError( - "Invalid upstream auth specification: %s" % auth - ) - return b"Basic" + b" " + base64.b64encode(strutils.always_bytes(auth)) - - class ProxyConfig: def __init__(self, options: moptions.Options) -> None: self.options = options - self.authenticator = None self.check_ignore = None self.check_tcp = None self.certstore = None @@ -134,54 +120,5 @@ class ProxyConfig: ) self.upstream_server = None - self.upstream_auth = None if options.upstream_server: self.upstream_server = parse_server_spec(options.upstream_server) - if options.upstream_auth: - self.upstream_auth = parse_upstream_auth(options.upstream_auth) - - self.authenticator = authentication.NullProxyAuth(None) - needsauth = any( - [ - options.auth_nonanonymous, - options.auth_singleuser, - options.auth_htpasswd - ] - ) - if needsauth: - if options.mode == "transparent": - raise exceptions.OptionsError( - "Proxy Authentication not supported in transparent mode." - ) - elif options.mode == "socks5": - raise exceptions.OptionsError( - "Proxy Authentication not supported in SOCKS mode. " - "https://github.com/mitmproxy/mitmproxy/issues/738" - ) - elif options.auth_singleuser: - parts = options.auth_singleuser.split(':') - if len(parts) != 2: - raise exceptions.OptionsError( - "Invalid single-user specification. " - "Please use the format username:password" - ) - password_manager = authentication.PassManSingleUser(*parts) - elif options.auth_nonanonymous: - password_manager = authentication.PassManNonAnon() - elif options.auth_htpasswd: - try: - password_manager = authentication.PassManHtpasswd( - options.auth_htpasswd - ) - except ValueError as v: - raise exceptions.OptionsError(str(v)) - if options.mode == "reverse": - self.authenticator = authentication.BasicWebsiteAuth( - password_manager, - self.upstream_server.address - ) - else: - self.authenticator = authentication.BasicProxyAuth( - password_manager, - "mitmproxy" - ) diff --git a/mitmproxy/proxy/modes/socks_proxy.py b/mitmproxy/proxy/modes/socks_proxy.py index 04258037..3121b731 100644 --- a/mitmproxy/proxy/modes/socks_proxy.py +++ b/mitmproxy/proxy/modes/socks_proxy.py @@ -5,9 +5,6 @@ from mitmproxy.net import socks class Socks5Proxy(protocol.Layer, protocol.ServerConnectionMixin): - def __init__(self, ctx): - super().__init__(ctx) - def __call__(self): try: # Parse Client Greeting diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index 5412827f..5f9dafab 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -1,12 +1,13 @@ import h2.exceptions import time import traceback +import enum + from mitmproxy import exceptions from mitmproxy import http from mitmproxy import flow from mitmproxy.proxy.protocol import base from mitmproxy.proxy.protocol import websockets as pwebsockets -import mitmproxy.net.http from mitmproxy.net import tcp from mitmproxy.net import websockets @@ -18,14 +19,6 @@ class _HttpTransmissionLayer(base.Layer): def read_request_body(self, request): raise NotImplementedError() - def read_request(self, f): - request = self.read_request_headers(f) - request.data.content = b"".join( - self.read_request_body(request) - ) - request.timestamp_end = time.time() - return request - def send_request(self, request): raise NotImplementedError() @@ -120,12 +113,42 @@ class UpstreamConnectLayer(base.Layer): self.server_conn.address = address +def is_ok(status): + return 200 <= status < 300 + + +class HTTPMode(enum.Enum): + regular = 1 + transparent = 2 + upstream = 3 + + +# At this point, we see only a subset of the proxy modes +MODE_REQUEST_FORMS = { + HTTPMode.regular: ("authority", "absolute"), + HTTPMode.transparent: ("relative"), + HTTPMode.upstream: ("authority", "absolute"), +} + + +def validate_request_form(mode, request): + if request.first_line_format == "absolute" and request.scheme != "http": + raise exceptions.HttpException( + "Invalid request scheme: %s" % request.scheme + ) + allowed_request_forms = MODE_REQUEST_FORMS[mode] + if request.first_line_format not in allowed_request_forms: + err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( + " or ".join(allowed_request_forms), request.first_line_format + ) + raise exceptions.HttpException(err_message) + + class HttpLayer(base.Layer): def __init__(self, ctx, mode): super().__init__(ctx) self.mode = mode - self.flow = None # type: http.HTTPFlow self.__initial_server_conn = None "Contains the original destination in transparent mode, which needs to be restored" "if an inline script modified the target server for a single http request" @@ -133,25 +156,108 @@ class HttpLayer(base.Layer): # see https://github.com/mitmproxy/mitmproxy/issues/925 self.__initial_server_tls = None # Requests happening after CONNECT do not need Proxy-Authorization headers. - self.http_authenticated = False + self.connect_request = False def __call__(self): - if self.mode == "transparent": + if self.mode == HTTPMode.transparent: self.__initial_server_tls = self.server_tls self.__initial_server_conn = self.server_conn while True: - self.flow = http.HTTPFlow(self.client_conn, self.server_conn, live=self) - if not self._process_flow(self.flow): + flow = http.HTTPFlow( + self.client_conn, + self.server_conn, + live=self, + mode=self.mode.name + ) + if not self._process_flow(flow): return - def _process_flow(self, f): + def handle_regular_connect(self, f): + self.connect_request = True + try: - request = self.get_request_from_client(f) - # Make sure that the incoming request matches our expectations - self.validate_request(request) - except exceptions.HttpReadDisconnect: - # don't throw an error for disconnects that happen before/between requests. + self.set_server((f.request.host, f.request.port)) + except ( + exceptions.ProtocolException, exceptions.NetlibException + ) as e: + # HTTPS tasting means that ordinary errors like resolution + # and connection errors can happen here. + self.send_error_response(502, repr(e)) + f.error = flow.Error(str(e)) + self.channel.ask("error", f) return False + + if f.response: + resp = f.response + else: + resp = http.make_connect_response(f.request.data.http_version) + + self.send_response(resp) + + if is_ok(resp.status_code): + layer = self.ctx.next_layer(self) + layer() + + return False + + def handle_upstream_connect(self, f): + self.establish_server_connection( + f.request.host, + f.request.port, + f.request.scheme + ) + self.send_request(f.request) + f.response = self.read_response_headers() + f.response.data.content = b"".join( + self.read_response_body(f.request, f.response) + ) + self.send_response(f.response) + if is_ok(f.response.status_code): + layer = UpstreamConnectLayer(self, f.request) + return layer() + return False + + def _process_flow(self, f): + try: + try: + request = self.read_request_headers(f) + except exceptions.HttpReadDisconnect: + # don't throw an error for disconnects that happen + # before/between requests. + return False + + f.request = request + + if request.first_line_format == "authority": + # The standards are silent on what we should do with a CONNECT + # request body, so although it's not common, it's allowed. + f.request.data.content = b"".join( + self.read_request_body(f.request) + ) + f.request.timestamp_end = time.time() + self.channel.ask("http_connect", f) + + if self.mode is HTTPMode.regular: + return self.handle_regular_connect(f) + elif self.mode is HTTPMode.upstream: + return self.handle_upstream_connect(f) + else: + msg = "Unexpected CONNECT request." + self.send_error_response(400, msg) + raise exceptions.ProtocolException(msg) + + self.channel.ask("requestheaders", f) + + if request.headers.get("expect", "").lower() == "100-continue": + # TODO: We may have to use send_response_headers for HTTP2 + # here. + self.send_response(http.expect_continue_response) + request.headers.pop("expect") + + request.data.content = b"".join(self.read_request_body(request)) + request.timestamp_end = time.time() + + validate_request_form(self.mode, request) except exceptions.HttpException as e: # We optimistically guess there might be an HTTP client on the # other end @@ -162,36 +268,25 @@ class HttpLayer(base.Layer): self.log("request", "debug", [repr(request)]) - # Handle Proxy Authentication - # Proxy Authentication conceptually does not work in transparent mode. - # We catch this misconfiguration on startup. Here, we sort out requests - # after a successful CONNECT request (which do not need to be validated anymore) - if not (self.http_authenticated or self.authenticate(request)): - return False - - f.request = request - - try: - # Regular Proxy Mode: Handle CONNECT - if self.mode == "regular" and request.first_line_format == "authority": - self.handle_regular_mode_connect(request) - return False - except (exceptions.ProtocolException, exceptions.NetlibException) as e: - # HTTPS tasting means that ordinary errors like resolution and - # connection errors can happen here. - self.send_error_response(502, repr(e)) - f.error = flow.Error(str(e)) - self.channel.ask("error", f) - return False - # update host header in reverse proxy mode if self.config.options.mode == "reverse": f.request.headers["Host"] = self.config.upstream_server.address.host - # set upstream auth - if self.mode == "upstream" and self.config.upstream_auth is not None: - f.request.headers["Proxy-Authorization"] = self.config.upstream_auth - self.process_request_hook(f) + # Determine .scheme, .host and .port attributes for inline scripts. For + # absolute-form requests, they are directly given in the request. For + # authority-form requests, we only need to determine the request + # scheme. For relative-form requests, we need to determine host and + # port as well. + if self.mode is HTTPMode.transparent: + # Setting request.host also updates the host header, which we want + # to preserve + host_header = f.request.headers.get("host", None) + f.request.host = self.__initial_server_conn.address.host + f.request.port = self.__initial_server_conn.address.port + if host_header: + f.request.headers["host"] = host_header + f.request.scheme = "https" if self.__initial_server_tls else "http" + self.channel.ask("request", f) try: if websockets.check_handshake(request.headers) and websockets.check_client_version(request.headers): @@ -205,7 +300,55 @@ class HttpLayer(base.Layer): f.request.port, f.request.scheme ) - self.get_response_from_server(f) + + def get_response(): + self.send_request(f.request) + f.response = self.read_response_headers() + + try: + get_response() + except exceptions.NetlibException as e: + self.log( + "server communication error: %s" % repr(e), + level="debug" + ) + # In any case, we try to reconnect at least once. This is + # necessary because it might be possible that we already + # initiated an upstream connection after clientconnect that + # has already been expired, e.g consider the following event + # log: + # > clientconnect (transparent mode destination known) + # > serverconnect (required for client tls handshake) + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + + if isinstance(e, exceptions.Http2ProtocolException): + # do not try to reconnect for HTTP2 + raise exceptions.ProtocolException( + "First and only attempt to get response via HTTP2 failed." + ) + + self.disconnect() + self.connect() + get_response() + + # call the appropriate script hook - this is an opportunity for + # an inline script to set f.stream = True + self.channel.ask("responseheaders", f) + + if f.response.stream: + f.response.data.content = None + else: + f.response.data.content = b"".join( + self.read_response_body(f.request, f.response) + ) + f.response.timestamp_end = time.time() + + # no further manipulation of self.server_conn beyond this point + # we can safely set it as the final attribute value here. + f.server_conn = self.server_conn else: # response was set by an inline script. # we now need to emulate the responseheaders hook. @@ -213,21 +356,50 @@ class HttpLayer(base.Layer): self.log("response", "debug", [repr(f.response)]) self.channel.ask("response", f) - self.send_response_to_client(f) + + if not f.response.stream: + # no streaming: + # we already received the full response from the server and can + # send it to the client straight away. + self.send_response(f.response) + else: + # streaming: + # First send the headers and then transfer the response incrementally + self.send_response_headers(f.response) + chunks = self.read_response_body( + f.request, + f.response + ) + if callable(f.response.stream): + chunks = f.response.stream(chunks) + self.send_response_body(f.response, chunks) + f.response.timestamp_end = time.time() if self.check_close_connection(f): return False # Handle 101 Switching Protocols if f.response.status_code == 101: - self.handle_101_switching_protocols(f) + # Handle a successful HTTP 101 Switching Protocols Response, + # received after e.g. a WebSocket upgrade request. + # Check for WebSockets handshake + is_websockets = ( + websockets.check_handshake(f.request.headers) and + websockets.check_handshake(f.response.headers) + ) + if is_websockets and not self.config.options.websockets: + self.log( + "Client requested WebSocket connection, but the protocol is disabled.", + "info" + ) + + if is_websockets and self.config.options.websockets: + layer = pwebsockets.WebSocketsLayer(self, f) + else: + layer = self.ctx.next_layer(self) + layer() return False # should never be reached - # Upstream Proxy Mode: Handle CONNECT - if f.request.first_line_format == "authority" and f.response.status_code == 200: - self.handle_upstream_mode_connect(f.request.copy()) - return False - except (exceptions.ProtocolException, exceptions.NetlibException) as e: self.send_error_response(502, repr(e)) if not f.response: @@ -244,135 +416,24 @@ class HttpLayer(base.Layer): return True - def get_request_from_client(self, f): - request = self.read_request(f) - f.request = request - self.channel.ask("requestheaders", f) - if request.headers.get("expect", "").lower() == "100-continue": - # TODO: We may have to use send_response_headers for HTTP2 here. - self.send_response(http.expect_continue_response) - request.headers.pop("expect") - request.content = b"".join(self.read_request_body(request)) - request.timestamp_end = time.time() - return request - - def send_error_response(self, code, message, headers=None): + def send_error_response(self, code, message, headers=None) -> None: try: response = http.make_error_response(code, message, headers) self.send_response(response) except (exceptions.NetlibException, h2.exceptions.H2Error, exceptions.Http2ProtocolException): self.log(traceback.format_exc(), "debug") - def change_upstream_proxy_server(self, address): + def change_upstream_proxy_server(self, address) -> None: # Make set_upstream_proxy_server always available, # even if there's no UpstreamConnectLayer if address != self.server_conn.address: - return self.set_server(address) + self.set_server(address) - def handle_regular_mode_connect(self, request): - self.http_authenticated = True - self.set_server((request.host, request.port)) - self.send_response(http.make_connect_response(request.data.http_version)) - layer = self.ctx.next_layer(self) - layer() - - def handle_upstream_mode_connect(self, connect_request): - layer = UpstreamConnectLayer(self, connect_request) - layer() - - def send_response_to_client(self, f): - if not f.response.stream: - # no streaming: - # we already received the full response from the server and can - # send it to the client straight away. - self.send_response(f.response) - else: - # streaming: - # First send the headers and then transfer the response incrementally - self.send_response_headers(f.response) - chunks = self.read_response_body( - f.request, - f.response - ) - if callable(f.response.stream): - chunks = f.response.stream(chunks) - self.send_response_body(f.response, chunks) - f.response.timestamp_end = time.time() - - def get_response_from_server(self, f): - def get_response(): - self.send_request(f.request) - f.response = self.read_response_headers() - - try: - get_response() - except exceptions.NetlibException as e: - self.log( - "server communication error: %s" % repr(e), - level="debug" - ) - # In any case, we try to reconnect at least once. This is - # necessary because it might be possible that we already - # initiated an upstream connection after clientconnect that - # has already been expired, e.g consider the following event - # log: - # > clientconnect (transparent mode destination known) - # > serverconnect (required for client tls handshake) - # > read n% of large request - # > server detects timeout, disconnects - # > read (100-n)% of large request - # > send large request upstream - - if isinstance(e, exceptions.Http2ProtocolException): - # do not try to reconnect for HTTP2 - raise exceptions.ProtocolException("First and only attempt to get response via HTTP2 failed.") - - self.disconnect() - self.connect() - get_response() - - # call the appropriate script hook - this is an opportunity for an - # inline script to set f.stream = True - self.channel.ask("responseheaders", f) - - if f.response.stream: - f.response.data.content = None - else: - f.response.data.content = b"".join(self.read_response_body( - f.request, - f.response - )) - f.response.timestamp_end = time.time() - - # no further manipulation of self.server_conn beyond this point - # we can safely set it as the final attribute value here. - f.server_conn = self.server_conn - - def process_request_hook(self, f): - # Determine .scheme, .host and .port attributes for inline scripts. - # For absolute-form requests, they are directly given in the request. - # For authority-form requests, we only need to determine the request scheme. - # For relative-form requests, we need to determine host and port as - # well. - if self.mode == "regular": - pass # only absolute-form at this point, nothing to do here. - elif self.mode == "upstream": - pass - else: - # Setting request.host also updates the host header, which we want to preserve - host_header = f.request.headers.get("host", None) - f.request.host = self.__initial_server_conn.address.host - f.request.port = self.__initial_server_conn.address.port - if host_header: - f.request.headers["host"] = host_header - f.request.scheme = "https" if self.__initial_server_tls else "http" - self.channel.ask("request", f) - - def establish_server_connection(self, host, port, scheme): + def establish_server_connection(self, host: str, port: int, scheme: str): address = tcp.Address((host, port)) tls = (scheme == "https") - if self.mode == "regular" or self.mode == "transparent": + if self.mode is HTTPMode.regular or self.mode is HTTPMode.transparent: # If there's an existing connection that doesn't match our expectations, kill it. if address != self.server_conn.address or tls != self.server_tls: self.set_server(address) @@ -385,80 +446,3 @@ class HttpLayer(base.Layer): self.connect() if tls: raise exceptions.HttpProtocolException("Cannot change scheme in upstream proxy mode.") - """ - # This is a very ugly (untested) workaround to solve a very ugly problem. - if self.server_conn and self.server_conn.tls_established and not ssl: - self.disconnect() - self.connect() - elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: - if self.server_conn.tls_established: - self.disconnect() - self.connect() - - self.send_request(make_connect_request(address)) - tls_layer = TlsLayer(self, False, True) - tls_layer._establish_tls_with_server() - """ - - def validate_request(self, request): - if request.first_line_format == "absolute" and request.scheme != "http": - raise exceptions.HttpException("Invalid request scheme: %s" % request.scheme) - - expected_request_forms = { - "regular": ("authority", "absolute",), - "upstream": ("authority", "absolute"), - "transparent": ("relative",) - } - - allowed_request_forms = expected_request_forms[self.mode] - if request.first_line_format not in allowed_request_forms: - err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( - " or ".join(allowed_request_forms), request.first_line_format - ) - raise exceptions.HttpException(err_message) - - if self.mode == "regular" and request.first_line_format == "absolute": - request.first_line_format = "relative" - - def authenticate(self, request): - if self.config.authenticator: - if self.config.authenticator.authenticate(request.headers): - self.config.authenticator.clean(request.headers) - else: - if self.mode == "transparent": - self.send_response(http.make_error_response( - 401, - "Authentication Required", - mitmproxy.net.http.Headers(**self.config.authenticator.auth_challenge_headers()) - )) - else: - self.send_response(http.make_error_response( - 407, - "Proxy Authentication Required", - mitmproxy.net.http.Headers(**self.config.authenticator.auth_challenge_headers()) - )) - return False - return True - - def handle_101_switching_protocols(self, f): - """ - Handle a successful HTTP 101 Switching Protocols Response, received after e.g. a WebSocket upgrade request. - """ - # Check for WebSockets handshake - is_websockets = ( - f and - websockets.check_handshake(f.request.headers) and - websockets.check_handshake(f.response.headers) - ) - if is_websockets and not self.config.options.websockets: - self.log( - "Client requested WebSocket connection, but the protocol is currently disabled in mitmproxy.", - "info" - ) - - if is_websockets and self.config.options.websockets: - layer = pwebsockets.WebSocketsLayer(self, f) - else: - layer = self.ctx.next_layer(self) - - layer() diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index eacf7e0b..f38f2a8c 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -2,6 +2,7 @@ from mitmproxy import log from mitmproxy import exceptions from mitmproxy.proxy import protocol from mitmproxy.proxy import modes +from mitmproxy.proxy.protocol import http class RootContext: @@ -63,16 +64,21 @@ class RootContext: # An inline script may upgrade from http to https, # in which case we need some form of TLS layer. if isinstance(top_layer, modes.ReverseProxy): - return protocol.TlsLayer(top_layer, client_tls, top_layer.server_tls, top_layer.server_conn.address.host) + return protocol.TlsLayer( + top_layer, + client_tls, + top_layer.server_tls, + top_layer.server_conn.address.host + ) if isinstance(top_layer, protocol.ServerConnectionMixin) or isinstance(top_layer, protocol.UpstreamConnectLayer): return protocol.TlsLayer(top_layer, client_tls, client_tls) # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. if isinstance(top_layer, protocol.TlsLayer): if isinstance(top_layer.ctx, modes.HttpProxy): - return protocol.Http1Layer(top_layer, "regular") + return protocol.Http1Layer(top_layer, http.HTTPMode.regular) if isinstance(top_layer.ctx, modes.HttpUpstreamProxy): - return protocol.Http1Layer(top_layer, "upstream") + return protocol.Http1Layer(top_layer, http.HTTPMode.upstream) # 4. Check for other TLS cases (e.g. after CONNECT). if client_tls: @@ -86,21 +92,12 @@ class RootContext: if isinstance(top_layer, protocol.TlsLayer): alpn = top_layer.client_conn.get_alpn_proto_negotiated() if alpn == b'h2': - return protocol.Http2Layer(top_layer, 'transparent') + return protocol.Http2Layer(top_layer, http.HTTPMode.transparent) if alpn == b'http/1.1': - return protocol.Http1Layer(top_layer, 'transparent') - - # 6. Check for raw tcp mode - is_ascii = ( - len(d) == 3 and - # expect A-Za-z - all(65 <= x <= 90 or 97 <= x <= 122 for x in d) - ) - if self.config.options.rawtcp and not is_ascii: - return protocol.RawTCPLayer(top_layer) + return protocol.Http1Layer(top_layer, http.HTTPMode.transparent) - # 7. Assume HTTP1 by default - return protocol.Http1Layer(top_layer, 'transparent') + # 6. Assume HTTP1 by default + return protocol.Http1Layer(top_layer, http.HTTPMode.transparent) def log(self, msg, level, subs=()): """ diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index debe6db9..8b579952 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -463,8 +463,8 @@ def proxy_options(parser): action="store", dest="upstream_auth", default=None, type=str, help=""" - Proxy Authentication: - username:password + Add HTTP Basic authentcation to upstream proxy and reverse proxy + requests. Format: username:password """ ) rawtcp = group.add_mutually_exclusive_group() diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index e35815ad..2cb5953f 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -12,7 +12,6 @@ from mitmproxy.addons import intercept from mitmproxy import options from mitmproxy import master from mitmproxy.tools.web import app -from mitmproxy.net.http import authentication class Stop(Exception): @@ -52,7 +51,7 @@ class Options(options.Options): wdebug: bool = False, wport: int = 8081, wiface: str = "127.0.0.1", - wauthenticator: Optional[authentication.PassMan] = None, + # wauthenticator: Optional[authentication.PassMan] = None, wsingleuser: Optional[str] = None, whtpasswd: Optional[str] = None, **kwargs @@ -60,29 +59,30 @@ class Options(options.Options): self.wdebug = wdebug self.wport = wport self.wiface = wiface - self.wauthenticator = wauthenticator - self.wsingleuser = wsingleuser - self.whtpasswd = whtpasswd + # self.wauthenticator = wauthenticator + # self.wsingleuser = wsingleuser + # self.whtpasswd = whtpasswd self.intercept = intercept super().__init__(**kwargs) # TODO: This doesn't belong here. def process_web_options(self, parser): - if self.wsingleuser or self.whtpasswd: - if self.wsingleuser: - if len(self.wsingleuser.split(':')) != 2: - return parser.error( - "Invalid single-user specification. Please use the format username:password" - ) - username, password = self.wsingleuser.split(':') - self.wauthenticator = authentication.PassManSingleUser(username, password) - elif self.whtpasswd: - try: - self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd) - except ValueError as v: - return parser.error(v.message) - else: - self.wauthenticator = None + # if self.wsingleuser or self.whtpasswd: + # if self.wsingleuser: + # if len(self.wsingleuser.split(':')) != 2: + # return parser.error( + # "Invalid single-user specification. Please use the format username:password" + # ) + # username, password = self.wsingleuser.split(':') + # # self.wauthenticator = authentication.PassManSingleUser(username, password) + # elif self.whtpasswd: + # try: + # self.wauthenticator = authentication.PassManHtpasswd(self.whtpasswd) + # except ValueError as v: + # return parser.error(v.message) + # else: + # self.wauthenticator = None + pass class WebMaster(master.Master): @@ -98,7 +98,7 @@ class WebMaster(master.Master): self.addons.add(*addons.default_addons()) self.addons.add(self.view, intercept.Intercept()) self.app = app.Application( - self, self.options.wdebug, self.options.wauthenticator + self, self.options.wdebug, False ) # This line is just for type hinting self.options = self.options # type: Options diff --git a/test/mitmproxy/addons/test_proxyauth.py b/test/mitmproxy/addons/test_proxyauth.py new file mode 100644 index 00000000..494a992f --- /dev/null +++ b/test/mitmproxy/addons/test_proxyauth.py @@ -0,0 +1,174 @@ +import binascii + +from mitmproxy import exceptions +from mitmproxy.test import taddons +from mitmproxy.test import tflow +from mitmproxy.test import tutils +from mitmproxy.addons import proxyauth + + +def test_parse_http_basic_auth(): + assert proxyauth.parse_http_basic_auth( + proxyauth.mkauth("test", "test") + ) == ("basic", "test", "test") + assert not proxyauth.parse_http_basic_auth("") + assert not proxyauth.parse_http_basic_auth("foo bar") + v = "basic " + binascii.b2a_base64(b"foo").decode("ascii") + assert not proxyauth.parse_http_basic_auth(v) + + +def test_configure(): + up = proxyauth.ProxyAuth() + with taddons.context() as ctx: + tutils.raises( + exceptions.OptionsError, + ctx.configure, up, auth_singleuser="foo" + ) + + ctx.configure(up, auth_singleuser="foo:bar") + assert up.singleuser == ["foo", "bar"] + + ctx.configure(up, auth_singleuser=None) + assert up.singleuser is None + + ctx.configure(up, auth_nonanonymous=True) + assert up.nonanonymous + ctx.configure(up, auth_nonanonymous=False) + assert not up.nonanonymous + + tutils.raises( + exceptions.OptionsError, + ctx.configure, + up, + auth_htpasswd = tutils.test_data.path( + "mitmproxy/net/data/server.crt" + ) + ) + tutils.raises( + exceptions.OptionsError, + ctx.configure, + up, + auth_htpasswd = "nonexistent" + ) + + ctx.configure( + up, + auth_htpasswd = tutils.test_data.path( + "mitmproxy/net/data/htpasswd" + ) + ) + assert up.htpasswd + assert up.htpasswd.check_password("test", "test") + assert not up.htpasswd.check_password("test", "foo") + ctx.configure(up, auth_htpasswd = None) + assert not up.htpasswd + + tutils.raises( + exceptions.OptionsError, + ctx.configure, + up, + auth_nonanonymous = True, + mode = "transparent" + ) + tutils.raises( + exceptions.OptionsError, + ctx.configure, + up, + auth_nonanonymous = True, + mode = "socks5" + ) + + +def test_check(): + up = proxyauth.ProxyAuth() + with taddons.context() as ctx: + ctx.configure(up, auth_nonanonymous=True) + f = tflow.tflow() + assert not up.check(f) + f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( + "test", "test" + ) + assert up.check(f) + + f.request.headers["Proxy-Authorization"] = "invalid" + assert not up.check(f) + + f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( + "test", "test", scheme = "unknown" + ) + assert not up.check(f) + + ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:test") + f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( + "test", "test" + ) + assert up.check(f) + ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:foo") + assert not up.check(f) + + ctx.configure( + up, + auth_singleuser = None, + auth_htpasswd = tutils.test_data.path( + "mitmproxy/net/data/htpasswd" + ) + ) + f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( + "test", "test" + ) + assert up.check(f) + f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( + "test", "foo" + ) + assert not up.check(f) + + +def test_authenticate(): + up = proxyauth.ProxyAuth() + with taddons.context() as ctx: + ctx.configure(up, auth_nonanonymous=True) + + f = tflow.tflow() + assert not f.response + up.authenticate(f) + assert f.response.status_code == 407 + + f = tflow.tflow() + f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( + "test", "test" + ) + up.authenticate(f) + assert not f.response + assert not f.request.headers.get("Proxy-Authorization") + + f = tflow.tflow() + f.mode = "transparent" + assert not f.response + up.authenticate(f) + assert f.response.status_code == 401 + + f = tflow.tflow() + f.mode = "transparent" + f.request.headers["Authorization"] = proxyauth.mkauth( + "test", "test" + ) + up.authenticate(f) + assert not f.response + assert not f.request.headers.get("Authorization") + + +def test_handlers(): + up = proxyauth.ProxyAuth() + with taddons.context() as ctx: + ctx.configure(up, auth_nonanonymous=True) + + f = tflow.tflow() + assert not f.response + up.requestheaders(f) + assert f.response.status_code == 407 + + f = tflow.tflow() + f.request.method = "CONNECT" + assert not f.response + up.http_connect(f) + assert f.response.status_code == 407 diff --git a/test/mitmproxy/addons/test_upstream_auth.py b/test/mitmproxy/addons/test_upstream_auth.py new file mode 100644 index 00000000..985b13a7 --- /dev/null +++ b/test/mitmproxy/addons/test_upstream_auth.py @@ -0,0 +1,65 @@ +import base64 + +from mitmproxy import exceptions +from mitmproxy.test import taddons +from mitmproxy.test import tflow +from mitmproxy.test import tutils +from mitmproxy.addons import upstream_auth + + +def test_configure(): + up = upstream_auth.UpstreamAuth() + with taddons.context() as tctx: + tctx.configure(up, upstream_auth="test:test") + assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:test") + + tctx.configure(up, upstream_auth="test:") + assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:") + + tctx.configure(up, upstream_auth=None) + assert not up.auth + + tutils.raises( + exceptions.OptionsError, + tctx.configure, + up, + upstream_auth="" + ) + tutils.raises( + exceptions.OptionsError, + tctx.configure, + up, + upstream_auth=":" + ) + tutils.raises( + exceptions.OptionsError, + tctx.configure, + up, + upstream_auth=":test" + ) + + +def test_simple(): + up = upstream_auth.UpstreamAuth() + with taddons.context() as tctx: + tctx.configure(up, upstream_auth="foo:bar") + + f = tflow.tflow() + f.mode = "upstream" + up.requestheaders(f) + assert "proxy-authorization" in f.request.headers + + f = tflow.tflow() + up.requestheaders(f) + assert "proxy-authorization" not in f.request.headers + + tctx.configure(up, mode="reverse") + f = tflow.tflow() + f.mode = "transparent" + up.requestheaders(f) + assert "proxy-authorization" in f.request.headers + + f = tflow.tflow() + f.mode = "upstream" + up.http_connect(f) + assert "proxy-authorization" in f.request.headers diff --git a/test/mitmproxy/net/http/test_authentication.py b/test/mitmproxy/net/http/test_authentication.py deleted file mode 100644 index 01eae52d..00000000 --- a/test/mitmproxy/net/http/test_authentication.py +++ /dev/null @@ -1,122 +0,0 @@ -import binascii - -from mitmproxy.test import tutils -from mitmproxy.net.http import authentication, Headers - - -def test_parse_http_basic_auth(): - vals = ("basic", "foo", "bar") - assert authentication.parse_http_basic_auth( - authentication.assemble_http_basic_auth(*vals) - ) == vals - assert not authentication.parse_http_basic_auth("") - assert not authentication.parse_http_basic_auth("foo bar") - v = "basic " + binascii.b2a_base64(b"foo").decode("ascii") - assert not authentication.parse_http_basic_auth(v) - - -class TestPassManNonAnon: - - def test_simple(self): - p = authentication.PassManNonAnon() - assert not p.test("", "") - assert p.test("user", "") - - -class TestPassManHtpasswd: - - def test_file_errors(self): - tutils.raises( - "malformed htpasswd file", - authentication.PassManHtpasswd, - tutils.test_data.path("mitmproxy/net/data/server.crt")) - - def test_simple(self): - pm = authentication.PassManHtpasswd(tutils.test_data.path("mitmproxy/net/data/htpasswd")) - - vals = ("basic", "test", "test") - authentication.assemble_http_basic_auth(*vals) - assert pm.test("test", "test") - assert not pm.test("test", "foo") - assert not pm.test("foo", "test") - assert not pm.test("test", "") - assert not pm.test("", "") - - -class TestPassManSingleUser: - - def test_simple(self): - pm = authentication.PassManSingleUser("test", "test") - assert pm.test("test", "test") - assert not pm.test("test", "foo") - assert not pm.test("foo", "test") - - -class TestNullProxyAuth: - - def test_simple(self): - na = authentication.NullProxyAuth(authentication.PassManNonAnon()) - assert not na.auth_challenge_headers() - assert na.authenticate("foo") - na.clean({}) - - -class TestBasicProxyAuth: - - def test_simple(self): - ba = authentication.BasicProxyAuth(authentication.PassManNonAnon(), "test") - headers = Headers() - assert ba.auth_challenge_headers() - assert not ba.authenticate(headers) - - def test_authenticate_clean(self): - ba = authentication.BasicProxyAuth(authentication.PassManNonAnon(), "test") - - headers = Headers() - vals = ("basic", "foo", "bar") - headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(*vals) - assert ba.authenticate(headers) - - ba.clean(headers) - assert ba.AUTH_HEADER not in headers - - headers[ba.AUTH_HEADER] = "" - assert not ba.authenticate(headers) - - headers[ba.AUTH_HEADER] = "foo" - assert not ba.authenticate(headers) - - vals = ("foo", "foo", "bar") - headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(*vals) - assert not ba.authenticate(headers) - - ba = authentication.BasicProxyAuth(authentication.PassMan(), "test") - vals = ("basic", "foo", "bar") - headers[ba.AUTH_HEADER] = authentication.assemble_http_basic_auth(*vals) - assert not ba.authenticate(headers) - - -class Bunch: - pass - - -class TestAuthAction: - - def test_nonanonymous(self): - m = Bunch() - aa = authentication.NonanonymousAuthAction(None, "authenticator") - aa(None, m, None, None) - assert m.authenticator - - def test_singleuser(self): - m = Bunch() - aa = authentication.SingleuserAuthAction(None, "authenticator") - aa(None, m, "foo:bar", None) - assert m.authenticator - tutils.raises("invalid", aa, None, m, "foo", None) - - def test_httppasswd(self): - m = Bunch() - aa = authentication.HtpasswdAuthAction(None, "authenticator") - aa(None, m, tutils.test_data.path("mitmproxy/net/data/htpasswd"), None) - assert m.authenticator diff --git a/test/mitmproxy/protocol/test_http1.py b/test/mitmproxy/protocol/test_http1.py index 5026bef1..cd937ada 100644 --- a/test/mitmproxy/protocol/test_http1.py +++ b/test/mitmproxy/protocol/test_http1.py @@ -20,7 +20,7 @@ class TestInvalidRequests(tservers.HTTPProxyTest): with p.connect(): r = p.request("connect:'%s:%s'" % ("127.0.0.1", self.server2.port)) assert r.status_code == 400 - assert b"Invalid HTTP request form" in r.content + assert b"Unexpected CONNECT" in r.content def test_relative_request(self): p = self.pathoc_raw() diff --git a/test/mitmproxy/test_eventsequence.py b/test/mitmproxy/test_eventsequence.py new file mode 100644 index 00000000..262df4b0 --- /dev/null +++ b/test/mitmproxy/test_eventsequence.py @@ -0,0 +1,81 @@ +from mitmproxy import events +import contextlib +from . import tservers + + +class Eventer: + def __init__(self, **handlers): + self.failure = None + self.called = [] + self.handlers = handlers + for i in events.Events - {"tick"}: + def mkprox(): + evt = i + + def prox(*args, **kwargs): + self.called.append(evt) + if evt in self.handlers: + try: + handlers[evt](*args, **kwargs) + except AssertionError as e: + self.failure = e + return prox + setattr(self, i, mkprox()) + + def fail(self): + pass + + +class SequenceTester: + @contextlib.contextmanager + def addon(self, addon): + self.master.addons.add(addon) + yield + self.master.addons.remove(addon) + if addon.failure: + raise addon.failure + + +class TestBasic(tservers.HTTPProxyTest, SequenceTester): + ssl = True + + def test_requestheaders(self): + + def hdrs(f): + assert f.request.headers + assert not f.request.content + + def req(f): + assert f.request.headers + assert f.request.content + + with self.addon(Eventer(requestheaders=hdrs, request=req)): + p = self.pathoc() + with p.connect(): + assert p.request("get:'/p/200':b@10").status_code == 200 + + def test_100_continue_fail(self): + e = Eventer() + with self.addon(e): + p = self.pathoc() + with p.connect(): + p.request( + """ + get:'/p/200' + h'expect'='100-continue' + h'content-length'='1000' + da + """ + ) + assert "requestheaders" in e.called + assert "responseheaders" not in e.called + + def test_connect(self): + e = Eventer() + with self.addon(e): + p = self.pathoc() + with p.connect(): + p.request("get:'/p/200:b@1'") + assert "http_connect" in e.called + assert e.called.count("requestheaders") == 1 + assert e.called.count("request") == 1 diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 7cadb6c2..aa3b8979 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -107,23 +107,12 @@ class TestProcessProxyOptions: self.assert_noerr("-T") self.assert_noerr("-U", "http://localhost") - self.assert_err("expected one argument", "-U") self.assert_err("Invalid server specification", "-U", "upstream") self.assert_noerr("--upstream-auth", "test:test") self.assert_err("expected one argument", "--upstream-auth") - self.assert_err( - "Invalid upstream auth specification", "--upstream-auth", "test" - ) self.assert_err("mutually exclusive", "-R", "http://localhost", "-T") - def test_socks_auth(self): - self.assert_err( - "Proxy Authentication not supported in SOCKS mode.", - "--socks", - "--nonanonymous" - ) - def test_client_certs(self): with tutils.tmpdir() as cadir: self.assert_noerr("--client-certs", cadir) @@ -141,26 +130,6 @@ class TestProcessProxyOptions: tutils.test_data.path("mitmproxy/data/testkey.pem")) self.assert_err("does not exist", "--cert", "nonexistent") - def test_auth(self): - p = self.assert_noerr("--nonanonymous") - assert p.authenticator - - p = self.assert_noerr( - "--htpasswd", - tutils.test_data.path("mitmproxy/data/htpasswd")) - assert p.authenticator - self.assert_err( - "malformed htpasswd file", - "--htpasswd", - tutils.test_data.path("mitmproxy/data/htpasswd.invalid")) - - p = self.assert_noerr("--singleuser", "test:test") - assert p.authenticator - self.assert_err( - "invalid single-user specification", - "--singleuser", - "test") - def test_insecure(self): p = self.assert_noerr("--insecure") assert p.openssl_verification_mode_server == SSL.VERIFY_NONE diff --git a/test/mitmproxy/test_proxy_config.py b/test/mitmproxy/test_proxy_config.py index e012cb5e..e2c39846 100644 --- a/test/mitmproxy/test_proxy_config.py +++ b/test/mitmproxy/test_proxy_config.py @@ -1,5 +1,4 @@ from mitmproxy.test import tutils -import base64 from mitmproxy.proxy import config @@ -26,23 +25,3 @@ def test_parse_server_spec(): config.parse_server_spec, "http://" ) - - -def test_parse_upstream_auth(): - tutils.raises( - "Invalid upstream auth specification", - config.parse_upstream_auth, - "" - ) - tutils.raises( - "Invalid upstream auth specification", - config.parse_upstream_auth, - ":" - ) - tutils.raises( - "Invalid upstream auth specification", - config.parse_upstream_auth, - ":test" - ) - assert config.parse_upstream_auth("test:test") == b"Basic" + b" " + base64.b64encode(b"test:test") - assert config.parse_upstream_auth("test:") == b"Basic" + b" " + base64.b64encode(b"test:") diff --git a/test/mitmproxy/test_server.py b/test/mitmproxy/test_server.py index 9fa6ed06..9429ab0f 100644 --- a/test/mitmproxy/test_server.py +++ b/test/mitmproxy/test_server.py @@ -6,6 +6,7 @@ from mitmproxy.test import tutils from mitmproxy import controller from mitmproxy import options from mitmproxy.addons import script +from mitmproxy.addons import proxyauth from mitmproxy import http from mitmproxy.proxy.config import HostMatcher, parse_server_spec import mitmproxy.net.http @@ -13,7 +14,6 @@ from mitmproxy.net import tcp from mitmproxy.net import socks from mitmproxy import certs from mitmproxy import exceptions -from mitmproxy.net.http import authentication from mitmproxy.net.http import http1 from mitmproxy.net.tcp import Address from pathod import pathoc @@ -50,10 +50,7 @@ class CommonMixin: def test_replay(self): assert self.pathod("304").status_code == 304 - if isinstance(self, tservers.HTTPUpstreamProxyTest) and self.ssl: - assert len(self.master.state.flows) == 2 - else: - assert len(self.master.state.flows) == 1 + assert len(self.master.state.flows) == 1 l = self.master.state.flows[-1] assert l.response.status_code == 304 l.request.path = "/p/305" @@ -288,6 +285,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin): class TestHTTPAuth(tservers.HTTPProxyTest): def test_auth(self): + self.master.addons.add(proxyauth.ProxyAuth()) self.master.options.auth_singleuser = "test:test" assert self.pathod("202").status_code == 407 p = self.pathoc() @@ -298,14 +296,15 @@ class TestHTTPAuth(tservers.HTTPProxyTest): h'%s'='%s' """ % ( self.server.port, - mitmproxy.net.http.authentication.BasicProxyAuth.AUTH_HEADER, - authentication.assemble_http_basic_auth("basic", "test", "test") + "Proxy-Authorization", + proxyauth.mkauth("test", "test") )) assert ret.status_code == 202 class TestHTTPReverseAuth(tservers.ReverseProxyTest): def test_auth(self): + self.master.addons.add(proxyauth.ProxyAuth()) self.master.options.auth_singleuser = "test:test" assert self.pathod("202").status_code == 401 p = self.pathoc() @@ -315,8 +314,8 @@ class TestHTTPReverseAuth(tservers.ReverseProxyTest): '/p/202' h'%s'='%s' """ % ( - mitmproxy.net.http.authentication.BasicWebsiteAuth.AUTH_HEADER, - authentication.assemble_http_basic_auth("basic", "test", "test") + "Authorization", + proxyauth.mkauth("test", "test") )) assert ret.status_code == 202 @@ -672,6 +671,13 @@ class TestProxySSL(tservers.HTTPProxyTest): first_flow = self.master.state.flows[0] assert first_flow.server_conn.timestamp_ssl_setup + def test_via(self): + # tests that the ssl timestamp is present when ssl is used + f = self.pathod("200:b@10") + assert f.status_code == 200 + first_flow = self.master.state.flows[0] + assert not first_flow.server_conn.via + class MasterRedirectRequest(tservers.TestMaster): redirect_port = None # Set by TestRedirectRequest @@ -952,12 +958,15 @@ class TestUpstreamProxySSL( assert req.status_code == 418 # CONNECT from pathoc to chain[0], - assert self.proxy.tmaster.state.flow_count() == 2 + assert self.proxy.tmaster.state.flow_count() == 1 + assert self.proxy.tmaster.state.flows[0].server_conn.via # request from pathoc to chain[0] # CONNECT from proxy to chain[1], - assert self.chain[0].tmaster.state.flow_count() == 2 + assert self.chain[0].tmaster.state.flow_count() == 1 + assert self.chain[0].tmaster.state.flows[0].server_conn.via # request from proxy to chain[1] # request from chain[0] (regular proxy doesn't store CONNECTs) + assert not self.chain[1].tmaster.state.flows[0].server_conn.via assert self.chain[1].tmaster.state.flow_count() == 1 @@ -978,21 +987,12 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest): def test_reconnect(self): """ Tests proper functionality of ConnectionHandler.server_reconnect mock. - If we have a disconnect on a secure connection that's transparently proxified to - an upstream http proxy, we need to send the CONNECT request again. + If we have a disconnect on a secure connection that's transparently + proxified to an upstream http proxy, we need to send the CONNECT + request again. """ - self.chain[1].tmaster.addons.add( - RequestKiller([2]) - ) - self.chain[0].tmaster.addons.add( - RequestKiller( - [ - 1, # CONNECT - 3, # reCONNECT - 4 # request - ] - ) - ) + self.chain[0].tmaster.addons.add(RequestKiller([1, 2])) + self.chain[1].tmaster.addons.add(RequestKiller([1])) p = self.pathoc() with p.connect(): @@ -1000,44 +1000,27 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxyTest): assert req.content == b"content" assert req.status_code == 418 - assert self.proxy.tmaster.state.flow_count() == 2 # CONNECT and request - # CONNECT, failing request, - assert self.chain[0].tmaster.state.flow_count() == 4 - # reCONNECT, request - # failing request, request - assert self.chain[1].tmaster.state.flow_count() == 2 - # (doesn't store (repeated) CONNECTs from chain[0] - # as it is a regular proxy) - - assert not self.chain[1].tmaster.state.flows[0].response # killed - assert self.chain[1].tmaster.state.flows[1].response - - assert self.proxy.tmaster.state.flows[0].request.first_line_format == "authority" - assert self.proxy.tmaster.state.flows[1].request.first_line_format == "relative" - - assert self.chain[0].tmaster.state.flows[ - 0].request.first_line_format == "authority" - assert self.chain[0].tmaster.state.flows[ - 1].request.first_line_format == "relative" - assert self.chain[0].tmaster.state.flows[ - 2].request.first_line_format == "authority" - assert self.chain[0].tmaster.state.flows[ - 3].request.first_line_format == "relative" - - assert self.chain[1].tmaster.state.flows[ - 0].request.first_line_format == "relative" - assert self.chain[1].tmaster.state.flows[ - 1].request.first_line_format == "relative" + # First request goes through all three proxies exactly once + assert self.proxy.tmaster.state.flow_count() == 1 + assert self.chain[0].tmaster.state.flow_count() == 1 + assert self.chain[1].tmaster.state.flow_count() == 1 req = p.request("get:'/p/418:b\"content2\"'") - assert req.status_code == 502 - assert self.proxy.tmaster.state.flow_count() == 3 # + new request - # + new request, repeated CONNECT from chain[1] - assert self.chain[0].tmaster.state.flow_count() == 6 - # (both terminated) - # nothing happened here - assert self.chain[1].tmaster.state.flow_count() == 2 + + assert self.proxy.tmaster.state.flow_count() == 2 + assert self.chain[0].tmaster.state.flow_count() == 2 + # Upstream sees two requests due to reconnection attempt + assert self.chain[1].tmaster.state.flow_count() == 3 + assert not self.chain[1].tmaster.state.flows[-1].response + assert not self.chain[1].tmaster.state.flows[-2].response + + # Reconnection failed, so we're now disconnected + tutils.raises( + exceptions.HttpException, + p.request, + "get:'/p/418:b\"content3\"'" + ) class AddUpstreamCertsToClientChainMixin: |