diff options
52 files changed, 1333 insertions, 1459 deletions
diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py index 97fa2dcd..16510640 100644 --- a/mitmproxy/addons/__init__.py +++ b/mitmproxy/addons/__init__.py @@ -3,6 +3,7 @@ from mitmproxy.addons import anticomp from mitmproxy.addons import check_alpn from mitmproxy.addons import check_ca from mitmproxy.addons import clientplayback +from mitmproxy.addons import core_option_validation from mitmproxy.addons import disable_h2c_upgrade from mitmproxy.addons import onboarding from mitmproxy.addons import proxyauth @@ -19,6 +20,7 @@ from mitmproxy.addons import upstream_auth def default_addons(): return [ + core_option_validation.CoreOptionValidation(), anticache.AntiCache(), anticomp.AntiComp(), check_alpn.CheckALPN(), diff --git a/mitmproxy/addons/core_option_validation.py b/mitmproxy/addons/core_option_validation.py new file mode 100644 index 00000000..fd5f2788 --- /dev/null +++ b/mitmproxy/addons/core_option_validation.py @@ -0,0 +1,45 @@ +""" + The core addon is responsible for verifying core settings that are not + checked by other addons. +""" +from mitmproxy import exceptions +from mitmproxy import platform +from mitmproxy.net import server_spec +from mitmproxy.utils import human + + +class CoreOptionValidation: + def configure(self, opts, updated): + if opts.add_upstream_certs_to_client_chain and not opts.upstream_cert: + raise exceptions.OptionsError( + "The no-upstream-cert and add-upstream-certs-to-client-chain " + "options are mutually exclusive. If no-upstream-cert is enabled " + "then the upstream certificate is not retrieved before generating " + "the client certificate chain." + ) + if "body_size_limit" in updated and opts.body_size_limit: + try: + opts._processed["body_size_limit"] = human.parse_size( + opts.body_size_limit + ) + except ValueError as e: + raise exceptions.OptionsError( + "Invalid body size limit specification: %s" % + opts.body_size_limit + ) + if "mode" in updated: + mode = opts.mode + if mode.startswith("reverse:") or mode.startswith("upstream:"): + try: + server_spec.parse_with_mode(mode) + except ValueError as e: + raise exceptions.OptionsError(str(e)) from e + elif mode == "transparent": + if not platform.original_addr: + raise exceptions.OptionsError( + "Transparent mode not supported on this platform." + ) + elif mode not in ["regular", "socks5"]: + raise exceptions.OptionsError( + "Invalid mode specification: %s" % mode + ) diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py index 18a85866..61477658 100644 --- a/mitmproxy/addons/proxyauth.py +++ b/mitmproxy/addons/proxyauth.py @@ -114,30 +114,28 @@ class ProxyAuth: # Handlers def configure(self, options, updated): - if "auth_nonanonymous" in updated: - self.nonanonymous = options.auth_nonanonymous - if "auth_singleuser" in updated: - if options.auth_singleuser: - parts = options.auth_singleuser.split(':') - if len(parts) != 2: - raise exceptions.OptionsError( - "Invalid single-user auth specification." - ) - self.singleuser = parts - else: - self.singleuser = None - if "auth_htpasswd" in updated: - if options.auth_htpasswd: - try: - self.htpasswd = passlib.apache.HtpasswdFile( - options.auth_htpasswd - ) - except (ValueError, OSError) as v: - raise exceptions.OptionsError( - "Could not open htpasswd file: %s" % v - ) - else: - self.htpasswd = None + if "proxyauth" in updated: + self.nonanonymous = False + self.singleuser = None + self.htpasswd = None + if options.proxyauth: + if options.proxyauth == "any": + self.nonanonymous = True + elif options.proxyauth.startswith("@"): + p = options.proxyauth[1:] + try: + self.htpasswd = passlib.apache.HtpasswdFile(p) + except (ValueError, OSError) as v: + raise exceptions.OptionsError( + "Could not open htpasswd file: %s" % p + ) + else: + parts = options.proxyauth.split(':') + if len(parts) != 2: + raise exceptions.OptionsError( + "Invalid single-user auth specification." + ) + self.singleuser = parts if "mode" in updated: self.mode = options.mode if self.enabled(): diff --git a/mitmproxy/addons/replace.py b/mitmproxy/addons/replace.py index 34bb40c2..0d0c3aa5 100644 --- a/mitmproxy/addons/replace.py +++ b/mitmproxy/addons/replace.py @@ -57,10 +57,7 @@ class _ReplaceBase: if self.optionName in updated: lst = [] for rep in getattr(options, self.optionName): - if isinstance(rep, str): - fpatt, rex, s = parse_hook(rep) - else: - fpatt, rex, s = rep + fpatt, rex, s = parse_hook(rep) flt = flowfilter.parse(fpatt) if not flt: diff --git a/mitmproxy/addons/setheaders.py b/mitmproxy/addons/setheaders.py index 95cf9a09..9e60eabd 100644 --- a/mitmproxy/addons/setheaders.py +++ b/mitmproxy/addons/setheaders.py @@ -54,10 +54,7 @@ class SetHeaders: if "setheaders" in updated: self.lst = [] for shead in options.setheaders: - if isinstance(shead, str): - fpatt, header, value = parse_setheader(shead) - else: - fpatt, header, value = shead + fpatt, header, value = parse_setheader(shead) flt = flowfilter.parse(fpatt) if not flt: diff --git a/mitmproxy/addons/streambodies.py b/mitmproxy/addons/streambodies.py index 3c2a153b..a10bdb93 100644 --- a/mitmproxy/addons/streambodies.py +++ b/mitmproxy/addons/streambodies.py @@ -1,6 +1,7 @@ from mitmproxy.net.http import http1 from mitmproxy import exceptions from mitmproxy import ctx +from mitmproxy.utils import human class StreamBodies: @@ -8,7 +9,11 @@ class StreamBodies: self.max_size = None def configure(self, options, updated): - self.max_size = options.stream_large_bodies + if "stream_large_bodies" in updated and options.stream_large_bodies: + try: + self.max_size = human.parse_size(options.stream_large_bodies) + except ValueError as e: + raise exceptions.OptionsError(e) def run(self, f, is_request): if self.max_size: diff --git a/mitmproxy/addons/streamfile.py b/mitmproxy/addons/streamfile.py index 5517e9dc..624297f2 100644 --- a/mitmproxy/addons/streamfile.py +++ b/mitmproxy/addons/streamfile.py @@ -35,11 +35,13 @@ class StreamFile: if self.stream: self.done() if options.streamfile: - if options.streamfile_append: + if options.streamfile.startswith("+"): + path = options.streamfile[1:] mode = "ab" else: + path = options.streamfile mode = "wb" - self.start_stream_to_path(options.streamfile, mode, self.filt) + self.start_stream_to_path(path, mode, self.filt) def tcp_start(self, flow): if self.stream: diff --git a/mitmproxy/master.py b/mitmproxy/master.py index 633f32aa..8855452c 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -148,7 +148,7 @@ class Master: Loads a flow """ if isinstance(f, http.HTTPFlow): - if self.server and self.options.mode == "reverse": + if self.server and self.options.mode.startswith("reverse:"): f.request.host = self.server.config.upstream_server.address[0] f.request.port = self.server.config.upstream_server.address[1] f.request.scheme = self.server.config.upstream_server.scheme diff --git a/mitmproxy/net/check.py b/mitmproxy/net/check.py index d30c1df6..aaea851f 100644 --- a/mitmproxy/net/check.py +++ b/mitmproxy/net/check.py @@ -29,5 +29,5 @@ def is_valid_host(host: bytes) -> bool: return False -def is_valid_port(port): +def is_valid_port(port: int) -> bool: return 0 <= port <= 65535 diff --git a/mitmproxy/net/server_spec.py b/mitmproxy/net/server_spec.py new file mode 100644 index 00000000..efbf1012 --- /dev/null +++ b/mitmproxy/net/server_spec.py @@ -0,0 +1,76 @@ +""" +Parse scheme, host and port from a string. +""" +import collections +import re +from typing import Tuple + +from mitmproxy.net import check + +ServerSpec = collections.namedtuple("ServerSpec", ["scheme", "address"]) + +server_spec_re = re.compile( + r""" + ^ + (?:(?P<scheme>\w+)://)? # scheme is optional + (?P<host>[^:/]+|\[.+\]) # hostname can be DNS name, IPv4, or IPv6 address. + (?::(?P<port>\d+))? # port is optional + /? # we allow a trailing backslash, but no path + $ + """, + re.VERBOSE +) + + +def parse(server_spec: str) -> ServerSpec: + """ + Parses a server mode specification, e.g.: + + - http://example.com/ + - example.org + - example.com:443 + + Raises: + ValueError, if the server specification is invalid. + """ + m = server_spec_re.match(server_spec) + if not m: + raise ValueError("Invalid server specification: {}".format(server_spec)) + + # defaulting to https/port 443 may annoy some folks, but it's secure-by-default. + scheme = m.group("scheme") or "https" + if scheme not in ("http", "https"): + raise ValueError("Invalid server scheme: {}".format(scheme)) + + host = m.group("host") + # IPv6 brackets + if host.startswith("[") and host.endswith("]"): + host = host[1:-1] + if not check.is_valid_host(host.encode("idna")): + raise ValueError("Invalid hostname: {}".format(host)) + + if m.group("port"): + port = int(m.group("port")) + else: + port = { + "http": 80, + "https": 443 + }[scheme] + if not check.is_valid_port(port): + raise ValueError("Invalid port: {}".format(port)) + + return ServerSpec(scheme, (host, port)) + + +def parse_with_mode(mode: str) -> Tuple[str, ServerSpec]: + """ + Parse a proxy mode specification, which is usually just (reverse|upstream):server-spec + + Returns: + A (mode, server_spec) tuple. + + Raises: + ValueError, if the specification is invalid. + """ + mode, server_spec = mode.split(":", maxsplit=1) + return mode, parse(server_spec) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index ff17fbbf..6dd8616b 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -1,6 +1,25 @@ -from typing import Tuple, Optional, Sequence, Union +from typing import Optional, Sequence from mitmproxy import optmanager +from mitmproxy.net import tcp + +# We redefine these here for now to avoid importing Urwid-related guff on +# platforms that don't support it, and circular imports. We can do better using +# a lazy checker down the track. +console_palettes = [ + "lowlight", + "lowdark", + "light", + "dark", + "solarized_light", + "solarized_dark" +] +view_orders = [ + "time", + "method", + "url", + "size", +] APP_HOST = "mitm.it" APP_PORT = 80 @@ -9,198 +28,416 @@ LISTEN_PORT = 8080 # We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. # https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old -DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:" \ - "ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:" \ - "ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:" \ - "ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:" \ - "DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:" \ - "DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:" \ - "AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:" \ - "HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:" \ +DEFAULT_CLIENT_CIPHERS = ( + "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:" + "ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:" + "ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:" + "ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:" + "DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:" + "DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:" + "AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:" + "HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:" "!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" +) class Options(optmanager.OptManager): - def __init__( - self, - *, # all args are keyword-only. - onboarding: bool = True, - onboarding_host: str = APP_HOST, - onboarding_port: int = APP_PORT, - anticache: bool = False, - anticomp: bool = False, - client_replay: Sequence[str] = [], - replay_kill_extra: bool = False, - keepserving: bool = True, - no_server: bool = False, - server_replay_nopop: bool = False, - refresh_server_playback: bool = True, - rfile: Optional[str] = None, - scripts: Sequence[str] = [], - showhost: bool = False, - replacements: Sequence[Union[Tuple[str, str, str], str]] = [], - replacement_files: Sequence[Union[Tuple[str, str, str], str]] = [], - server_replay_use_headers: Sequence[str] = [], - setheaders: Sequence[Union[Tuple[str, str, str], str]] = [], - server_replay: Sequence[str] = [], - stickycookie: Optional[str] = None, - stickyauth: Optional[str] = None, - stream_large_bodies: Optional[int] = None, - verbosity: int = 2, - default_contentview: str = "auto", - streamfile: Optional[str] = None, - streamfile_append: bool = False, - server_replay_ignore_content: bool = False, - server_replay_ignore_params: Sequence[str] = [], - server_replay_ignore_payload_params: Sequence[str] = [], - server_replay_ignore_host: bool = False, - - # Proxy options - auth_nonanonymous: bool = False, - auth_singleuser: Optional[str] = None, - auth_htpasswd: Optional[str] = None, - add_upstream_certs_to_client_chain: bool = False, - body_size_limit: Optional[int] = None, - cadir: str = CA_DIR, - certs: Sequence[Tuple[str, str]] = [], - ciphers_client: str=DEFAULT_CLIENT_CIPHERS, - ciphers_server: Optional[str]=None, - clientcerts: Optional[str] = None, - ignore_hosts: Sequence[str] = [], - listen_host: str = "", - listen_port: int = LISTEN_PORT, - upstream_bind_address: str = "", - mode: str = "regular", - no_upstream_cert: bool = False, - keep_host_header: bool = False, - - http2: bool = True, - http2_priority: bool = False, - websocket: bool = True, - rawtcp: bool = False, - - spoof_source_address: bool = False, - upstream_server: Optional[str] = None, - upstream_auth: Optional[str] = None, - ssl_version_client: str = "secure", - ssl_version_server: str = "secure", - ssl_insecure: bool = False, - ssl_verify_upstream_trusted_cadir: Optional[str] = None, - ssl_verify_upstream_trusted_ca: Optional[str] = None, - tcp_hosts: Sequence[str] = [], - - intercept: Optional[str] = None, - - # Console options - console_eventlog: bool = False, - console_focus_follow: bool = False, - console_palette: Optional[str] = "dark", - console_palette_transparent: bool = False, - console_no_mouse: bool = False, - console_order: Optional[str] = None, - console_order_reversed: bool = False, - - filter: Optional[str] = None, - - # Web options - web_open_browser: bool = True, - web_debug: bool = False, - web_port: int = 8081, - web_iface: str = "127.0.0.1", - - # Dump options - filtstr: Optional[str] = None, - flow_detail: int = 1 - ) -> None: - # We could replace all assignments with clever metaprogramming, - # but type hints are a much more valueable asset. - - self.onboarding = onboarding - self.onboarding_host = onboarding_host - self.onboarding_port = onboarding_port - self.anticache = anticache - self.anticomp = anticomp - self.client_replay = client_replay - self.keepserving = keepserving - self.replay_kill_extra = replay_kill_extra - self.no_server = no_server - self.server_replay_nopop = server_replay_nopop - self.refresh_server_playback = refresh_server_playback - self.rfile = rfile - self.scripts = scripts - self.showhost = showhost - self.replacements = replacements - self.replacement_files = replacement_files - self.server_replay_use_headers = server_replay_use_headers - self.setheaders = setheaders - self.server_replay = server_replay - self.stickycookie = stickycookie - self.stickyauth = stickyauth - self.stream_large_bodies = stream_large_bodies - self.verbosity = verbosity - self.default_contentview = default_contentview - self.streamfile = streamfile - self.streamfile_append = streamfile_append - self.server_replay_ignore_content = server_replay_ignore_content - self.server_replay_ignore_params = server_replay_ignore_params - self.server_replay_ignore_payload_params = server_replay_ignore_payload_params - self.server_replay_ignore_host = server_replay_ignore_host + def __init__(self, **kwargs) -> None: + super().__init__() + self.add_option( + "onboarding", bool, True, + "Toggle the mitmproxy onboarding app." + ) + self.add_option( + "onboarding_host", str, APP_HOST, + """ + Domain to serve the onboarding app from. For transparent mode, use + an IP when a DNS entry for the app domain is not present. """ + ) + self.add_option( + "onboarding_port", int, APP_PORT, + "Port to serve the onboarding app from." + ) + self.add_option( + "anticache", bool, False, + """ + Strip out request headers that might cause the server to return + 304-not-modified. + """ + ) + self.add_option( + "anticomp", bool, False, + "Try to convince servers to send us un-compressed data." + ) + self.add_option( + "client_replay", Sequence[str], [], + "Replay client requests from a saved file." + ) + self.add_option( + "replay_kill_extra", bool, False, + "Kill extra requests during replay." + ) + self.add_option( + "keepserving", bool, True, + "Continue serving after client playback or file read." + ) + self.add_option( + "server", bool, True, + "Start a proxy server." + ) + self.add_option( + "server_replay_nopop", bool, False, + """ + Disable response pop from response flow. This makes it possible to + replay same response multiple times. + """ + ) + self.add_option( + "refresh_server_playback", bool, True, + """ + Refresh server replay responses by adjusting date, expires and + last-modified headers, as well as adjusting cookie expiration. + """ + ) + self.add_option( + "rfile", Optional[str], None, + "Read flows from file." + ) + self.add_option( + "scripts", Sequence[str], [], + """ + Execute a script. + """ + ) + self.add_option( + "showhost", bool, False, + "Use the Host header to construct URLs for display." + ) + self.add_option( + "replacements", Sequence[str], [], + """ + Replacement patterns of the form "/pattern/regex/replacement", where + the separator can be any character. + """ + ) + self.add_option( + "replacement_files", Sequence[str], [], + """ + Replacement pattern, where the replacement clause is a path to a + file. + """ + ) + self.add_option( + "server_replay_use_headers", Sequence[str], [], + "Request headers to be considered during replay." + ) + self.add_option( + "setheaders", Sequence[str], [], + """ + Header set pattern of the form "/pattern/header/value", where the + separator can be any character. + """ + ) + self.add_option( + "server_replay", Sequence[str], [], + "Replay server responses from a saved file." + ) + self.add_option( + "stickycookie", Optional[str], None, + "Set sticky cookie filter. Matched against requests." + ) + self.add_option( + "stickyauth", Optional[str], None, + "Set sticky auth filter. Matched against requests." + ) + self.add_option( + "stream_large_bodies", Optional[str], None, + """ + Stream data to the client if response body exceeds the given + threshold. If streamed, the body will not be stored in any way. + Understands k/m/g suffixes, i.e. 3m for 3 megabytes. + """ + ) + self.add_option( + "verbosity", int, 2, + "Log verbosity." + ) + self.add_option( + "default_contentview", str, "auto", + "The default content view mode." + ) + self.add_option( + "streamfile", Optional[str], None, + "Write flows to file. Prefix path with + to append." + ) + self.add_option( + "server_replay_ignore_content", bool, False, + "Ignore request's content while searching for a saved flow to replay." + ) + self.add_option( + "server_replay_ignore_params", Sequence[str], [], + """ + Request's parameters to be ignored while searching for a saved flow + to replay. Can be passed multiple times. + """ + ) + self.add_option( + "server_replay_ignore_payload_params", Sequence[str], [], + """ + Request's payload parameters (application/x-www-form-urlencoded or + multipart/form-data) to be ignored while searching for a saved flow + to replay. + """ + ) + self.add_option( + "server_replay_ignore_host", bool, False, + """ + Ignore request's destination host while searching for a saved flow + to replay. + """ + ) # Proxy options - self.auth_nonanonymous = auth_nonanonymous - self.auth_singleuser = auth_singleuser - self.auth_htpasswd = auth_htpasswd - self.add_upstream_certs_to_client_chain = add_upstream_certs_to_client_chain - self.body_size_limit = body_size_limit - self.cadir = cadir - self.certs = certs - self.ciphers_client = ciphers_client - self.ciphers_server = ciphers_server - self.clientcerts = clientcerts - self.ignore_hosts = ignore_hosts - self.listen_host = listen_host - self.listen_port = listen_port - self.upstream_bind_address = upstream_bind_address - self.mode = mode - self.no_upstream_cert = no_upstream_cert - self.keep_host_header = keep_host_header + self.add_option( + "proxyauth", Optional[str], None, + """ + Require authentication before proxying requests. If the value is + "any", we prompt for authentication, but permit any values. If it + starts with an "@", it is treated as a path to an Apache htpasswd + file. If its is of the form "username:password", it is treated as a + single-user credential. + """ + ) + self.add_option( + "add_upstream_certs_to_client_chain", bool, False, + """ + Add all certificates of the upstream server to the certificate chain + that will be served to the proxy client, as extras. + """ + ) + self.add_option( + "body_size_limit", Optional[str], None, + """ + Byte size limit of HTTP request and response bodies. Understands + k/m/g suffixes, i.e. 3m for 3 megabytes. + """ + ) + self.add_option( + "cadir", str, CA_DIR, + "Location of the default mitmproxy CA files." + ) + self.add_option( + "certs", Sequence[str], [], + """ + SSL certificates. SPEC is of the form "[domain=]path". The + domain may include a wildcard, and is equal to "*" if not specified. + The file at path is a certificate in PEM format. If a private key is + included in the PEM, it is used, else the default key in the conf + dir is used. The PEM file should contain the full certificate chain, + with the leaf certificate as the first entry. Can be passed multiple + times. + """ + ) + self.add_option( + "ciphers_client", str, DEFAULT_CLIENT_CIPHERS, + "Set supported ciphers for client connections using OpenSSL syntax." + ) + self.add_option( + "ciphers_server", Optional[str], None, + "Set supported ciphers for server connections using OpenSSL syntax." + ) + self.add_option( + "client_certs", Optional[str], None, + "Client certificate file or directory." + ) + self.add_option( + "ignore_hosts", Sequence[str], [], + """ + Ignore host and forward all traffic without processing it. In + transparent mode, it is recommended to use an IP address (range), + not the hostname. In regular mode, only SSL traffic is ignored and + the hostname should be used. The supplied value is interpreted as a + regular expression and matched on the ip or the hostname. + """ + ) + self.add_option( + "listen_host", str, "", + "Address to bind proxy to." + ) + self.add_option( + "listen_port", int, LISTEN_PORT, + "Proxy service port." + ) + self.add_option( + "upstream_bind_address", str, "", + "Address to bind upstream requests to." + ) + self.add_option( + "mode", str, "regular", + """ + Mode can be "regular", "transparent", "socks5", "reverse:SPEC", + or "upstream:SPEC". For reverse and upstream proxy modes, SPEC + is proxy specification in the form of "http[s]://host[:port]". + """ + ) + self.add_option( + "upstream_cert", bool, True, + "Connect to upstream server to look up certificate details." + ) + self.add_option( + "keep_host_header", bool, False, + """ + Reverse Proxy: Keep the original host header instead of rewriting it + to the reverse proxy target. + """ + ) - self.http2 = http2 - self.http2_priority = http2_priority - self.websocket = websocket - self.rawtcp = rawtcp + self.add_option( + "http2", bool, True, + "Enable/disable HTTP/2 support. " + "HTTP/2 support is enabled by default.", + ) + self.add_option( + "http2_priority", bool, False, + """ + PRIORITY forwarding for HTTP/2 connections. PRIORITY forwarding is + disabled by default, because some webservers fail to implement the + RFC properly. + """ + ) + self.add_option( + "websocket", bool, True, + "Enable/disable WebSocket support. " + "WebSocket support is enabled by default.", + ) + self.add_option( + "rawtcp", bool, False, + "Enable/disable experimental raw TCP support. " + "Disabled by default. " + ) - self.spoof_source_address = spoof_source_address - self.upstream_server = upstream_server - self.upstream_auth = upstream_auth - self.ssl_version_client = ssl_version_client - self.ssl_version_server = ssl_version_server - self.ssl_insecure = ssl_insecure - self.ssl_verify_upstream_trusted_cadir = ssl_verify_upstream_trusted_cadir - self.ssl_verify_upstream_trusted_ca = ssl_verify_upstream_trusted_ca - self.tcp_hosts = tcp_hosts + self.add_option( + "spoof_source_address", bool, False, + """ + Use the client's IP for server-side connections. Combine with + --upstream-bind-address to spoof a fixed source address. + """ + ) + self.add_option( + "upstream_auth", Optional[str], None, + """ + Add HTTP Basic authentcation to upstream proxy and reverse proxy + requests. Format: username:password. + """ + ) + self.add_option( + "ssl_version_client", str, "secure", + """ + Set supported SSL/TLS versions for client connections. SSLv2, SSLv3 + and 'all' are INSECURE. Defaults to secure, which is TLS1.0+. + """, + choices=tcp.sslversion_choices.keys(), + ) + self.add_option( + "ssl_version_server", str, "secure", + """ + Set supported SSL/TLS versions for server connections. SSLv2, SSLv3 + and 'all' are INSECURE. Defaults to secure, which is TLS1.0+. + """, + choices=tcp.sslversion_choices.keys(), + ) + self.add_option( + "ssl_insecure", bool, False, + "Do not verify upstream server SSL/TLS certificates." + ) + self.add_option( + "ssl_verify_upstream_trusted_cadir", Optional[str], None, + """ + Path to a directory of trusted CA certificates for upstream server + verification prepared using the c_rehash tool. + """ + ) + self.add_option( + "ssl_verify_upstream_trusted_ca", Optional[str], None, + "Path to a PEM formatted trusted CA certificate." + ) + self.add_option( + "tcp_hosts", Sequence[str], [], + """ + Generic TCP SSL proxy mode for all hosts that match the pattern. + Similar to --ignore, but SSL connections are intercepted. The + communication contents are printed to the log in verbose mode. + """ + ) - self.intercept = intercept + self.add_option( + "intercept", Optional[str], None, + "Intercept filter expression." + ) # Console options - self.console_eventlog = console_eventlog - self.console_focus_follow = console_focus_follow - self.console_palette = console_palette - self.console_palette_transparent = console_palette_transparent - self.console_no_mouse = console_no_mouse - self.console_order = console_order - self.console_order_reversed = console_order_reversed + self.add_option( + "console_eventlog", bool, False, + "Show event log." + ) + self.add_option( + "console_focus_follow", bool, False, + "Focus follows new flows." + ) + self.add_option( + "console_palette", str, "dark", + "Color palette.", + choices=sorted(console_palettes), + ) + self.add_option( + "console_palette_transparent", bool, False, + "Set transparent background for palette." + ) + self.add_option( + "console_mouse", bool, True, + "Console mouse interaction." + ) + self.add_option( + "console_order", Optional[str], None, + "Flow sort order.", + choices=view_orders, + ) + self.add_option( + "console_order_reversed", bool, False, + "Reverse the sorting order." + ) - self.filter = filter + self.add_option( + "filter", Optional[str], None, + "Filter view expression." + ) # Web options - self.web_open_browser = web_open_browser - self.web_debug = web_debug - self.web_port = web_port - self.web_iface = web_iface + self.add_option( + "web_open_browser", bool, True, + "Start a browser." + ) + self.add_option( + "web_debug", bool, False, + "Mitmweb debugging." + ) + self.add_option( + "web_port", int, 8081, + "Mitmweb port." + ) + self.add_option( + "web_iface", str, "127.0.0.1", + "Mitmweb interface." + ) # Dump options - self.filtstr = filtstr - self.flow_detail = flow_detail + self.add_option( + "filtstr", Optional[str], None, + "The filter string for mitmdump." + ) + self.add_option( + "flow_detail", int, 1, + "Flow detail display level." + ) - super().__init__() + self.update(**kwargs) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index f95ce836..8661aece 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -1,37 +1,84 @@ import contextlib import blinker import pprint -import inspect import copy import functools import weakref import os +import typing +import textwrap import ruamel.yaml from mitmproxy import exceptions from mitmproxy.utils import typecheck - """ The base implementation for Options. """ +unset = object() + + +class _Option: + __slots__ = ("name", "typespec", "value", "_default", "choices", "help") + + def __init__( + self, + name: str, + typespec: type, + default: typing.Any, + help: str, + choices: typing.Optional[typing.Sequence[str]] + ) -> None: + typecheck.check_type(name, default, typespec) + self.name = name + self.typespec = typespec + self._default = default + self.value = unset + self.help = help + self.choices = choices + + def __repr__(self): + return "{value} [{type}]".format(value=self.current(), type=self.typespec) + + @property + def default(self): + return copy.deepcopy(self._default) + + def current(self) -> typing.Any: + if self.value is unset: + v = self.default + else: + v = self.value + return copy.deepcopy(v) + + def set(self, value: typing.Any) -> None: + typecheck.check_type(self.name, value, self.typespec) + self.value = value -class _DefaultsMeta(type): - def __new__(cls, name, bases, namespace, **kwds): - ret = type.__new__(cls, name, bases, dict(namespace)) - defaults = {} - for klass in reversed(inspect.getmro(ret)): - for p in inspect.signature(klass.__init__).parameters.values(): - if p.kind in (p.KEYWORD_ONLY, p.POSITIONAL_OR_KEYWORD): - if not p.default == p.empty: - defaults[p.name] = p.default - ret._defaults = defaults - return ret + def reset(self) -> None: + self.value = unset + def has_changed(self) -> bool: + return self.value is not unset -class OptManager(metaclass=_DefaultsMeta): + def __eq__(self, other) -> bool: + for i in self.__slots__: + if getattr(self, i) != getattr(other, i): + return False + return True + + def __deepcopy__(self, _): + o = _Option( + self.name, self.typespec, self.default, self.help, self.choices + ) + if self.has_changed(): + o.value = self.current() + return o + + +class OptManager: """ OptManager is the base class from which Options objects are derived. Note that the __init__ method of all child classes must force all @@ -45,33 +92,37 @@ class OptManager(metaclass=_DefaultsMeta): Optmanager always returns a deep copy of options to ensure that mutation doesn't change the option state inadvertently. """ - _initialized = False - attributes = [] - - def __new__(cls, *args, **kwargs): - # Initialize instance._opts before __init__ is called. - # This allows us to call super().__init__() last, which then sets - # ._initialized = True as the final operation. - instance = super().__new__(cls) - instance.__dict__["_opts"] = {} - return instance - def __init__(self): + self.__dict__["_options"] = {} self.__dict__["changed"] = blinker.Signal() self.__dict__["errored"] = blinker.Signal() - self.__dict__["_initialized"] = True + self.__dict__["_processed"] = {} + + def add_option( + self, + name: str, + typespec: type, + default: typing.Any, + help: str, + choices: typing.Optional[typing.Sequence[str]] = None + ) -> None: + if name in self._options: + raise ValueError("Option %s already exists" % name) + self._options[name] = _Option(name, typespec, default, help, choices) @contextlib.contextmanager - def rollback(self, updated): - old = self._opts.copy() + def rollback(self, updated, reraise=False): + old = copy.deepcopy(self._options) try: yield except exceptions.OptionsError as e: # Notify error handlers self.errored.send(self, exc=e) # Rollback - self.__dict__["_opts"] = old + self.__dict__["_options"] = old self.changed.send(self, updated=updated) + if reraise: + raise e def subscribe(self, func, opts): """ @@ -95,61 +146,51 @@ class OptManager(metaclass=_DefaultsMeta): self.changed.connect(_call, weak=False) def __eq__(self, other): - return self._opts == other._opts + return self._options == other._options def __copy__(self): - return self.__class__(**self._opts) + o = OptManager() + o.__dict__["_options"] = copy.deepcopy(self._options) + return o def __getattr__(self, attr): - if attr in self._opts: - return copy.deepcopy(self._opts[attr]) + if attr in self._options: + return self._options[attr].current() else: raise AttributeError("No such option: %s" % attr) def __setattr__(self, attr, value): - if not self._initialized: - self._typecheck(attr, value) - self._opts[attr] = value - return self.update(**{attr: value}) - def _typecheck(self, attr, value): - expected_type = typecheck.get_arg_type_from_constructor_annotation( - type(self), attr - ) - if expected_type is None: - return # no type info :( - typecheck.check_type(attr, value, expected_type) - def keys(self): - return set(self._opts.keys()) + return set(self._options.keys()) + + def __contains__(self, k): + return k in self._options def reset(self): """ Restore defaults for all options. """ - self.update(**self._defaults) - - @classmethod - def default(klass, opt): - return copy.deepcopy(klass._defaults[opt]) + for o in self._options.values(): + o.reset() def update(self, **kwargs): updated = set(kwargs.keys()) - for k, v in kwargs.items(): - if k not in self._opts: - raise KeyError("No such option: %s" % k) - self._typecheck(k, v) with self.rollback(updated): - self._opts.update(kwargs) + for k, v in kwargs.items(): + if k not in self._options: + raise KeyError("No such option: %s" % k) + self._options[k].set(v) self.changed.send(self, updated=updated) + return self def setter(self, attr): """ Generate a setter for a given attribute. This returns a callable taking a single argument. """ - if attr not in self._opts: + if attr not in self._options: raise KeyError("No such option: %s" % attr) def setter(x): @@ -161,19 +202,24 @@ class OptManager(metaclass=_DefaultsMeta): Generate a toggler for a boolean attribute. This returns a callable that takes no arguments. """ - if attr not in self._opts: + if attr not in self._options: raise KeyError("No such option: %s" % attr) + o = self._options[attr] + if o.typespec != bool: + raise ValueError("Toggler can only be used with boolean options") def toggle(): setattr(self, attr, not getattr(self, attr)) return toggle + def default(self, option: str) -> typing.Any: + return self._options[option].default + def has_changed(self, option): """ Has the option changed from the default? """ - if getattr(self, option) != self._defaults[option]: - return True + return self._options[option].has_changed() def save(self, path, defaults=False): """ @@ -204,7 +250,7 @@ class OptManager(metaclass=_DefaultsMeta): if defaults or self.has_changed(k): data[k] = getattr(self, k) for k in list(data.keys()): - if k not in self._opts: + if k not in self._options: del data[k] return ruamel.yaml.round_trip_dump(data) @@ -268,7 +314,7 @@ class OptManager(metaclass=_DefaultsMeta): self.update(**toset) def __repr__(self): - options = pprint.pformat(self._opts, indent=4).strip(" {}") + options = pprint.pformat(self._options, indent=4).strip(" {}") if "\n" in options: options = "\n " + options + "\n" return "{mod}.{cls}({{{options}}})".format( @@ -276,3 +322,141 @@ class OptManager(metaclass=_DefaultsMeta): cls=type(self).__name__, options=options ) + + def set(self, spec): + parts = spec.split("=", maxsplit=1) + if len(parts) == 1: + optname, optval = parts[0], None + else: + optname, optval = parts[0], parts[1] + o = self._options[optname] + + if o.typespec in (str, typing.Optional[str]): + setattr(self, optname, optval) + elif o.typespec in (int, typing.Optional[int]): + if optval: + try: + optval = int(optval) + except ValueError: + raise exceptions.OptionsError("Not an integer: %s" % optval) + setattr(self, optname, optval) + elif o.typespec == bool: + if not optval or optval == "true": + v = True + elif optval == "false": + v = False + else: + raise exceptions.OptionsError( + "Boolean must be \"true\", \"false\", or have the value " "omitted (a synonym for \"true\")." + ) + setattr(self, optname, v) + elif o.typespec == typing.Sequence[str]: + if not optval: + setattr(self, optname, []) + else: + setattr( + self, + optname, + getattr(self, optname) + [optval] + ) + else: # pragma: no cover + raise NotImplementedError("Unsupported option type: %s", o.typespec) + + def make_parser(self, parser, optname, metavar=None, short=None): + o = self._options[optname] + + def mkf(l, s): + l = l.replace("_", "-") + f = ["--%s" % l] + if s: + f.append("-" + s) + return f + + flags = mkf(optname, short) + + if o.typespec == bool: + g = parser.add_mutually_exclusive_group(required=False) + onf = mkf(optname, None) + offf = mkf("no-" + optname, None) + # The short option for a bool goes to whatever is NOT the default + if short: + if o.default: + offf = mkf("no-" + optname, short) + else: + onf = mkf(optname, short) + g.add_argument( + *offf, + action="store_false", + dest=optname, + ) + g.add_argument( + *onf, + action="store_true", + dest=optname, + help=o.help + ) + parser.set_defaults(**{optname: None}) + elif o.typespec in (int, typing.Optional[int]): + parser.add_argument( + *flags, + action="store", + type=int, + dest=optname, + help=o.help, + metavar=metavar, + ) + elif o.typespec in (str, typing.Optional[str]): + parser.add_argument( + *flags, + action="store", + type=str, + dest=optname, + help=o.help, + metavar=metavar, + choices=o.choices + ) + elif o.typespec == typing.Sequence[str]: + parser.add_argument( + *flags, + action="append", + type=str, + dest=optname, + help=o.help + " May be passed multiple times.", + metavar=metavar, + choices=o.choices, + ) + else: + raise ValueError("Unsupported option type: %s", o.typespec) + + +def dump(opts): + """ + Dumps an annotated file with all options. + """ + # Sort data + s = ruamel.yaml.comments.CommentedMap() + for k in sorted(opts.keys()): + o = opts._options[k] + s[k] = o.default + txt = o.help.strip() + + if o.choices: + txt += " Valid values are %s." % ", ".join(repr(c) for c in o.choices) + else: + if o.typespec in (str, int, bool): + t = o.typespec.__name__ + elif o.typespec == typing.Optional[str]: + t = "optional str" + elif o.typespec == typing.Sequence[str]: + t = "sequence of str" + else: # pragma: no cover + raise NotImplementedError + txt += " Type %s." % t + + txt = "\n".join( + textwrap.wrap( + textwrap.dedent(txt) + ) + ) + s.yaml_set_comment_before_after_key(k, before = "\n" + txt) + return ruamel.yaml.round_trip_dump(s) diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index ea2f7c7f..8417ebad 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -1,4 +1,3 @@ -import collections import os import re from typing import Any @@ -9,7 +8,7 @@ from mitmproxy import exceptions from mitmproxy import options as moptions from mitmproxy import certs from mitmproxy.net import tcp -from mitmproxy.net.http import url +from mitmproxy.net import server_spec CONF_BASENAME = "mitmproxy" @@ -33,24 +32,6 @@ class HostMatcher: return bool(self.patterns) -ServerSpec = collections.namedtuple("ServerSpec", "scheme address") - - -def parse_server_spec(spec): - try: - p = url.parse(spec) - if p[0] not in (b"http", b"https"): - raise ValueError() - except ValueError: - raise exceptions.OptionsError( - "Invalid server specification: %s" % spec - ) - host, port = p[1:3] - address = (host.decode("ascii"), port) - scheme = p[0].decode("ascii").lower() - return ServerSpec(scheme, address) - - class ProxyConfig: def __init__(self, options: moptions.Options) -> None: @@ -59,7 +40,7 @@ class ProxyConfig: self.check_ignore = None self.check_tcp = None self.certstore = None - self.clientcerts = None + self.client_certs = None self.openssl_verification_mode_server = None self.configure(options, set(options.keys())) options.changed.connect(self.configure) @@ -96,28 +77,32 @@ class ProxyConfig: CONF_BASENAME ) - if options.clientcerts: - clientcerts = os.path.expanduser(options.clientcerts) - if not os.path.exists(clientcerts): + if options.client_certs: + client_certs = os.path.expanduser(options.client_certs) + if not os.path.exists(client_certs): raise exceptions.OptionsError( "Client certificate path does not exist: %s" % - options.clientcerts + options.client_certs ) - self.clientcerts = clientcerts + self.client_certs = client_certs + + for c in options.certs: + parts = c.split("=", 1) + if len(parts) == 1: + parts = ["*", parts[0]] - for spec, cert in options.certs: - cert = os.path.expanduser(cert) + cert = os.path.expanduser(parts[1]) if not os.path.exists(cert): raise exceptions.OptionsError( "Certificate file does not exist: %s" % cert ) try: - self.certstore.add_cert_file(spec, cert) + self.certstore.add_cert_file(parts[0], cert) except crypto.Error: raise exceptions.OptionsError( "Invalid certificate format: %s" % cert ) - - self.upstream_server = None - if options.upstream_server: - self.upstream_server = parse_server_spec(options.upstream_server) + m = options.mode + if m.startswith("upstream:") or m.startswith("reverse:"): + _, spec = server_spec.parse_with_mode(options.mode) + self.upstream_server = spec diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index d2f6d374..d9e53fed 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -290,7 +290,7 @@ class HttpLayer(base.Layer): request.first_line_format = "relative" # update host header in reverse proxy mode - if self.config.options.mode == "reverse" and not self.config.options.keep_host_header: + if self.config.options.mode.startswith("reverse:") and not self.config.options.keep_host_header: f.request.host_header = self.config.upstream_server.address[0] # Determine .scheme, .host and .port attributes for inline scripts. For diff --git a/mitmproxy/proxy/protocol/http1.py b/mitmproxy/proxy/protocol/http1.py index b1fd0ecd..cafc2682 100644 --- a/mitmproxy/proxy/protocol/http1.py +++ b/mitmproxy/proxy/protocol/http1.py @@ -19,7 +19,7 @@ class Http1Layer(httpbase._HttpTransmissionLayer): return http1.read_body( self.client_conn.rfile, expected_size, - self.config.options.body_size_limit + self.config.options._processed.get("body_size_limit") ) def send_request(self, request): @@ -35,7 +35,7 @@ class Http1Layer(httpbase._HttpTransmissionLayer): return http1.read_body( self.server_conn.rfile, expected_size, - self.config.options.body_size_limit + self.config.options._processed.get("body_size_limit") ) def send_response_headers(self, response): diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index 01406798..a6e8a4dd 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -183,7 +183,7 @@ class Http2Layer(base.Layer): return True def _handle_data_received(self, eid, event, source_conn): - bsl = self.config.options.body_size_limit + bsl = self.config.options._processed.get("body_size_limit") if bsl and self.streams[eid].queued_data_length > bsl: self.streams[eid].kill() self.connections[source_conn].safe_reset_stream( diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py index 38f511c4..25867871 100644 --- a/mitmproxy/proxy/protocol/http_replay.py +++ b/mitmproxy/proxy/protocol/http_replay.py @@ -31,6 +31,7 @@ class RequestReplayThread(basethread.BaseThread): def run(self): r = self.f.request + bsl = self.config.options._processed.get("body_size_limit") first_line_format_backup = r.first_line_format server = None try: @@ -44,7 +45,7 @@ class RequestReplayThread(basethread.BaseThread): if not self.f.response: # In all modes, we directly connect to the server displayed - if self.config.options.mode == "upstream": + if self.config.options.mode.startswith("upstream:"): server_address = self.config.upstream_server.address server = connections.ServerConnection(server_address, (self.config.options.listen_host, 0)) server.connect() @@ -55,12 +56,12 @@ class RequestReplayThread(basethread.BaseThread): resp = http1.read_response( server.rfile, connect_request, - body_size_limit=self.config.options.body_size_limit + body_size_limit=bsl ) if resp.status_code != 200: raise exceptions.ReplayException("Upstream server refuses CONNECT request") server.establish_ssl( - self.config.clientcerts, + self.config.client_certs, sni=self.f.server_conn.sni ) r.first_line_format = "relative" @@ -75,7 +76,7 @@ class RequestReplayThread(basethread.BaseThread): server.connect() if r.scheme == "https": server.establish_ssl( - self.config.clientcerts, + self.config.client_certs, sni=self.f.server_conn.sni ) r.first_line_format = "relative" @@ -87,7 +88,7 @@ class RequestReplayThread(basethread.BaseThread): http1.read_response( server.rfile, r, - body_size_limit=self.config.options.body_size_limit + body_size_limit=bsl ) ) if self.channel: diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index 7d15130f..acc0c6e3 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -358,7 +358,7 @@ class TlsLayer(base.Layer): # 2.5 The client did not sent a SNI value, we don't know the certificate subject. client_tls_requires_server_connection = ( self._server_tls and - not self.config.options.no_upstream_cert and + self.config.options.upstream_cert and ( self.config.options.add_upstream_certs_to_client_chain or self._client_tls and ( @@ -527,7 +527,7 @@ class TlsLayer(base.Layer): ciphers_server = ':'.join(ciphers_server) self.server_conn.establish_ssl( - self.config.clientcerts, + self.config.client_certs, self.server_sni, method=self.config.openssl_method_server, options=self.config.openssl_options_server, @@ -574,7 +574,7 @@ class TlsLayer(base.Layer): use_upstream_cert = ( self.server_conn and self.server_conn.tls_established and - (not self.config.options.no_upstream_cert) + self.config.options.upstream_cert ) if use_upstream_cert: upstream_cert = self.server_conn.cert diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 8082cb64..16692234 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -85,14 +85,14 @@ class ConnectionHandler: ) mode = self.config.options.mode - if mode == "upstream": + if mode.startswith("upstream:"): return modes.HttpUpstreamProxy( root_ctx, self.config.upstream_server.address ) elif mode == "transparent": return modes.TransparentProxy(root_ctx) - elif mode == "reverse": + elif mode.startswith("reverse:"): server_tls = self.config.upstream_server.scheme == "https" return modes.ReverseProxy( root_ctx, diff --git a/mitmproxy/test/taddons.py b/mitmproxy/test/taddons.py index bb8daa02..8d6baa12 100644 --- a/mitmproxy/test/taddons.py +++ b/mitmproxy/test/taddons.py @@ -4,7 +4,6 @@ import mitmproxy.master import mitmproxy.options from mitmproxy import proxy from mitmproxy import eventsequence -from mitmproxy import exceptions class RecordingMaster(mitmproxy.master.Master): @@ -43,14 +42,6 @@ class context: return False @contextlib.contextmanager - def _rollback(self, opts, updates): - old = opts._opts.copy() - try: - yield - except exceptions.OptionsError as e: - opts.__dict__["_opts"] = old - raise - def cycle(self, addon, f): """ Cycles the flow through the events for the flow. Stops if a reply @@ -70,6 +61,6 @@ class context: Options object with the given keyword arguments, then calls the configure method on the addon with the updated value. """ - with self._rollback(self.options, kwargs): + with self.options.rollback(kwargs.keys(), reraise=True): self.options.update(**kwargs) addon.configure(self.options, kwargs.keys()) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 11558cc3..aaefd10a 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -1,168 +1,14 @@ import argparse import os -from mitmproxy import exceptions from mitmproxy import options -from mitmproxy import platform -from mitmproxy.utils import human -from mitmproxy.net import tcp from mitmproxy import version -from mitmproxy.addons import view CONFIG_PATH = os.path.join(options.CA_DIR, "config.yaml") -class ParseException(Exception): - pass - - -def get_common_options(args): - stickycookie, stickyauth = None, None - if args.stickycookie_filt: - stickycookie = args.stickycookie_filt - - if args.stickyauth_filt: - stickyauth = args.stickyauth_filt - - stream_large_bodies = args.stream_large_bodies - if stream_large_bodies: - stream_large_bodies = human.parse_size(stream_large_bodies) - - if args.streamfile and args.streamfile[0] == args.rfile: - if args.streamfile[1] == "wb": - raise exceptions.OptionsError( - "Cannot use '{}' for both reading and writing flows. " - "Are you looking for --afile?".format(args.rfile) - ) - else: - raise exceptions.OptionsError( - "Cannot use '{}' for both reading and appending flows. " - "That would trigger an infinite loop." - ) - - # Proxy config - certs = [] - for i in args.certs or []: - parts = i.split("=", 1) - if len(parts) == 1: - parts = ["*", parts[0]] - certs.append(parts) - - body_size_limit = args.body_size_limit - if body_size_limit: - try: - body_size_limit = human.parse_size(body_size_limit) - except ValueError as e: - raise exceptions.OptionsError( - "Invalid body size limit specification: %s" % body_size_limit - ) - - # Establish proxy mode - c = 0 - mode, upstream_server = "regular", None - if args.transparent_proxy: - c += 1 - if not platform.original_addr: - raise exceptions.OptionsError( - "Transparent mode not supported on this platform." - ) - mode = "transparent" - if args.socks_proxy: - c += 1 - mode = "socks5" - if args.reverse_proxy: - c += 1 - mode = "reverse" - upstream_server = args.reverse_proxy - if args.upstream_proxy: - c += 1 - mode = "upstream" - upstream_server = args.upstream_proxy - if c > 1: - raise exceptions.OptionsError( - "Transparent, SOCKS5, reverse and upstream proxy mode " - "are mutually exclusive. Read the docs on proxy modes " - "to understand why." - ) - if args.add_upstream_certs_to_client_chain and args.no_upstream_cert: - raise exceptions.OptionsError( - "The no-upstream-cert and add-upstream-certs-to-client-chain " - "options are mutually exclusive. If no-upstream-cert is enabled " - "then the upstream certificate is not retrieved before generating " - "the client certificate chain." - ) - - if args.quiet: - args.verbose = 0 - - return dict( - onboarding=args.onboarding, - onboarding_host=args.onboarding_host, - onboarding_port=args.onboarding_port, - - anticache=args.anticache, - anticomp=args.anticomp, - client_replay=args.client_replay, - replay_kill_extra=args.replay_kill_extra, - no_server=args.no_server, - refresh_server_playback=not args.norefresh, - server_replay_use_headers=args.server_replay_use_headers, - rfile=args.rfile, - replacements=args.replacements, - replacement_files=args.replacement_files, - setheaders=args.setheaders, - keep_host_header=args.keep_host_header, - server_replay=args.server_replay, - scripts=args.scripts, - stickycookie=stickycookie, - stickyauth=stickyauth, - stream_large_bodies=stream_large_bodies, - showhost=args.showhost, - streamfile=args.streamfile[0] if args.streamfile else None, - streamfile_append=True if args.streamfile and args.streamfile[1] == "a" else False, - verbosity=args.verbose, - server_replay_nopop=args.server_replay_nopop, - server_replay_ignore_content=args.server_replay_ignore_content, - server_replay_ignore_params=args.server_replay_ignore_params, - server_replay_ignore_payload_params=args.server_replay_ignore_payload_params, - server_replay_ignore_host=args.server_replay_ignore_host, - - auth_nonanonymous = args.auth_nonanonymous, - auth_singleuser = args.auth_singleuser, - auth_htpasswd = args.auth_htpasswd, - add_upstream_certs_to_client_chain = args.add_upstream_certs_to_client_chain, - body_size_limit = body_size_limit, - cadir = args.cadir, - certs = certs, - ciphers_client = args.ciphers_client, - ciphers_server = args.ciphers_server, - clientcerts = args.clientcerts, - ignore_hosts = args.ignore_hosts, - listen_host = args.addr, - listen_port = args.port, - upstream_bind_address = args.upstream_bind_address, - mode = mode, - no_upstream_cert = args.no_upstream_cert, - spoof_source_address = args.spoof_source_address, - - http2 = args.http2, - http2_priority = args.http2_priority, - websocket = args.websocket, - rawtcp = args.rawtcp, - - upstream_server = upstream_server, - upstream_auth = args.upstream_auth, - ssl_version_client = args.ssl_version_client, - ssl_version_server = args.ssl_version_server, - ssl_insecure = args.ssl_insecure, - ssl_verify_upstream_trusted_cadir = args.ssl_verify_upstream_trusted_cadir, - ssl_verify_upstream_trusted_ca = args.ssl_verify_upstream_trusted_ca, - tcp_hosts = args.tcp_hosts, - ) - - -def basic_options(parser): +def common_options(parser, opts): parser.add_argument( '--version', action='store_true', @@ -175,22 +21,26 @@ def basic_options(parser): version=version.VERSION ) parser.add_argument( - "--anticache", - action="store_true", dest="anticache", - help=""" - Strip out request headers that might cause the server to return - 304-not-modified. - """ + '--options', + action='store_true', + help="Dump all options", ) parser.add_argument( - "--cadir", - action="store", type=str, dest="cadir", - help="Location of the default mitmproxy CA files. (%s)" % options.CA_DIR + "--conf", + type=str, dest="conf", default=CONFIG_PATH, + metavar="PATH", + help="Read options from a configuration file" ) parser.add_argument( - "--host", - action="store_true", dest="showhost", - help="Use the Host header to construct URLs for display." + "--set", + type=str, dest="setoptions", default=[], + action="append", + metavar="option[=value]", + help=""" + Set an option. When the value is omitted, booleans are set to true, + strings and integers are set to None (if permitted), and sequences + are emptied. + """ ) parser.add_argument( "-q", "--quiet", @@ -198,591 +48,100 @@ def basic_options(parser): help="Quiet." ) parser.add_argument( - "-r", "--read-flows", - action="store", dest="rfile", - help="Read flows from file." - ) - parser.add_argument( - "-s", "--script", - action="append", type=str, dest="scripts", - metavar='"script.py --bar"', - help=""" - Run a script. Surround with quotes to pass script arguments. Can be - passed multiple times. - """ - ) - parser.add_argument( - "-t", "--stickycookie", - action="store", - dest="stickycookie_filt", - metavar="FILTER", - help="Set sticky cookie filter. Matched against requests." - ) - parser.add_argument( - "-u", "--stickyauth", - action="store", dest="stickyauth_filt", metavar="FILTER", - help="Set sticky auth filter. Matched against requests." - ) - parser.add_argument( "-v", "--verbose", action="store_const", dest="verbose", const=3, help="Increase log verbosity." ) - streamfile = parser.add_mutually_exclusive_group() - streamfile.add_argument( - "-w", "--wfile", - action="store", dest="streamfile", type=lambda f: (f, "w"), - help="Write flows to file." - ) - streamfile.add_argument( - "-a", "--afile", - action="store", dest="streamfile", type=lambda f: (f, "a"), - help="Append flows to file." - ) - parser.add_argument( - "-z", "--anticomp", - action="store_true", dest="anticomp", - help="Try to convince servers to send us un-compressed data." - ) - parser.add_argument( - "-Z", "--body-size-limit", - action="store", dest="body_size_limit", - metavar="SIZE", - help="Byte size limit of HTTP request and response bodies." - " Understands k/m/g suffixes, i.e. 3m for 3 megabytes." - ) - parser.add_argument( - "--stream", - action="store", dest="stream_large_bodies", - metavar="SIZE", - help=""" - Stream data to the client if response body exceeds the given - threshold. If streamed, the body will not be stored in any way. - Understands k/m/g suffixes, i.e. 3m for 3 megabytes. - """ - ) + # Basic options + opts.make_parser(parser, "mode", short="m") + opts.make_parser(parser, "anticache") + opts.make_parser(parser, "showhost") + opts.make_parser(parser, "rfile", metavar="PATH", short="r") + opts.make_parser(parser, "scripts", metavar="SCRIPT", short="s") + opts.make_parser(parser, "stickycookie", metavar="FILTER") + opts.make_parser(parser, "stickyauth", metavar="FILTER") + opts.make_parser(parser, "streamfile", metavar="PATH", short="w") + opts.make_parser(parser, "anticomp") -def proxy_modes(parser): - group = parser.add_argument_group("Proxy Modes") - group.add_argument( - "-R", "--reverse", - action="store", - type=str, - dest="reverse_proxy", - help=""" - Forward all requests to upstream HTTP server: - http[s]://host[:port]. Clients can always connect both - via HTTPS and HTTP, the connection to the server is - determined by the specified scheme. - """ - ) - group.add_argument( - "--socks", - action="store_true", dest="socks_proxy", - help="Set SOCKS5 proxy mode." - ) - group.add_argument( - "-T", "--transparent", - action="store_true", dest="transparent_proxy", - help="Set transparent proxy mode." - ) - group.add_argument( - "-U", "--upstream", - action="store", - type=str, - dest="upstream_proxy", - help="Forward all requests to upstream proxy server: http://host[:port]" - ) - - -def proxy_options(parser): + # Proxy options group = parser.add_argument_group("Proxy Options") - group.add_argument( - "-b", "--bind-address", - action="store", type=str, dest="addr", - help="Address to bind proxy to (defaults to all interfaces)" - ) - group.add_argument( - "-I", "--ignore", - action="append", type=str, dest="ignore_hosts", - metavar="HOST", - help=""" - Ignore host and forward all traffic without processing it. In - transparent mode, it is recommended to use an IP address (range), - not the hostname. In regular mode, only SSL traffic is ignored and - the hostname should be used. The supplied value is interpreted as a - regular expression and matched on the ip or the hostname. Can be - passed multiple times. - """ - ) - group.add_argument( - "--tcp", - action="append", type=str, dest="tcp_hosts", - metavar="HOST", - help=""" - Generic TCP SSL proxy mode for all hosts that match the pattern. - Similar to --ignore, but SSL connections are intercepted. The - communication contents are printed to the log in verbose mode. - """ - ) - group.add_argument( - "-n", "--no-server", - action="store_true", dest="no_server", - help="Don't start a proxy server." - ) - group.add_argument( - "-p", "--port", - action="store", type=int, dest="port", - help="Proxy service port." - ) - - http2 = group.add_mutually_exclusive_group() - http2.add_argument("--no-http2", action="store_false", dest="http2") - http2.add_argument("--http2", action="store_true", dest="http2", - help="Explicitly enable/disable HTTP/2 support. " - "HTTP/2 support is enabled by default.", - ) - - http2_priority = group.add_mutually_exclusive_group() - http2_priority.add_argument("--http2-priority", action="store_true", dest="http2_priority") - http2_priority.add_argument("--no-http2-priority", action="store_false", dest="http2_priority", - help="Explicitly enable/disable PRIORITY forwarding for HTTP/2 connections. " - "PRIORITY forwarding is disabled by default, " - "because some webservers fail at implementing the RFC properly.", - ) - - websocket = group.add_mutually_exclusive_group() - websocket.add_argument("--no-websocket", action="store_false", dest="websocket") - websocket.add_argument("--websocket", action="store_true", dest="websocket", - help="Explicitly enable/disable WebSocket support. " - "WebSocket support is enabled by default.", - ) - - parser.add_argument( - "--upstream-auth", - action="store", dest="upstream_auth", - type=str, - help=""" - Add HTTP Basic authentcation to upstream proxy and reverse proxy - requests. Format: username:password - """ - ) - - rawtcp = group.add_mutually_exclusive_group() - rawtcp.add_argument("--raw-tcp", action="store_true", dest="rawtcp") - rawtcp.add_argument("--no-raw-tcp", action="store_false", dest="rawtcp", - help="Explicitly enable/disable experimental raw tcp support. " - "Disabled by default. " - "Default value will change in a future version." - ) - - group.add_argument( - "--spoof-source-address", - action="store_true", dest="spoof_source_address", - help="Use the client's IP for server-side connections. " - "Combine with --upstream-bind-address to spoof a fixed source address." - ) - group.add_argument( - "--upstream-bind-address", - action="store", type=str, dest="upstream_bind_address", - help="Address to bind upstream requests to (defaults to none)" - ) - group.add_argument( - "--keep-host-header", - action="store_true", dest="keep_host_header", - help="Reverse Proxy: Keep the original host header instead of rewriting it to the reverse proxy target." - ) - - -def proxy_ssl_options(parser): - # TODO: Agree to consistently either use "upstream" or "server". + opts.make_parser(group, "listen_host", metavar="HOST") + opts.make_parser(group, "listen_port", metavar="PORT", short="p") + opts.make_parser(group, "server", short="n") + opts.make_parser(group, "ignore_hosts", metavar="HOST") + opts.make_parser(group, "tcp_hosts", metavar="HOST") + opts.make_parser(group, "upstream_auth", metavar="USER:PASS") + opts.make_parser(group, "proxyauth", metavar="SPEC") + opts.make_parser(group, "rawtcp") + + # Proxy SSL options group = parser.add_argument_group("SSL") - group.add_argument( - "--cert", - dest='certs', - type=str, - metavar="SPEC", - action="append", - help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' - 'The domain may include a wildcard, and is equal to "*" if not specified. ' - 'The file at path is a certificate in PEM format. If a private key is included ' - 'in the PEM, it is used, else the default key in the conf dir is used. ' - 'The PEM file should contain the full certificate chain, with the leaf certificate ' - 'as the first entry. Can be passed multiple times.') - group.add_argument( - "--ciphers-client", action="store", - type=str, dest="ciphers_client", - help="Set supported ciphers for client connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--ciphers-server", action="store", - type=str, dest="ciphers_server", - help="Set supported ciphers for server connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--client-certs", action="store", - type=str, dest="clientcerts", - help="Client certificate file or directory." - ) - group.add_argument( - "--no-upstream-cert", - action="store_true", dest="no_upstream_cert", - help="Don't connect to upstream server to look up certificate details." - ) - group.add_argument( - "--add-upstream-certs-to-client-chain", - action="store_true", dest="add_upstream_certs_to_client_chain", - help="Add all certificates of the upstream server to the certificate chain " - "that will be served to the proxy client, as extras." - ) - group.add_argument( - "--insecure", - action="store_true", dest="ssl_insecure", - help="Do not verify upstream server SSL/TLS certificates." - ) - group.add_argument( - "--upstream-trusted-cadir", action="store", - dest="ssl_verify_upstream_trusted_cadir", - help="Path to a directory of trusted CA certificates for upstream " - "server verification prepared using the c_rehash tool." - ) - group.add_argument( - "--upstream-trusted-ca", action="store", - dest="ssl_verify_upstream_trusted_ca", - help="Path to a PEM formatted trusted CA certificate." - ) - group.add_argument( - "--ssl-version-client", dest="ssl_version_client", - action="store", - choices=tcp.sslversion_choices.keys(), - help="Set supported SSL/TLS versions for client connections. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." - ) - group.add_argument( - "--ssl-version-server", dest="ssl_version_server", - action="store", - choices=tcp.sslversion_choices.keys(), - help="Set supported SSL/TLS versions for server connections. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." - ) - + opts.make_parser(group, "certs", metavar="SPEC") + opts.make_parser(group, "ssl_insecure", short="k") -def onboarding_app(parser): - group = parser.add_argument_group("Onboarding App") - group.add_argument( - "--no-onboarding", - action="store_false", dest="onboarding", - help="Disable the mitmproxy onboarding app." - ) - group.add_argument( - "--onboarding-host", - action="store", dest="onboarding_host", - help=""" - Domain to serve the onboarding app from. For transparent mode, use - an IP when a DNS entry for the app domain is not present. Default: - %s - """ % options.APP_HOST - ) - group.add_argument( - "--onboarding-port", - action="store", - dest="onboarding_port", - type=int, - metavar="80", - help="Port to serve the onboarding app from." - ) - - -def client_replay(parser): + # Client replay group = parser.add_argument_group("Client Replay") - group.add_argument( - "-c", "--client-replay", - action="append", dest="client_replay", metavar="PATH", - help="Replay client requests from a saved file." - ) - + opts.make_parser(group, "client_replay", metavar="PATH", short="C") -def server_replay(parser): + # Server replay group = parser.add_argument_group("Server Replay") - group.add_argument( - "-S", "--server-replay", - action="append", dest="server_replay", metavar="PATH", - help="Replay server responses from a saved file." - ) - group.add_argument( - "-k", "--replay-kill-extra", - action="store_true", dest="replay_kill_extra", - help="Kill extra requests during replay." - ) - group.add_argument( - "--server-replay-use-header", - action="append", dest="server_replay_use_headers", type=str, - help="Request headers to be considered during replay. " - "Can be passed multiple times." - ) - group.add_argument( - "--norefresh", - action="store_true", dest="norefresh", - help=""" - Disable response refresh, which updates times in cookies and headers - for replayed responses. - """ - ) - group.add_argument( - "--no-pop", - action="store_true", dest="server_replay_nopop", - help="Disable response pop from response flow. " - "This makes it possible to replay same response multiple times." - ) - payload = group.add_mutually_exclusive_group() - payload.add_argument( - "--replay-ignore-content", - action="store_true", dest="server_replay_ignore_content", - help=""" - Ignore request's content while searching for a saved flow to replay - """ - ) - payload.add_argument( - "--replay-ignore-payload-param", - action="append", dest="server_replay_ignore_payload_params", type=str, - help=""" - Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to - be ignored while searching for a saved flow to replay. - Can be passed multiple times. - """ - ) - - group.add_argument( - "--replay-ignore-param", - action="append", dest="server_replay_ignore_params", type=str, - help=""" - Request's parameters to be ignored while searching for a saved flow - to replay. Can be passed multiple times. - """ - ) - group.add_argument( - "--replay-ignore-host", - action="store_true", - dest="server_replay_ignore_host", - help="Ignore request's destination host while searching for a saved flow to replay") - - -def replacements(parser): - group = parser.add_argument_group( - "Replacements", - """ - Replacements are of the form "/pattern/regex/replacement", where - the separator can be any character. Please see the documentation - for more information. - """.strip() - ) - group.add_argument( - "--replace", - action="append", type=str, dest="replacements", - metavar="PATTERN", - help="Replacement pattern." - ) - group.add_argument( - "--replace-from-file", - action="append", type=str, dest="replacement_files", - metavar="PATH", - help=""" - Replacement pattern, where the replacement clause is a path to a - file. - """ - ) - - -def set_headers(parser): - group = parser.add_argument_group( - "Set Headers", - """ - Header specifications are of the form "/pattern/header/value", - where the separator can be any character. Please see the - documentation for more information. - """.strip() - ) - group.add_argument( - "--setheader", - action="append", type=str, dest="setheaders", - metavar="PATTERN", - help="Header set pattern." - ) - - -def proxy_authentication(parser): - group = parser.add_argument_group( - "Proxy Authentication", - """ - Specify which users are allowed to access the proxy and the method - used for authenticating them. - """ - ).add_mutually_exclusive_group() - group.add_argument( - "--nonanonymous", - action="store_true", dest="auth_nonanonymous", - help="Allow access to any user long as a credentials are specified." - ) - - group.add_argument( - "--singleuser", - action="store", dest="auth_singleuser", type=str, - metavar="USER", - help=""" - Allows access to a a single user, specified in the form - username:password. - """ - ) - group.add_argument( - "--htpasswd", - action="store", dest="auth_htpasswd", type=str, - metavar="PATH", - help="Allow access to users specified in an Apache htpasswd file." - ) - - -def common_options(parser): - parser.add_argument( - "--conf", - type=str, dest="conf", default=CONFIG_PATH, - metavar="PATH", - help=""" - Configuration file - """ - ) + opts.make_parser(group, "server_replay", metavar="PATH", short="S") + opts.make_parser(group, "replay_kill_extra") + opts.make_parser(group, "server_replay_nopop") - basic_options(parser) - proxy_modes(parser) - proxy_options(parser) - proxy_ssl_options(parser) - onboarding_app(parser) - client_replay(parser) - server_replay(parser) - replacements(parser) - set_headers(parser) - proxy_authentication(parser) + # Replacements + group = parser.add_argument_group("Replacements") + opts.make_parser(group, "replacements", metavar="PATTERN", short="R") + opts.make_parser(group, "replacement_files", metavar="PATTERN") + # Set headers + group = parser.add_argument_group("Set Headers") + opts.make_parser(group, "setheaders", metavar="PATTERN", short="H") -def mitmproxy(): - # Don't import mitmproxy.tools.console for mitmdump, urwid is not available - # on all platforms. - from .console import palettes +def mitmproxy(opts): parser = argparse.ArgumentParser(usage="%(prog)s [options]") - common_options(parser) - parser.add_argument( - "--palette", type=str, - action="store", dest="console_palette", - choices=sorted(palettes.palettes.keys()), - help="Select color palette: " + ", ".join(palettes.palettes.keys()) - ) - parser.add_argument( - "--palette-transparent", - action="store_true", dest="console_palette_transparent", - help="Set transparent background for palette." - ) - parser.add_argument( - "-e", "--eventlog", - action="store_true", dest="console_eventlog", - help="Show event log." - ) - parser.add_argument( - "--follow", - action="store_true", dest="console_focus_follow", - help="Focus follows new flows." - ) - parser.add_argument( - "--order", - type=str, dest="console_order", - choices=[o[1] for o in view.orders], - help="Flow sort order." - ) - parser.add_argument( - "--no-mouse", - action="store_true", dest="console_no_mouse", - help="Disable mouse interaction." - ) + common_options(parser, opts) + + opts.make_parser(parser, "console_eventlog") group = parser.add_argument_group( "Filters", "See help in mitmproxy for filter expression syntax." ) - group.add_argument( - "-i", "--intercept", action="store", - type=str, dest="intercept", - help="Intercept filter expression." - ) - group.add_argument( - "-f", "--filter", action="store", - type=str, dest="filter", - help="Filter view expression." - ) + opts.make_parser(group, "intercept", metavar="FILTER") + opts.make_parser(group, "filter", metavar="FILTER") return parser -def mitmdump(): +def mitmdump(opts): parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]") - common_options(parser) + common_options(parser, opts) + opts.make_parser(parser, "flow_detail", metavar = "LEVEL") parser.add_argument( - "--keepserving", - action="store_true", dest="keepserving", - help=""" - Continue serving after client playback or file read. We exit by - default. - """ - ) - parser.add_argument( - "-d", "--detail", - action="count", dest="flow_detail", - help="Increase flow detail display level. Can be passed multiple times." - ) - parser.add_argument( - 'filter', + 'filter_args', nargs="...", help=""" - Filter view expression, used to only show flows that match a certain filter. - See help in mitmproxy for filter expression syntax. + Filter view expression, used to only show flows that match a certain + filter. See help in mitmproxy for filter expression syntax. """ ) return parser -def mitmweb(): +def mitmweb(opts): parser = argparse.ArgumentParser(usage="%(prog)s [options]") group = parser.add_argument_group("Mitmweb") - group.add_argument( - "--no-browser", - action="store_false", dest="web_open_browser", - help="Don't start a browser" - ) - group.add_argument( - "--web-port", - action="store", type=int, dest="web_port", - metavar="PORT", - help="Mitmweb port." - ) - group.add_argument( - "--web-iface", - action="store", dest="web_iface", - metavar="IFACE", - help="Mitmweb interface." - ) - group.add_argument( - "--web-debug", - action="store_true", dest="web_debug", - help="Turn on mitmweb debugging" - ) + opts.make_parser(group, "web_open_browser") + opts.make_parser(group, "web_port", metavar="PORT") + opts.make_parser(group, "web_iface", metavar="INTERFACE") - common_options(parser) + common_options(parser, opts) group = parser.add_argument_group( "Filters", "See help in mitmproxy for filter expression syntax." ) - group.add_argument( - "-i", "--intercept", action="store", - type=str, dest="intercept", - help="Intercept filter expression." - ) + opts.make_parser(group, "intercept", metavar="FILTER") return parser diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index d68dc93c..e75105cf 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -252,7 +252,7 @@ class ConsoleMaster(master.Master): self.loop = urwid.MainLoop( urwid.SolidFill("x"), screen = self.ui, - handle_mouse = not self.options.console_no_mouse, + handle_mouse = self.options.console_mouse, ) self.ab = statusbar.ActionBar() diff --git a/mitmproxy/tools/console/options.py b/mitmproxy/tools/console/options.py index 4115bd18..33e3ec38 100644 --- a/mitmproxy/tools/console/options.py +++ b/mitmproxy/tools/console/options.py @@ -90,10 +90,10 @@ class Options(urwid.WidgetWrap): select.Heading("Network"), select.Option( - "No Upstream Certs", + "Upstream Certs", "U", - checker("no_upstream_cert", master.options), - master.options.toggler("no_upstream_cert") + checker("upstream_cert", master.options), + master.options.toggler("upstream_cert") ), select.Option( "TCP Proxying", diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index d90d932b..3e524972 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -2,7 +2,6 @@ import os.path import urwid -import mitmproxy.net.http.url from mitmproxy.tools.console import common from mitmproxy.tools.console import pathedit from mitmproxy.tools.console import signals @@ -220,7 +219,7 @@ class StatusBar(urwid.WidgetWrap): opts.append("norefresh") if self.master.options.replay_kill_extra: opts.append("killextra") - if self.master.options.no_upstream_cert: + if not self.master.options.upstream_cert: opts.append("no-upstream-cert") if self.master.options.console_focus_follow: opts.append("following") @@ -234,13 +233,8 @@ class StatusBar(urwid.WidgetWrap): if opts: r.append("[%s]" % (":".join(opts))) - if self.master.options.mode in ["reverse", "upstream"]: - dst = self.master.server.config.upstream_server - r.append("[dest:%s]" % mitmproxy.net.http.url.unparse( - dst.scheme, - dst.address[0], - dst.address[1], - )) + if self.master.options.mode != "regular": + r.append("[%s]" % self.master.options.mode) if self.master.options.scripts: r.append("[") r.append(("heading_key", "s")) diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index fefbddfb..e70ce2f9 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -1,5 +1,3 @@ -from typing import Optional - from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import addons @@ -8,30 +6,11 @@ from mitmproxy import master from mitmproxy.addons import dumper, termlog -class DumpError(Exception): - pass - - -class Options(options.Options): - def __init__( - self, - *, # all args are keyword-only. - keepserving: bool = False, - filtstr: Optional[str] = None, - flow_detail: int = 1, - **kwargs - ) -> None: - self.filtstr = filtstr - self.flow_detail = flow_detail - self.keepserving = keepserving - super().__init__(**kwargs) - - class DumpMaster(master.Master): def __init__( self, - options: Options, + options: options.Options, server, with_termlog=True, with_dumper=True, @@ -44,7 +23,7 @@ class DumpMaster(master.Master): if with_dumper: self.addons.add(dumper.Dumper()) - if not self.options.no_server: + if self.options.server: self.add_log( "Proxy server listening at http://{}:{}".format(server.address[0], server.address[1]), "info" @@ -55,7 +34,7 @@ class DumpMaster(master.Master): self.load_flows_file(options.rfile) except exceptions.FlowReadException as v: self.add_log("Flow file corrupted.", "error") - raise DumpError(v) + raise exceptions.OptionsError(v) @controller.handler def log(self, e): diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index ce78cd13..17c1abbb 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -12,6 +12,7 @@ import signal # noqa from mitmproxy.tools import cmdline # noqa from mitmproxy import exceptions # noqa from mitmproxy import options # noqa +from mitmproxy import optmanager # noqa from mitmproxy.proxy import config # noqa from mitmproxy.proxy import server # noqa from mitmproxy.utils import version_check # noqa @@ -34,134 +35,80 @@ def assert_utf8_env(): sys.exit(1) -def process_options(parser, options, args): +def process_options(parser, opts, args): if args.version: print(debug.dump_system_info()) sys.exit(0) + if args.options: + print(optmanager.dump(opts)) + sys.exit(0) + if args.quiet: + args.flow_detail = 0 - debug.register_info_dumpers() - pconf = config.ProxyConfig(options) - if options.no_server: - return server.DummyServer(pconf) - else: + for i in args.setoptions: + opts.set(i) + + adict = {} + for n in dir(args): + if n in opts: + adict[n] = getattr(args, n) + opts.merge(adict) + + pconf = config.ProxyConfig(opts) + if opts.server: try: return server.ProxyServer(pconf) except exceptions.ServerException as v: print(str(v), file=sys.stderr) sys.exit(1) + else: + return server.DummyServer(pconf) -def mitmproxy(args=None): # pragma: no cover - if os.name == "nt": - print("Error: mitmproxy's console interface is not supported on Windows. " - "You can run mitmdump or mitmweb instead.", file=sys.stderr) - sys.exit(1) - from mitmproxy.tools import console - - version_check.check_pyopenssl_version() - assert_utf8_env() - - parser = cmdline.mitmproxy() - args = parser.parse_args(args) - - try: - console_options = options.Options() - console_options.load_paths(args.conf) - console_options.merge(cmdline.get_common_options(args)) - console_options.merge( - dict( - console_palette = args.console_palette, - console_palette_transparent = args.console_palette_transparent, - console_eventlog = args.console_eventlog, - console_focus_follow = args.console_focus_follow, - console_no_mouse = args.console_no_mouse, - console_order = args.console_order, - - filter = args.filter, - intercept = args.intercept, - ) - ) - - server = process_options(parser, console_options, args) - m = console.master.ConsoleMaster(console_options, server) - except exceptions.OptionsError as e: - print("mitmproxy: %s" % e, file=sys.stderr) - sys.exit(1) - try: - m.run() - except (KeyboardInterrupt, RuntimeError): - pass - - -def mitmdump(args=None): # pragma: no cover - from mitmproxy.tools import dump - +def run(MasterKlass, args): # pragma: no cover version_check.check_pyopenssl_version() + debug.register_info_dumpers() - parser = cmdline.mitmdump() + opts = options.Options() + parser = cmdline.mitmdump(opts) args = parser.parse_args(args) - if args.quiet: - args.flow_detail = 0 - master = None try: - dump_options = options.Options() - dump_options.load_paths(args.conf) - dump_options.merge(cmdline.get_common_options(args)) - dump_options.merge( - dict( - flow_detail = args.flow_detail, - keepserving = args.keepserving, - filtstr = " ".join(args.filter) if args.filter else None, - ) - ) - - server = process_options(parser, dump_options, args) - master = dump.DumpMaster(dump_options, server) + opts.load_paths(args.conf) + server = process_options(parser, opts, args) + master = MasterKlass(opts, server) def cleankill(*args, **kwargs): master.shutdown() signal.signal(signal.SIGTERM, cleankill) master.run() - except (dump.DumpError, exceptions.OptionsError) as e: - print("mitmdump: %s" % e, file=sys.stderr) + except exceptions.OptionsError as e: + print("%s: %s" % (sys.argv[0], e), file=sys.stderr) sys.exit(1) except (KeyboardInterrupt, RuntimeError): pass - if master is None or master.has_errored: - print("mitmdump: errors occurred during run", file=sys.stderr) + if master is None or getattr(master, "has_errored", None): + print("%s: errors occurred during run" % sys.argv[0], file=sys.stderr) sys.exit(1) -def mitmweb(args=None): # pragma: no cover - from mitmproxy.tools import web +def mitmproxy(args=None): # pragma: no cover + if os.name == "nt": + print("Error: mitmproxy's console interface is not supported on Windows. " + "You can run mitmdump or mitmweb instead.", file=sys.stderr) + sys.exit(1) + assert_utf8_env() - version_check.check_pyopenssl_version() + from mitmproxy.tools import console + run(console.master.ConsoleMaster, args) - parser = cmdline.mitmweb() - args = parser.parse_args(args) +def mitmdump(args=None): # pragma: no cover + from mitmproxy.tools import dump + run(dump.DumpMaster, args) - try: - web_options = options.Options() - web_options.load_paths(args.conf) - web_options.merge(cmdline.get_common_options(args)) - web_options.merge( - dict( - intercept = args.intercept, - web_open_browser = args.web_open_browser, - web_debug = args.web_debug, - web_iface = args.web_iface, - web_port = args.web_port, - ) - ) - server = process_options(parser, web_options, args) - m = web.master.WebMaster(web_options, server) - except exceptions.OptionsError as e: - print("mitmweb: %s" % e, file=sys.stderr) - sys.exit(1) - try: - m.run() - except (KeyboardInterrupt, RuntimeError): - pass + +def mitmweb(args=None): # pragma: no cover + from mitmproxy.tools import web + run(web.master.WebMaster, args) diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 35b549ee..eddaa3e1 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -408,7 +408,7 @@ class Settings(RequestHandler): mode=str(self.master.options.mode), intercept=self.master.options.intercept, showhost=self.master.options.showhost, - no_upstream_cert=self.master.options.no_upstream_cert, + upstream_cert=self.master.options.upstream_cert, rawtcp=self.master.options.rawtcp, http2=self.master.options.http2, websocket=self.master.options.websocket, @@ -425,7 +425,7 @@ class Settings(RequestHandler): def put(self): update = self.json option_whitelist = { - "intercept", "showhost", "no_upstream_cert", + "intercept", "showhost", "upstream_cert", "rawtcp", "http2", "websocket", "anticache", "anticomp", "stickycookie", "stickyauth", "stream_large_bodies" } diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py index 2cdf7f51..bdd83ee6 100644 --- a/mitmproxy/utils/typecheck.py +++ b/mitmproxy/utils/typecheck.py @@ -1,7 +1,7 @@ import typing -def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: +def check_type(name: str, value: typing.Any, typeinfo: type) -> None: """ This function checks if the provided value is an instance of typeinfo and raises a TypeError otherwise. @@ -17,7 +17,7 @@ def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: e = TypeError("Expected {} for {}, but got {}.".format( typeinfo, - attr_name, + name, type(value) )) @@ -32,7 +32,7 @@ def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: for T in types: try: - check_type(attr_name, value, T) + check_type(name, value, T) except TypeError: pass else: @@ -50,7 +50,7 @@ def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: if len(types) != len(value): raise e for i, (x, T) in enumerate(zip(value, types)): - check_type("{}[{}]".format(attr_name, i), x, T) + check_type("{}[{}]".format(name, i), x, T) return elif typename.startswith("typing.Sequence"): try: @@ -62,7 +62,7 @@ def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: if not isinstance(value, (tuple, list)): raise e for v in value: - check_type(attr_name, v, T) + check_type(name, v, T) elif typename.startswith("typing.IO"): if hasattr(value, "read"): return @@ -70,12 +70,3 @@ def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: raise e elif not isinstance(value, typeinfo): raise e - - -def get_arg_type_from_constructor_annotation(cls: type, attr: str) -> typing.Optional[type]: - """ - Returns the first type annotation for attr in the class hierarchy. - """ - for c in cls.mro(): - if attr in getattr(c.__init__, "__annotations__", ()): - return c.__init__.__annotations__[attr] diff --git a/test/helper_tools/dumperview.py b/test/helper_tools/dumperview.py index be56fe14..d417d767 100755 --- a/test/helper_tools/dumperview.py +++ b/test/helper_tools/dumperview.py @@ -4,12 +4,12 @@ import click from mitmproxy.addons import dumper from mitmproxy.test import tflow from mitmproxy.test import taddons -from mitmproxy.tools import dump +from mitmproxy.tools import options def show(flow_detail, flows): d = dumper.Dumper() - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=flow_detail) for f in flows: ctx.cycle(d, f) diff --git a/test/mitmproxy/addons/test_core_option_validation.py b/test/mitmproxy/addons/test_core_option_validation.py new file mode 100644 index 00000000..0bb2bb0d --- /dev/null +++ b/test/mitmproxy/addons/test_core_option_validation.py @@ -0,0 +1,43 @@ +from mitmproxy import exceptions +from mitmproxy.addons import core_option_validation +from mitmproxy.test import taddons +import pytest +from unittest import mock + + +def test_simple(): + sa = core_option_validation.CoreOptionValidation() + with taddons.context() as tctx: + with pytest.raises(exceptions.OptionsError): + tctx.configure(sa, body_size_limit = "invalid") + tctx.configure(sa, body_size_limit = "1m") + assert tctx.options._processed["body_size_limit"] + + with pytest.raises(exceptions.OptionsError, match="mutually exclusive"): + tctx.configure( + sa, + add_upstream_certs_to_client_chain = True, + upstream_cert = False + ) + with pytest.raises(exceptions.OptionsError, match="Invalid mode"): + tctx.configure( + sa, + mode = "Flibble" + ) + + +@mock.patch("mitmproxy.platform.original_addr", None) +def test_no_transparent(): + sa = core_option_validation.CoreOptionValidation() + with taddons.context() as tctx: + with pytest.raises(Exception, match="Transparent mode not supported"): + tctx.configure(sa, mode = "transparent") + + +@mock.patch("mitmproxy.platform.original_addr") +def test_modes(m): + sa = core_option_validation.CoreOptionValidation() + with taddons.context() as tctx: + tctx.configure(sa, mode = "reverse:http://localhost") + with pytest.raises(Exception, match="Invalid server specification"): + tctx.configure(sa, mode = "reverse:") diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index 22d2c2c6..47374617 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -9,13 +9,13 @@ from mitmproxy.test import tutils from mitmproxy.addons import dumper from mitmproxy import exceptions -from mitmproxy.tools import dump from mitmproxy import http +from mitmproxy import options def test_configure(): d = dumper.Dumper() - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, filtstr="~b foo") assert d.filter @@ -34,7 +34,7 @@ def test_configure(): def test_simple(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=0) d.response(tflow.tflow(resp=True)) assert not sio.getvalue() @@ -103,7 +103,7 @@ def test_echo_body(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=3) d._echo_message(f.response) t = sio.getvalue() @@ -113,7 +113,7 @@ def test_echo_body(): def test_echo_request_line(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.tflow(client_conn=None, server_conn=True, resp=True) f.request.is_replay = True @@ -148,7 +148,7 @@ class TestContentView: view_auto.side_effect = exceptions.ContentViewException("") sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=4, verbosity=3) d.response(tflow.tflow()) assert "Content viewer failed" in ctx.master.event_log[0][1] @@ -157,7 +157,7 @@ class TestContentView: def test_tcp(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.ttcpflow() d.tcp_message(f) @@ -172,7 +172,7 @@ def test_tcp(): def test_websocket(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.twebsocketflow() d.websocket_message(f) diff --git a/test/mitmproxy/addons/test_intercept.py b/test/mitmproxy/addons/test_intercept.py index cf5ba6e8..465e6433 100644 --- a/test/mitmproxy/addons/test_intercept.py +++ b/test/mitmproxy/addons/test_intercept.py @@ -7,15 +7,9 @@ from mitmproxy.test import taddons from mitmproxy.test import tflow -class Options(options.Options): - def __init__(self, *, intercept=None, **kwargs): - self.intercept = intercept - super().__init__(**kwargs) - - def test_simple(): r = intercept.Intercept() - with taddons.context(options=Options()) as tctx: + with taddons.context(options=options.Options()) as tctx: assert not r.filt tctx.configure(r, intercept="~q") assert r.filt diff --git a/test/mitmproxy/addons/test_proxyauth.py b/test/mitmproxy/addons/test_proxyauth.py index dd5829ab..14782755 100644 --- a/test/mitmproxy/addons/test_proxyauth.py +++ b/test/mitmproxy/addons/test_proxyauth.py @@ -28,40 +28,43 @@ def test_configure(): up = proxyauth.ProxyAuth() with taddons.context() as ctx: with pytest.raises(exceptions.OptionsError): - ctx.configure(up, auth_singleuser="foo") + ctx.configure(up, proxyauth="foo") - ctx.configure(up, auth_singleuser="foo:bar") + ctx.configure(up, proxyauth="foo:bar") assert up.singleuser == ["foo", "bar"] - ctx.configure(up, auth_singleuser=None) + ctx.configure(up, proxyauth=None) assert up.singleuser is None - ctx.configure(up, auth_nonanonymous=True) + ctx.configure(up, proxyauth="any") assert up.nonanonymous - ctx.configure(up, auth_nonanonymous=False) + ctx.configure(up, proxyauth=None) assert not up.nonanonymous with pytest.raises(exceptions.OptionsError): - ctx.configure(up, auth_htpasswd=tutils.test_data.path("mitmproxy/net/data/server.crt")) + ctx.configure( + up, + proxyauth= "@" + tutils.test_data.path("mitmproxy/net/data/server.crt") + ) with pytest.raises(exceptions.OptionsError): - ctx.configure(up, auth_htpasswd="nonexistent") + ctx.configure(up, proxyauth="@nonexistent") ctx.configure( up, - auth_htpasswd=tutils.test_data.path( + proxyauth= "@" + tutils.test_data.path( "mitmproxy/net/data/htpasswd" ) ) assert up.htpasswd assert up.htpasswd.check_password("test", "test") assert not up.htpasswd.check_password("test", "foo") - ctx.configure(up, auth_htpasswd=None) + ctx.configure(up, proxyauth=None) assert not up.htpasswd with pytest.raises(exceptions.OptionsError): - ctx.configure(up, auth_nonanonymous=True, mode="transparent") + ctx.configure(up, proxyauth="any", mode="transparent") with pytest.raises(exceptions.OptionsError): - ctx.configure(up, auth_nonanonymous=True, mode="socks5") + ctx.configure(up, proxyauth="any", mode="socks5") ctx.configure(up, mode="regular") assert up.mode == "regular" @@ -70,7 +73,7 @@ def test_configure(): def test_check(): up = proxyauth.ProxyAuth() with taddons.context() as ctx: - ctx.configure(up, auth_nonanonymous=True, mode="regular") + ctx.configure(up, proxyauth="any", mode="regular") f = tflow.tflow() assert not up.check(f) f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( @@ -86,18 +89,17 @@ def test_check(): ) assert not up.check(f) - ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:test") + ctx.configure(up, proxyauth="test:test") f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( "test", "test" ) assert up.check(f) - ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:foo") + ctx.configure(up, proxyauth="test:foo") assert not up.check(f) ctx.configure( up, - auth_singleuser=None, - auth_htpasswd=tutils.test_data.path( + proxyauth="@" + tutils.test_data.path( "mitmproxy/net/data/htpasswd" ) ) @@ -114,7 +116,7 @@ def test_check(): def test_authenticate(): up = proxyauth.ProxyAuth() with taddons.context() as ctx: - ctx.configure(up, auth_nonanonymous=True, mode="regular") + ctx.configure(up, proxyauth="any", mode="regular") f = tflow.tflow() assert not f.response @@ -147,7 +149,7 @@ def test_authenticate(): def test_handlers(): up = proxyauth.ProxyAuth() with taddons.context() as ctx: - ctx.configure(up, auth_nonanonymous=True, mode="regular") + ctx.configure(up, proxyauth="any", mode="regular") f = tflow.tflow() assert not f.response diff --git a/test/mitmproxy/addons/test_replace.py b/test/mitmproxy/addons/test_replace.py index 126c6e3d..8c280c51 100644 --- a/test/mitmproxy/addons/test_replace.py +++ b/test/mitmproxy/addons/test_replace.py @@ -22,11 +22,11 @@ class TestReplace: def test_configure(self): r = replace.Replace() with taddons.context() as tctx: - tctx.configure(r, replacements=[("one", "two", "three")]) + tctx.configure(r, replacements=["one/two/three"]) with pytest.raises(Exception, match="Invalid filter pattern"): - tctx.configure(r, replacements=[("~b", "two", "three")]) + tctx.configure(r, replacements=["/~b/two/three"]) with pytest.raises(Exception, match="Invalid regular expression"): - tctx.configure(r, replacements=[("foo", "+", "three")]) + tctx.configure(r, replacements=["/foo/+/three"]) tctx.configure(r, replacements=["/a/b/c/"]) def test_simple(self): @@ -35,8 +35,8 @@ class TestReplace: tctx.configure( r, replacements = [ - ("~q", "foo", "bar"), - ("~s", "foo", "bar"), + "/~q/foo/bar", + "/~s/foo/bar", ] ) f = tflow.tflow() @@ -58,10 +58,10 @@ class TestUpstreamProxy(tservers.HTTPUpstreamProxyTest): self.proxy.tmaster.addons.add(sa) self.proxy.tmaster.options.replacements = [ - ("~q", "foo", "bar"), - ("~q", "bar", "baz"), - ("~q", "foo", "oh noes!"), - ("~s", "baz", "ORLY") + "/~q/foo/bar", + "/~q/bar/baz", + "/~q/foo/oh noes!", + "/~s/baz/ORLY" ] p = self.pathoc() with p.connect(): @@ -81,9 +81,9 @@ class TestReplaceFile: tctx.configure( r, replacement_files = [ - ("~q", "foo", rp), - ("~s", "foo", rp), - ("~b nonexistent", "nonexistent", "nonexistent"), + "/~q/foo/" + rp, + "/~s/foo/" + rp, + "/~b nonexistent/nonexistent/nonexistent", ] ) f = tflow.tflow() diff --git a/test/mitmproxy/addons/test_setheaders.py b/test/mitmproxy/addons/test_setheaders.py index 6355f2be..3aaee7f4 100644 --- a/test/mitmproxy/addons/test_setheaders.py +++ b/test/mitmproxy/addons/test_setheaders.py @@ -21,7 +21,7 @@ class TestSetHeaders: sh = setheaders.SetHeaders() with taddons.context() as tctx: with pytest.raises(Exception, match="Invalid setheader filter pattern"): - tctx.configure(sh, setheaders = [("~b", "one", "two")]) + tctx.configure(sh, setheaders = ["/~b/one/two"]) tctx.configure(sh, setheaders = ["/foo/bar/voing"]) def test_setheaders(self): @@ -30,8 +30,8 @@ class TestSetHeaders: tctx.configure( sh, setheaders = [ - ("~q", "one", "two"), - ("~s", "one", "three") + "/~q/one/two", + "/~s/one/three" ] ) f = tflow.tflow() @@ -47,8 +47,8 @@ class TestSetHeaders: tctx.configure( sh, setheaders = [ - ("~s", "one", "two"), - ("~s", "one", "three") + "/~s/one/two", + "/~s/one/three" ] ) f = tflow.tflow(resp=True) @@ -60,8 +60,8 @@ class TestSetHeaders: tctx.configure( sh, setheaders = [ - ("~q", "one", "two"), - ("~q", "one", "three") + "/~q/one/two", + "/~q/one/three" ] ) f = tflow.tflow() diff --git a/test/mitmproxy/addons/test_streambodies.py b/test/mitmproxy/addons/test_streambodies.py index b982c66d..c6ce5e81 100644 --- a/test/mitmproxy/addons/test_streambodies.py +++ b/test/mitmproxy/addons/test_streambodies.py @@ -1,13 +1,16 @@ +from mitmproxy import exceptions from mitmproxy.test import tflow from mitmproxy.test import taddons - from mitmproxy.addons import streambodies +import pytest def test_simple(): sa = streambodies.StreamBodies() with taddons.context() as tctx: - tctx.configure(sa, stream_large_bodies = 10) + with pytest.raises(exceptions.OptionsError): + tctx.configure(sa, stream_large_bodies = "invalid") + tctx.configure(sa, stream_large_bodies = "10") f = tflow.tflow() f.request.content = b"" diff --git a/test/mitmproxy/addons/test_streamfile.py b/test/mitmproxy/addons/test_streamfile.py index 4922fc0b..4105c1fc 100644 --- a/test/mitmproxy/addons/test_streamfile.py +++ b/test/mitmproxy/addons/test_streamfile.py @@ -7,13 +7,13 @@ from mitmproxy.test import taddons from mitmproxy import io from mitmproxy import exceptions -from mitmproxy.tools import dump +from mitmproxy import options from mitmproxy.addons import streamfile def test_configure(): sa = streamfile.StreamFile() - with taddons.context(options=dump.Options()) as tctx: + with taddons.context(options=options.Options()) as tctx: with tutils.tmpdir() as tdir: p = os.path.join(tdir, "foo") with pytest.raises(exceptions.OptionsError): @@ -59,7 +59,7 @@ def test_simple(): tctx.configure(sa, streamfile=None) assert rd(p)[0].response - tctx.configure(sa, streamfile=p, streamfile_append=True) + tctx.configure(sa, streamfile="+" + p) f = tflow.tflow() sa.request(f) tctx.configure(sa, streamfile=None) diff --git a/test/mitmproxy/addons/test_termlog.py b/test/mitmproxy/addons/test_termlog.py index 70c3a7f2..2133b74d 100644 --- a/test/mitmproxy/addons/test_termlog.py +++ b/test/mitmproxy/addons/test_termlog.py @@ -3,7 +3,7 @@ import pytest from mitmproxy.addons import termlog from mitmproxy import log -from mitmproxy.tools.dump import Options +from mitmproxy.options import Options from mitmproxy.test import taddons diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index a063416f..b7842314 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -15,23 +15,6 @@ def tft(*, method="get", start=0): return f -class Options(options.Options): - def __init__( - self, - *, - filter=None, - console_order=None, - console_order_reversed=False, - console_focus_follow=False, - **kwargs - ): - self.filter = filter - self.console_order = console_order - self.console_order_reversed = console_order_reversed - self.console_focus_follow = console_focus_follow - super().__init__(**kwargs) - - def test_order_refresh(): v = view.View() sargs = [] @@ -42,7 +25,7 @@ def test_order_refresh(): v.sig_view_refresh.connect(save) tf = tflow.tflow(resp=True) - with taddons.context(options=Options()) as tctx: + with taddons.context(options=options.Options()) as tctx: tctx.configure(v, console_order="time") v.add(tf) tf.request.timestamp_start = 1 @@ -149,7 +132,7 @@ def test_filter(): def test_order(): v = view.View() - with taddons.context(options=Options()) as tctx: + with taddons.context(options=options.Options()) as tctx: v.request(tft(method="get", start=1)) v.request(tft(method="put", start=2)) v.request(tft(method="get", start=3)) @@ -280,7 +263,7 @@ def test_signals(): def test_focus_follow(): v = view.View() - with taddons.context(options=Options()) as tctx: + with taddons.context(options=options.Options()) as tctx: tctx.configure(v, console_focus_follow=True, filter="~m get") v.add(tft(start=5)) @@ -394,7 +377,7 @@ def test_settings(): def test_configure(): v = view.View() - with taddons.context(options=Options()) as tctx: + with taddons.context(options=options.Options()) as tctx: tctx.configure(v, filter="~q") with pytest.raises(Exception, match="Invalid interception filter"): tctx.configure(v, filter="~~") diff --git a/test/mitmproxy/net/test_server_spec.py b/test/mitmproxy/net/test_server_spec.py new file mode 100644 index 00000000..095ad519 --- /dev/null +++ b/test/mitmproxy/net/test_server_spec.py @@ -0,0 +1,32 @@ +import pytest + +from mitmproxy.net import server_spec + + +def test_parse(): + assert server_spec.parse("example.com") == ("https", ("example.com", 443)) + assert server_spec.parse("example.com") == ("https", ("example.com", 443)) + assert server_spec.parse("http://example.com") == ("http", ("example.com", 80)) + assert server_spec.parse("http://127.0.0.1") == ("http", ("127.0.0.1", 80)) + assert server_spec.parse("http://[::1]") == ("http", ("::1", 80)) + assert server_spec.parse("http://[::1]/") == ("http", ("::1", 80)) + assert server_spec.parse("https://[::1]/") == ("https", ("::1", 443)) + assert server_spec.parse("http://[::1]:8080") == ("http", ("::1", 8080)) + + with pytest.raises(ValueError, match="Invalid server specification"): + server_spec.parse(":") + + with pytest.raises(ValueError, match="Invalid server scheme"): + server_spec.parse("ftp://example.com") + + with pytest.raises(ValueError, match="Invalid hostname"): + server_spec.parse("$$$") + + with pytest.raises(ValueError, match="Invalid port"): + server_spec.parse("example.com:999999") + + +def test_parse_with_mode(): + assert server_spec.parse_with_mode("m:example.com") == ("m", ("https", ("example.com", 443))) + with pytest.raises(ValueError): + server_spec.parse_with_mode("moo") diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index 871d02fe..1f695cc5 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -100,7 +100,7 @@ class _Http2TestBase: def get_options(cls): opts = options.Options( listen_port=0, - no_upstream_cert=False, + upstream_cert=True, ssl_insecure=True ) opts.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy") @@ -499,7 +499,8 @@ class TestBodySizeLimit(_Http2Test): return True def test_body_size_limit(self): - self.config.options.body_size_limit = 20 + self.config.options.body_size_limit = "20" + self.config.options._processed["body_size_limit"] = 20 client, h2_conn = self._setup_connection() diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py index bac0e527..486e9d64 100644 --- a/test/mitmproxy/proxy/protocol/test_websocket.py +++ b/test/mitmproxy/proxy/protocol/test_websocket.py @@ -64,7 +64,7 @@ class _WebSocketTestBase: def get_options(cls): opts = options.Options( listen_port=0, - no_upstream_cert=False, + upstream_cert=True, ssl_insecure=True, websocket=True, ) diff --git a/test/mitmproxy/proxy/test_config.py b/test/mitmproxy/proxy/test_config.py index 4272d952..777ab4dd 100644 --- a/test/mitmproxy/proxy/test_config.py +++ b/test/mitmproxy/proxy/test_config.py @@ -1,20 +1 @@ -import pytest -from mitmproxy.proxy import config - - -def test_parse_server_spec(): - with pytest.raises(Exception, match="Invalid server specification"): - config.parse_server_spec("") - assert config.parse_server_spec("http://foo.com:88") == ( - "http", ("foo.com", 88) - ) - assert config.parse_server_spec("http://foo.com") == ( - "http", ("foo.com", 80) - ) - assert config.parse_server_spec("https://foo.com") == ( - "https", ("foo.com", 443) - ) - with pytest.raises(Exception, match="Invalid server specification"): - config.parse_server_spec("foo.com") - with pytest.raises(Exception, match="Invalid server specification"): - config.parse_server_spec("http://") +# TODO: write tests diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 56b09b9a..aa45761a 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -10,7 +10,7 @@ from mitmproxy import options from mitmproxy.addons import script from mitmproxy.addons import proxyauth from mitmproxy import http -from mitmproxy.proxy.config import HostMatcher, parse_server_spec +from mitmproxy.proxy.config import HostMatcher import mitmproxy.net.http from mitmproxy.net import tcp from mitmproxy.net import socks @@ -302,7 +302,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin): class TestHTTPAuth(tservers.HTTPProxyTest): def test_auth(self): self.master.addons.add(proxyauth.ProxyAuth()) - self.master.options.auth_singleuser = "test:test" + self.master.options.proxyauth = "test:test" assert self.pathod("202").status_code == 407 p = self.pathoc() with p.connect(): @@ -321,7 +321,7 @@ class TestHTTPAuth(tservers.HTTPProxyTest): class TestHTTPReverseAuth(tservers.ReverseProxyTest): def test_auth(self): self.master.addons.add(proxyauth.ProxyAuth()) - self.master.options.auth_singleuser = "test:test" + self.master.options.proxyauth = "test:test" assert self.pathod("202").status_code == 401 p = self.pathoc() with p.connect(): @@ -342,22 +342,22 @@ class TestHTTPS(tservers.HTTPProxyTest, CommonMixin, TcpMixin): def test_clientcert_file(self): try: - self.config.clientcerts = os.path.join( + self.config.client_certs = os.path.join( tutils.test_data.path("mitmproxy/data/clientcert"), "client.pem") f = self.pathod("304") assert f.status_code == 304 assert self.server.last_log()["request"]["clientcert"]["keyinfo"] finally: - self.config.clientcerts = None + self.config.client_certs = None def test_clientcert_dir(self): try: - self.config.clientcerts = tutils.test_data.path("mitmproxy/data/clientcert") + self.config.client_certs = tutils.test_data.path("mitmproxy/data/clientcert") f = self.pathod("304") assert f.status_code == 304 assert self.server.last_log()["request"]["clientcert"]["keyinfo"] finally: - self.config.clientcerts = None + self.config.client_certs = None def test_error_post_connect(self): p = self.pathoc() @@ -579,8 +579,6 @@ class TestHttps2Http(tservers.ReverseProxyTest): @classmethod def get_options(cls): opts = super().get_options() - s = parse_server_spec(opts.upstream_server) - opts.upstream_server = "http://{}:{}".format(s.address[0], s.address[1]) return opts def pathoc(self, ssl, sni=None): @@ -870,11 +868,11 @@ class TestServerConnect(tservers.HTTPProxyTest): @classmethod def get_options(cls): opts = tservers.HTTPProxyTest.get_options() - opts.no_upstream_cert = True + opts.upstream_cert = False return opts def test_unnecessary_serverconnect(self): - """A replayed/fake response with no_upstream_cert should not connect to an upstream server""" + """A replayed/fake response with no upstream_cert should not connect to an upstream server""" assert self.pathod("200").status_code == 200 for msg in self.proxy.tmaster.tlog: assert "serverconnect" not in msg diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 0ac3bfd6..f4d32cbb 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -63,8 +63,7 @@ class TestSerialize: r = self._treader() s = tservers.TestState() opts = options.Options( - mode="reverse", - upstream_server="https://use-this-domain" + mode="reverse:https://use-this-domain" ) conf = ProxyConfig(opts) fm = master.Master(opts, DummyServer(conf)) diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 161b0dcf..db33cddd 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -1,6 +1,8 @@ import copy import os import pytest +import typing +import argparse from mitmproxy import options from mitmproxy import optmanager @@ -9,48 +11,51 @@ from mitmproxy.test import tutils class TO(optmanager.OptManager): - def __init__(self, one=None, two=None): - self.one = one - self.two = two + def __init__(self): super().__init__() + self.add_option("one", typing.Optional[int], None, "help") + self.add_option("two", typing.Optional[int], 2, "help") + self.add_option("bool", bool, False, "help") class TD(optmanager.OptManager): - def __init__(self, *, one="done", two="dtwo", three="error"): - self.one = one - self.two = two - self.three = three + def __init__(self): super().__init__() + self.add_option("one", str, "done", "help") + self.add_option("two", str, "dtwo", "help") class TD2(TD): - def __init__(self, *, three="dthree", four="dfour", **kwargs): - self.three = three - self.four = four - super().__init__(three=three, **kwargs) + def __init__(self): + super().__init__() + self.add_option("three", str, "dthree", "help") + self.add_option("four", str, "dfour", "help") class TM(optmanager.OptManager): - def __init__(self, one="one", two=["foo"], three=None): - self.one = one - self.two = two - self.three = three + def __init__(self): super().__init__() + self.add_option("two", typing.Sequence[str], ["foo"], "help") + self.add_option("one", typing.Optional[str], None, "help") -def test_defaults(): - assert TD2.default("one") == "done" - assert TD2.default("two") == "dtwo" - assert TD2.default("three") == "dthree" - assert TD2.default("four") == "dfour" +def test_add_option(): + o = TO() + with pytest.raises(ValueError, match="already exists"): + o.add_option("one", typing.Optional[int], None, "help") + +def test_defaults(): o = TD2() - assert o._defaults == { + defaults = { "one": "done", "two": "dtwo", "three": "dthree", "four": "dfour", } + for k, v in defaults.items(): + assert o.default(k) == v + assert not o.has_changed("one") newvals = dict( one="xone", @@ -64,18 +69,19 @@ def test_defaults(): assert v == getattr(o, k) o.reset() assert not o.has_changed("one") - for k, v in o._defaults.items(): - assert v == getattr(o, k) + + for k in o.keys(): + assert not o.has_changed(k) def test_options(): - o = TO(two="three") - assert o.keys() == set(["one", "two"]) + o = TO() + assert o.keys() == {"bool", "one", "two"} assert o.one is None - assert o.two == "three" - o.one = "one" - assert o.one == "one" + assert o.two == 2 + o.one = 1 + assert o.one == 1 with pytest.raises(TypeError): TO(nonexistent = "value") @@ -91,34 +97,38 @@ def test_options(): o.changed.connect(sub) - o.one = "ninety" + o.one = 90 assert len(rec) == 1 - assert rec[-1].one == "ninety" + assert rec[-1].one == 90 - o.update(one="oink") + o.update(one=3) assert len(rec) == 2 - assert rec[-1].one == "oink" + assert rec[-1].one == 3 def test_setter(): - o = TO(two="three") + o = TO() f = o.setter("two") - f("xxx") - assert o.two == "xxx" + f(99) + assert o.two == 99 with pytest.raises(Exception, match="No such option"): o.setter("nonexistent") def test_toggler(): - o = TO(two=True) - f = o.toggler("two") + o = TO() + f = o.toggler("bool") + assert o.bool is False f() - assert o.two is False + assert o.bool is True f() - assert o.two is True + assert o.bool is False with pytest.raises(Exception, match="No such option"): o.toggler("nonexistent") + with pytest.raises(Exception, match="boolean options"): + o.toggler("one") + class Rec(): def __init__(self): @@ -132,19 +142,19 @@ def test_subscribe(): o = TO() r = Rec() o.subscribe(r, ["two"]) - o.one = "foo" + o.one = 2 assert not r.called - o.two = "foo" + o.two = 3 assert r.called assert len(o.changed.receivers) == 1 del r - o.two = "bar" + o.two = 4 assert len(o.changed.receivers) == 0 def test_rollback(): - o = TO(one="two") + o = TO() rec = [] @@ -157,27 +167,35 @@ def test_rollback(): recerr.append(kwargs) def err(opts, updated): - if opts.one == "ten": + if opts.one == 10: + raise exceptions.OptionsError() + if opts.bool is True: raise exceptions.OptionsError() o.changed.connect(sub) o.changed.connect(err) o.errored.connect(errsub) - o.one = "ten" + assert o.one is None + o.one = 10 + o.bool = True assert isinstance(recerr[0]["exc"], exceptions.OptionsError) - assert o.one == "two" - assert len(rec) == 2 - assert rec[0].one == "ten" - assert rec[1].one == "two" + assert o.one is None + assert o.bool is False + assert len(rec) == 4 + assert rec[0].one == 10 + assert rec[1].one is None + assert rec[2].bool is True + assert rec[3].bool is False + + with pytest.raises(exceptions.OptionsError): + with o.rollback({"one"}, reraise=True): + raise exceptions.OptionsError() -def test_repr(): - assert repr(TO()) == "test.mitmproxy.test_optmanager.TO({'one': None, 'two': None})" - assert repr(TO(one='x' * 60)) == """test.mitmproxy.test_optmanager.TO({ - 'one': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', - 'two': None -})""" +def test_simple(): + assert repr(TO()) + assert "one" in TO() def test_serialize(): @@ -249,3 +267,85 @@ def test_merge(): assert m.one == "two" m.merge(dict(two=["bar"])) assert m.two == ["foo", "bar"] + + +def test_option(): + o = optmanager._Option("test", int, 1, None, None) + assert o.current() == 1 + with pytest.raises(TypeError): + o.set("foo") + with pytest.raises(TypeError): + optmanager._Option("test", str, 1, None, None) + + o2 = optmanager._Option("test", int, 1, None, None) + assert o2 == o + o2.set(5) + assert o2 != o + + +def test_dump(): + o = options.Options() + assert optmanager.dump(o) + + +class TTypes(optmanager.OptManager): + def __init__(self): + super().__init__() + self.add_option("str", str, "str", "help") + self.add_option("optstr", typing.Optional[str], "optstr", "help", "help") + self.add_option("bool", bool, False, "help") + self.add_option("bool_on", bool, True, "help") + self.add_option("int", int, 0, "help") + self.add_option("optint", typing.Optional[int], 0, "help") + self.add_option("seqstr", typing.Sequence[str], [], "help") + self.add_option("unknown", float, 0.0, "help") + + +def test_make_parser(): + parser = argparse.ArgumentParser() + opts = TTypes() + opts.make_parser(parser, "str", short="a") + opts.make_parser(parser, "bool", short="b") + opts.make_parser(parser, "int", short="c") + opts.make_parser(parser, "seqstr", short="d") + opts.make_parser(parser, "bool_on", short="e") + with pytest.raises(ValueError): + opts.make_parser(parser, "unknown") + + +def test_set(): + opts = TTypes() + + opts.set("str=foo") + assert opts.str == "foo" + with pytest.raises(TypeError): + opts.set("str") + + opts.set("optstr=foo") + assert opts.optstr == "foo" + opts.set("optstr") + assert opts.optstr is None + + opts.set("bool=false") + assert opts.bool is False + opts.set("bool") + assert opts.bool is True + opts.set("bool=true") + assert opts.bool is True + with pytest.raises(exceptions.OptionsError): + opts.set("bool=wobble") + + opts.set("int=1") + assert opts.int == 1 + with pytest.raises(exceptions.OptionsError): + opts.set("int=wobble") + opts.set("optint") + assert opts.optint is None + + assert opts.seqstr == [] + opts.set("seqstr=foo") + assert opts.seqstr == ["foo"] + opts.set("seqstr=bar") + assert opts.seqstr == ["foo", "bar"] + opts.set("seqstr") + assert opts.seqstr == [] diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 37cec57a..7a49c530 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -1,4 +1,3 @@ -import os import argparse from unittest import mock from OpenSSL import SSL @@ -6,6 +5,7 @@ import pytest from mitmproxy.tools import cmdline +from mitmproxy.tools import main from mitmproxy import options from mitmproxy.proxy import ProxyConfig from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler @@ -30,10 +30,10 @@ class TestProcessProxyOptions: def p(self, *args): parser = MockParser() - cmdline.common_options(parser) - args = parser.parse_args(args=args) opts = options.Options() - opts.merge(cmdline.get_common_options(args)) + cmdline.common_options(parser, opts) + args = parser.parse_args(args=args) + main.process_options(parser, opts, args) pconf = config.ProxyConfig(opts) return parser, pconf @@ -45,44 +45,6 @@ class TestProcessProxyOptions: def test_simple(self): assert self.p() - def test_cadir(self): - with tutils.tmpdir() as cadir: - self.assert_noerr("--cadir", cadir) - - @mock.patch("mitmproxy.platform.original_addr", None) - def test_no_transparent(self): - with pytest.raises(Exception, match="Transparent mode not supported"): - self.p("-T") - - @mock.patch("mitmproxy.platform.original_addr") - def test_modes(self, _): - self.assert_noerr("-R", "http://localhost") - with pytest.raises(Exception, match="expected one argument"): - self.p("-R") - with pytest.raises(Exception, match="Invalid server specification"): - self.p("-R", "reverse") - - self.assert_noerr("-T") - - self.assert_noerr("-U", "http://localhost") - with pytest.raises(Exception, match="Invalid server specification"): - self.p("-U", "upstream") - - self.assert_noerr("--upstream-auth", "test:test") - with pytest.raises(Exception, match="expected one argument"): - self.p("--upstream-auth") - with pytest.raises(Exception, match="mutually exclusive"): - self.p("-R", "http://localhost", "-T") - - def test_client_certs(self): - with tutils.tmpdir() as cadir: - self.assert_noerr("--client-certs", cadir) - self.assert_noerr( - "--client-certs", - os.path.join(tutils.test_data.path("mitmproxy/data/clientcert"), "client.pem")) - with pytest.raises(Exception, match="path does not exist"): - self.p("--client-certs", "nonexistent") - def test_certs(self): self.assert_noerr( "--cert", @@ -91,19 +53,9 @@ class TestProcessProxyOptions: self.p("--cert", "nonexistent") def test_insecure(self): - p = self.assert_noerr("--insecure") + p = self.assert_noerr("--ssl-insecure") assert p.openssl_verification_mode_server == SSL.VERIFY_NONE - def test_upstream_trusted_cadir(self): - expected_dir = "/path/to/a/ca/dir" - p = self.assert_noerr("--upstream-trusted-cadir", expected_dir) - assert p.options.ssl_verify_upstream_trusted_cadir == expected_dir - - def test_upstream_trusted_ca(self): - expected_file = "/path/to/a/cert/file" - p = self.assert_noerr("--upstream-trusted-ca", expected_file) - assert p.options.ssl_verify_upstream_trusted_ca == expected_file - class TestProxyServer: @@ -131,19 +83,19 @@ class TestDummyServer: class TestConnectionHandler: def test_fatal_error(self, capsys): - config = mock.Mock() - root_layer = mock.Mock() - root_layer.side_effect = RuntimeError - config.options.mode.return_value = root_layer + opts = options.Options() + pconf = config.ProxyConfig(opts) + channel = mock.Mock() def ask(_, x): - return x + raise RuntimeError + channel.ask = ask c = ConnectionHandler( mock.MagicMock(), ("127.0.0.1", 8080), - config, + pconf, channel ) c.handle() diff --git a/test/mitmproxy/tools/test_cmdline.py b/test/mitmproxy/tools/test_cmdline.py index 96d5ae31..65cfeb07 100644 --- a/test/mitmproxy/tools/test_cmdline.py +++ b/test/mitmproxy/tools/test_cmdline.py @@ -1,31 +1,30 @@ import argparse from mitmproxy.tools import cmdline +from mitmproxy.tools import main +from mitmproxy import options def test_common(): parser = argparse.ArgumentParser() - cmdline.common_options(parser) - opts = parser.parse_args(args=[]) - - assert cmdline.get_common_options(opts) - - opts.stickycookie_filt = "foo" - opts.stickyauth_filt = "foo" - v = cmdline.get_common_options(opts) - assert v["stickycookie"] == "foo" - assert v["stickyauth"] == "foo" + opts = options.Options() + cmdline.common_options(parser, opts) + args = parser.parse_args(args=[]) + assert main.process_options(parser, opts, args) def test_mitmproxy(): - ap = cmdline.mitmproxy() + opts = options.Options() + ap = cmdline.mitmproxy(opts) assert ap def test_mitmdump(): - ap = cmdline.mitmdump() + opts = options.Options() + ap = cmdline.mitmdump(opts) assert ap def test_mitmweb(): - ap = cmdline.mitmweb() + opts = options.Options() + ap = cmdline.mitmweb(opts) assert ap diff --git a/test/mitmproxy/tools/test_dump.py b/test/mitmproxy/tools/test_dump.py index b4183725..2542ec4b 100644 --- a/test/mitmproxy/tools/test_dump.py +++ b/test/mitmproxy/tools/test_dump.py @@ -3,8 +3,10 @@ import pytest from unittest import mock from mitmproxy import proxy +from mitmproxy import exceptions from mitmproxy import log from mitmproxy import controller +from mitmproxy import options from mitmproxy.tools import dump from mitmproxy.test import tutils @@ -12,8 +14,8 @@ from .. import tservers class TestDumpMaster(tservers.MasterTest): - def mkmaster(self, flt, **options): - o = dump.Options(filtstr=flt, verbosity=-1, flow_detail=0, **options) + def mkmaster(self, flt, **opts): + o = options.Options(filtstr=flt, verbosity=-1, flow_detail=0, **opts) m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=False, with_dumper=False) return m @@ -25,9 +27,9 @@ class TestDumpMaster(tservers.MasterTest): self.mkmaster(None, rfile=p), 1, b"", ) - with pytest.raises(dump.DumpError): + with pytest.raises(exceptions.OptionsError): self.mkmaster(None, rfile="/nonexistent") - with pytest.raises(dump.DumpError): + with pytest.raises(exceptions.OptionsError): self.mkmaster(None, rfile="test_dump.py") def test_has_error(self): @@ -40,13 +42,13 @@ class TestDumpMaster(tservers.MasterTest): @pytest.mark.parametrize("termlog", [False, True]) def test_addons_termlog(self, termlog): with mock.patch('sys.stdout'): - o = dump.Options() + o = options.Options() m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=termlog) assert (m.addons.get('termlog') is not None) == termlog @pytest.mark.parametrize("dumper", [False, True]) def test_addons_dumper(self, dumper): with mock.patch('sys.stdout'): - o = dump.Options() + o = options.Options() m = dump.DumpMaster(o, proxy.DummyServer(), with_dumper=dumper) assert (m.addons.get('dumper') is not None) == dumper diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 9a289ae5..a8aaa358 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -288,7 +288,7 @@ class ReverseProxyTest(ProxyTestBase): @classmethod def get_options(cls): opts = ProxyTestBase.get_options() - opts.upstream_server = "".join( + s = "".join( [ "https" if cls.ssl else "http", "://", @@ -296,7 +296,7 @@ class ReverseProxyTest(ProxyTestBase): str(cls.server.port) ] ) - opts.mode = "reverse" + opts.mode = "reverse:" + s return opts def pathoc(self, sni=None): @@ -373,9 +373,9 @@ class ChainProxyTest(ProxyTestBase): def get_options(cls): opts = super().get_options() if cls.chain: # First proxy is in normal mode. + s = "http://127.0.0.1:%s" % cls.chain[0].port opts.update( - mode="upstream", - upstream_server="http://127.0.0.1:%s" % cls.chain[0].port + mode="upstream:" + s, ) return opts diff --git a/test/mitmproxy/utils/test_typecheck.py b/test/mitmproxy/utils/test_typecheck.py index 67981be4..d99a914f 100644 --- a/test/mitmproxy/utils/test_typecheck.py +++ b/test/mitmproxy/utils/test_typecheck.py @@ -16,12 +16,6 @@ class T(TBase): super(T, self).__init__(42) -def test_get_arg_type_from_constructor_annotation(): - assert typecheck.get_arg_type_from_constructor_annotation(T, "foo") == str - assert typecheck.get_arg_type_from_constructor_annotation(T, "bar") == int - assert not typecheck.get_arg_type_from_constructor_annotation(T, "baz") - - def test_check_type(): typecheck.check_type("foo", 42, int) with pytest.raises(TypeError): @@ -30,6 +30,7 @@ commands = mypy --ignore-missing-imports --follow-imports=skip \ mitmproxy/addons/ \ mitmproxy/addonmanager.py \ + mitmproxy/optmanager.py \ mitmproxy/proxy/protocol/ \ mitmproxy/log.py \ mitmproxy/tools/dump.py \ |