diff options
27 files changed, 598 insertions, 472 deletions
@@ -1,6 +1,6 @@ .DS_Store MANIFEST -*/tmp +**/tmp /venv* *.py[cdo] *.swp diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 29f60cfe..e94d6a79 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -1,4 +1,5 @@ import itertools +import sys import click @@ -25,10 +26,10 @@ def colorful(line, styles): class Dumper: - def __init__(self): + def __init__(self, outfile=sys.stdout): self.filter = None # type: flowfilter.TFilter self.flow_detail = None # type: int - self.outfp = None # type: typing.io.TextIO + self.outfp = outfile # type: typing.io.TextIO self.showhost = None # type: bool self.default_contentview = "auto" # type: str @@ -43,7 +44,6 @@ class Dumper: else: self.filter = None self.flow_detail = options.flow_detail - self.outfp = options.tfile self.showhost = options.showhost self.default_contentview = options.default_contentview diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index 12544b27..c89fa085 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -32,7 +32,7 @@ def parse_command(command): Returns a (path, args) tuple. """ if not command or not command.strip(): - raise exceptions.AddonError("Empty script command.") + raise exceptions.OptionsError("Empty script command.") # Windows: escape all backslashes in the path. if os.name == "nt": # pragma: no cover backslashes = shlex.split(command, posix=False)[0].count("\\") @@ -40,13 +40,13 @@ def parse_command(command): args = shlex.split(command) # pragma: no cover args[0] = os.path.expanduser(args[0]) if not os.path.exists(args[0]): - raise exceptions.AddonError( + raise exceptions.OptionsError( ("Script file not found: %s.\r\n" "If your script path contains spaces, " "make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") % args[0]) elif os.path.isdir(args[0]): - raise exceptions.AddonError("Not a file: %s" % args[0]) + raise exceptions.OptionsError("Not a file: %s" % args[0]) return args[0], args[1:] diff --git a/mitmproxy/addons/streamfile.py b/mitmproxy/addons/streamfile.py index 377f277d..2fc61015 100644 --- a/mitmproxy/addons/streamfile.py +++ b/mitmproxy/addons/streamfile.py @@ -23,7 +23,7 @@ class StreamFile: def configure(self, options, updated): # We're already streaming - stop the previous stream and restart if "filtstr" in updated: - if options.get("filtstr"): + if options.filtstr: self.filt = flowfilter.parse(options.filtstr) if not self.filt: raise exceptions.OptionsError( diff --git a/mitmproxy/addons/termlog.py b/mitmproxy/addons/termlog.py index 05be32d0..b75f5f5a 100644 --- a/mitmproxy/addons/termlog.py +++ b/mitmproxy/addons/termlog.py @@ -1,11 +1,13 @@ +import sys import click from mitmproxy import log class TermLog: - def __init__(self): + def __init__(self, outfile=sys.stdout): self.options = None + self.outfile = outfile def configure(self, options, updated): self.options = options @@ -14,7 +16,7 @@ class TermLog: if self.options.verbosity >= log.log_tier(e.level): click.secho( e.msg, - file=self.options.tfile, + file=self.outfile, fg=dict(error="red", warn="yellow").get(e.level), dim=(e.level == "debug"), err=(e.level == "error") diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 8a9385da..157b0168 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -30,19 +30,19 @@ class Options(optmanager.OptManager): app_port: int = APP_PORT, anticache: bool = False, anticomp: bool = False, - client_replay: Sequence[str] = (), + client_replay: Sequence[str] = [], replay_kill_extra: bool = False, keepserving: bool = True, no_server: bool = False, server_replay_nopop: bool = False, - refresh_server_playback: bool = False, + refresh_server_playback: bool = True, rfile: Optional[str] = None, - scripts: Sequence[str] = (), + scripts: Sequence[str] = [], showhost: bool = False, - replacements: Sequence[Tuple[str, str, str]] = (), - server_replay_use_headers: Sequence[str] = (), - setheaders: Sequence[Tuple[str, str, str]] = (), - server_replay: Sequence[str] = (), + replacements: Sequence[Tuple[str, str, str]] = [], + server_replay_use_headers: Sequence[str] = [], + setheaders: Sequence[Tuple[str, str, str]] = [], + server_replay: Sequence[str] = [], stickycookie: Optional[str] = None, stickyauth: Optional[str] = None, stream_large_bodies: Optional[int] = None, @@ -51,8 +51,8 @@ class Options(optmanager.OptManager): streamfile: Optional[str] = None, streamfile_append: bool = False, server_replay_ignore_content: bool = False, - server_replay_ignore_params: Sequence[str] = (), - server_replay_ignore_payload_params: Sequence[str] = (), + server_replay_ignore_params: Sequence[str] = [], + server_replay_ignore_payload_params: Sequence[str] = [], server_replay_ignore_host: bool = False, # Proxy options auth_nonanonymous: bool = False, @@ -61,12 +61,12 @@ class Options(optmanager.OptManager): add_upstream_certs_to_client_chain: bool = False, body_size_limit: Optional[int] = None, cadir: str = CA_DIR, - certs: Sequence[Tuple[str, str]] = (), + certs: Sequence[Tuple[str, str]] = [], ciphers_client: str=DEFAULT_CLIENT_CIPHERS, ciphers_server: Optional[str]=None, clientcerts: Optional[str] = None, http2: bool = True, - ignore_hosts: Sequence[str] = (), + ignore_hosts: Sequence[str] = [], listen_host: str = "", listen_port: int = LISTEN_PORT, upstream_bind_address: str = "", @@ -82,7 +82,29 @@ class Options(optmanager.OptManager): ssl_insecure: bool = False, ssl_verify_upstream_trusted_cadir: Optional[str] = None, ssl_verify_upstream_trusted_ca: Optional[str] = None, - tcp_hosts: Sequence[str] = () + tcp_hosts: Sequence[str] = [], + + intercept: Optional[str] = None, + + # Console options + eventlog: bool = False, + focus_follow: bool = False, + filter: Optional[str] = None, + palette: Optional[str] = "dark", + palette_transparent: bool = False, + no_mouse: bool = False, + order: Optional[str] = None, + order_reversed: bool = False, + + # Web options + open_browser: bool = True, + wdebug: bool = False, + wport: int = 8081, + wiface: str = "127.0.0.1", + + # Dump options + filtstr: Optional[str] = None, + flow_detail: int = 1 ) -> None: # We could replace all assignments with clever metaprogramming, # but type hints are a much more valueable asset. @@ -146,4 +168,27 @@ class Options(optmanager.OptManager): self.ssl_verify_upstream_trusted_cadir = ssl_verify_upstream_trusted_cadir self.ssl_verify_upstream_trusted_ca = ssl_verify_upstream_trusted_ca self.tcp_hosts = tcp_hosts + + self.intercept = intercept + + # Console options + self.eventlog = eventlog + self.focus_follow = focus_follow + self.filter = filter + self.palette = palette + self.palette_transparent = palette_transparent + self.no_mouse = no_mouse + self.order = order + self.order_reversed = order_reversed + + # Web options + self.open_browser = open_browser + self.wdebug = wdebug + self.wport = wport + self.wiface = wiface + + # Dump options + self.filtstr = filtstr + self.flow_detail = flow_detail + super().__init__() diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 20492f82..78b358c9 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -1,21 +1,49 @@ import contextlib import blinker import pprint +import inspect +import copy +import functools +import weakref +import os + +import ruamel.yaml from mitmproxy import exceptions from mitmproxy.utils import typecheck + """ The base implementation for Options. """ -class OptManager: +class _DefaultsMeta(type): + def __new__(cls, name, bases, namespace, **kwds): + ret = type.__new__(cls, name, bases, dict(namespace)) + defaults = {} + for klass in reversed(inspect.getmro(ret)): + for p in inspect.signature(klass.__init__).parameters.values(): + if p.kind in (p.KEYWORD_ONLY, p.POSITIONAL_OR_KEYWORD): + if not p.default == p.empty: + defaults[p.name] = p.default + ret._defaults = defaults + return ret + + +class OptManager(metaclass=_DefaultsMeta): """ + OptManager is the base class from which Options objects are derived. + Note that the __init__ method of all child classes must force all + arguments to be positional only, by including a "*" argument. + .changed is a blinker Signal that triggers whenever options are updated. If any handler in the chain raises an exceptions.OptionsError exception, all changes are rolled back, the exception is suppressed, and the .errored signal is notified. + + Optmanager always returns a deep copy of options to ensure that + mutation doesn't change the option state inadvertently. """ _initialized = False attributes = [] @@ -45,6 +73,27 @@ class OptManager: self.__dict__["_opts"] = old self.changed.send(self, updated=updated) + def subscribe(self, func, opts): + """ + Subscribe a callable to the .changed signal, but only for a + specified list of options. The callable should accept arguments + (options, updated), and may raise an OptionsError. + """ + func = weakref.proxy(func) + + @functools.wraps(func) + def _call(options, updated): + if updated.intersection(set(opts)): + try: + func(options, updated) + except ReferenceError: + self.changed.disconnect(_call) + + # Our wrapper function goes out of scope immediately, so we have to set + # weakrefs to false. This means we need to keep our own weakref, and + # clean up the hook when it's gone. + self.changed.connect(_call, weak=False) + def __eq__(self, other): return self._opts == other._opts @@ -53,7 +102,7 @@ class OptManager: def __getattr__(self, attr): if attr in self._opts: - return self._opts[attr] + return copy.deepcopy(self._opts[attr]) else: raise AttributeError("No such option: %s" % attr) @@ -75,8 +124,15 @@ class OptManager: def keys(self): return set(self._opts.keys()) - def get(self, k, d=None): - return self._opts.get(k, d) + def reset(self): + """ + Restore defaults for all options. + """ + self.update(**self._defaults) + + @classmethod + def default(klass, opt): + return copy.deepcopy(klass._defaults[opt]) def update(self, **kwargs): updated = set(kwargs.keys()) @@ -112,6 +168,97 @@ class OptManager: setattr(self, attr, not getattr(self, attr)) return toggle + def has_changed(self, option): + """ + Has the option changed from the default? + """ + if getattr(self, option) != self._defaults[option]: + return True + + def save(self, path, defaults=False): + """ + Save to path. If the destination file exists, modify it in-place. + """ + if os.path.exists(path) and os.path.isfile(path): + with open(path, "r") as f: + data = f.read() + else: + data = "" + data = self.serialize(data, defaults) + with open(path, "w") as f: + f.write(data) + + def serialize(self, text, defaults=False): + """ + Performs a round-trip serialization. If text is not None, it is + treated as a previous serialization that should be modified + in-place. + + - If "defaults" is False, only options with non-default values are + serialized. Default values in text are preserved. + - Unknown options in text are removed. + - Raises OptionsError if text is invalid. + """ + data = self._load(text) + for k in self.keys(): + if defaults or self.has_changed(k): + data[k] = getattr(self, k) + for k in list(data.keys()): + if k not in self._opts: + del data[k] + return ruamel.yaml.round_trip_dump(data) + + def _load(self, text): + if not text: + return {} + try: + data = ruamel.yaml.load(text, ruamel.yaml.Loader) + except ruamel.yaml.error.YAMLError as v: + snip = v.problem_mark.get_snippet() + raise exceptions.OptionsError( + "Config error at line %s:\n%s\n%s" % + (v.problem_mark.line + 1, snip, v.problem) + ) + if isinstance(data, str): + raise exceptions.OptionsError("Config error - no keys found.") + return data + + def load(self, text): + """ + Load configuration from text, over-writing options already set in + this object. May raise OptionsError if the config file is invalid. + """ + data = self._load(text) + self.update(**data) + + def load_paths(self, *paths): + """ + Load paths in order. Each path takes precedence over the previous + path. Paths that don't exist are ignored, errors raise an + OptionsError. + """ + for p in paths: + p = os.path.expanduser(p) + if os.path.exists(p) and os.path.isfile(p): + with open(p, "r") as f: + txt = f.read() + self.load(txt) + + def merge(self, opts): + """ + Merge a dict of options into this object. Options that have None + value are ignored. Lists and tuples are appended to the current + option value. + """ + toset = {} + for k, v in opts.items(): + if v is not None: + if isinstance(v, (list, tuple)): + toset[k] = getattr(self, k) + v + else: + toset[k] = v + self.update(**toset) + def __repr__(self): options = pprint.pformat(self._opts, indent=4).strip(" {}") if "\n" in options: diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index a59fe25f..1c90a7a0 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -7,7 +7,7 @@ import struct import threading import time -import configargparse +import argparse import pydivert import pydivert.consts import pickle @@ -386,8 +386,9 @@ class TransparentProxy: if __name__ == "__main__": - parser = configargparse.ArgumentParser( - description="Windows Transparent Proxy") + parser = argparse.ArgumentParser( + description="Windows Transparent Proxy" + ) parser.add_argument( '--mode', choices=[ diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index f8246199..925491d7 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -1,6 +1,7 @@ -import configargparse -import os +import argparse import re +import os + from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import options @@ -11,6 +12,9 @@ from mitmproxy import version from mitmproxy.addons import view +CONFIG_PATH = os.path.join(options.CA_DIR, "config.yaml") + + class ParseException(Exception): pass @@ -113,13 +117,13 @@ def get_common_options(args): stream_large_bodies = human.parse_size(stream_large_bodies) reps = [] - for i in args.replace: + for i in args.replace or []: try: p = parse_replace_hook(i) except ParseException as e: raise exceptions.OptionsError(e) reps.append(p) - for i in args.replace_file: + for i in args.replace_file or []: try: patt, rex, path = parse_replace_hook(i) except ParseException as e: @@ -133,7 +137,7 @@ def get_common_options(args): reps.append((patt, rex, v)) setheaders = [] - for i in args.setheader: + for i in args.setheader or []: try: p = parse_setheader(i) except ParseException as e: @@ -154,7 +158,7 @@ def get_common_options(args): # Proxy config certs = [] - for i in args.certs: + for i in args.certs or []: parts = i.split("=", 1) if len(parts) == 1: parts = ["*", parts[0]] @@ -287,8 +291,7 @@ def basic_options(parser): ) parser.add_argument( "--anticache", - action="store_true", dest="anticache", default=False, - + action="store_true", dest="anticache", help=""" Strip out request headers that might cause the server to return 304-not-modified. @@ -296,12 +299,12 @@ def basic_options(parser): ) parser.add_argument( "--cadir", - action="store", type=str, dest="cadir", default=options.CA_DIR, + action="store", type=str, dest="cadir", help="Location of the default mitmproxy CA files. (%s)" % options.CA_DIR ) parser.add_argument( "--host", - action="store_true", dest="showhost", default=False, + action="store_true", dest="showhost", help="Use the Host header to construct URLs for display." ) parser.add_argument( @@ -311,12 +314,12 @@ def basic_options(parser): ) parser.add_argument( "-r", "--read-flows", - action="store", dest="rfile", default=None, + action="store", dest="rfile", help="Read flows from file." ) parser.add_argument( "-s", "--script", - action="append", type=str, dest="scripts", default=[], + action="append", type=str, dest="scripts", metavar='"script.py --bar"', help=""" Run a script. Surround with quotes to pass script arguments. Can be @@ -327,18 +330,17 @@ def basic_options(parser): "-t", "--stickycookie", action="store", dest="stickycookie_filt", - default=None, metavar="FILTER", help="Set sticky cookie filter. Matched against requests." ) parser.add_argument( "-u", "--stickyauth", - action="store", dest="stickyauth_filt", default=None, metavar="FILTER", + action="store", dest="stickyauth_filt", metavar="FILTER", help="Set sticky auth filter. Matched against requests." ) parser.add_argument( "-v", "--verbose", - action="store_const", dest="verbose", default=2, const=3, + action="store_const", dest="verbose", const=3, help="Increase log verbosity." ) streamfile = parser.add_mutually_exclusive_group() @@ -354,19 +356,19 @@ def basic_options(parser): ) parser.add_argument( "-z", "--anticomp", - action="store_true", dest="anticomp", default=False, + action="store_true", dest="anticomp", help="Try to convince servers to send us un-compressed data." ) parser.add_argument( "-Z", "--body-size-limit", - action="store", dest="body_size_limit", default=None, + action="store", dest="body_size_limit", metavar="SIZE", help="Byte size limit of HTTP request and response bodies." " Understands k/m/g suffixes, i.e. 3m for 3 megabytes." ) parser.add_argument( "--stream", - action="store", dest="stream_large_bodies", default=None, + action="store", dest="stream_large_bodies", metavar="SIZE", help=""" Stream data to the client if response body exceeds the given @@ -383,7 +385,6 @@ def proxy_modes(parser): action="store", type=str, dest="reverse_proxy", - default=None, help=""" Forward all requests to upstream HTTP server: http[s]://host[:port]. Clients can always connect both @@ -393,12 +394,12 @@ def proxy_modes(parser): ) group.add_argument( "--socks", - action="store_true", dest="socks_proxy", default=False, + action="store_true", dest="socks_proxy", help="Set SOCKS5 proxy mode." ) group.add_argument( "-T", "--transparent", - action="store_true", dest="transparent_proxy", default=False, + action="store_true", dest="transparent_proxy", help="Set transparent proxy mode." ) group.add_argument( @@ -406,7 +407,6 @@ def proxy_modes(parser): action="store", type=str, dest="upstream_proxy", - default=None, help="Forward all requests to upstream proxy server: http://host[:port]" ) @@ -415,12 +415,12 @@ def proxy_options(parser): group = parser.add_argument_group("Proxy Options") group.add_argument( "-b", "--bind-address", - action="store", type=str, dest="addr", default='', + action="store", type=str, dest="addr", help="Address to bind proxy to (defaults to all interfaces)" ) group.add_argument( "-I", "--ignore", - action="append", type=str, dest="ignore_hosts", default=[], + action="append", type=str, dest="ignore_hosts", metavar="HOST", help=""" Ignore host and forward all traffic without processing it. In @@ -433,7 +433,7 @@ def proxy_options(parser): ) group.add_argument( "--tcp", - action="append", type=str, dest="tcp_hosts", default=[], + action="append", type=str, dest="tcp_hosts", metavar="HOST", help=""" Generic TCP SSL proxy mode for all hosts that match the pattern. @@ -448,7 +448,7 @@ def proxy_options(parser): ) group.add_argument( "-p", "--port", - action="store", type=int, dest="port", default=options.LISTEN_PORT, + action="store", type=int, dest="port", help="Proxy service port." ) group.add_argument( @@ -467,7 +467,7 @@ def proxy_options(parser): parser.add_argument( "--upstream-auth", - action="store", dest="upstream_auth", default=None, + action="store", dest="upstream_auth", type=str, help=""" Add HTTP Basic authentcation to upstream proxy and reverse proxy @@ -491,7 +491,7 @@ def proxy_options(parser): ) group.add_argument( "--upstream-bind-address", - action="store", type=str, dest="upstream_bind_address", default='', + action="store", type=str, dest="upstream_bind_address", help="Address to bind upstream requests to (defaults to none)" ) @@ -502,7 +502,6 @@ def proxy_ssl_options(parser): group.add_argument( "--cert", dest='certs', - default=[], type=str, metavar="SPEC", action="append", @@ -514,56 +513,56 @@ def proxy_ssl_options(parser): 'as the first entry. Can be passed multiple times.') group.add_argument( "--ciphers-client", action="store", - type=str, dest="ciphers_client", default=options.DEFAULT_CLIENT_CIPHERS, + type=str, dest="ciphers_client", help="Set supported ciphers for client connections. (OpenSSL Syntax)" ) group.add_argument( "--ciphers-server", action="store", - type=str, dest="ciphers_server", default=None, + type=str, dest="ciphers_server", help="Set supported ciphers for server connections. (OpenSSL Syntax)" ) group.add_argument( "--client-certs", action="store", - type=str, dest="clientcerts", default=None, + type=str, dest="clientcerts", help="Client certificate file or directory." ) group.add_argument( - "--no-upstream-cert", default=False, + "--no-upstream-cert", action="store_true", dest="no_upstream_cert", help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--add-upstream-certs-to-client-chain", default=False, + "--add-upstream-certs-to-client-chain", action="store_true", dest="add_upstream_certs_to_client_chain", help="Add all certificates of the upstream server to the certificate chain " "that will be served to the proxy client, as extras." ) group.add_argument( - "--insecure", default=False, + "--insecure", action="store_true", dest="ssl_insecure", help="Do not verify upstream server SSL/TLS certificates." ) group.add_argument( - "--upstream-trusted-cadir", default=None, action="store", + "--upstream-trusted-cadir", action="store", dest="ssl_verify_upstream_trusted_cadir", help="Path to a directory of trusted CA certificates for upstream " "server verification prepared using the c_rehash tool." ) group.add_argument( - "--upstream-trusted-ca", default=None, action="store", + "--upstream-trusted-ca", action="store", dest="ssl_verify_upstream_trusted_ca", help="Path to a PEM formatted trusted CA certificate." ) group.add_argument( "--ssl-version-client", dest="ssl_version_client", - default="secure", action="store", + action="store", choices=tcp.sslversion_choices.keys(), help="Set supported SSL/TLS versions for client connections. " "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) group.add_argument( "--ssl-version-server", dest="ssl_version_server", - default="secure", action="store", + action="store", choices=tcp.sslversion_choices.keys(), help="Set supported SSL/TLS versions for server connections. " "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." @@ -574,12 +573,12 @@ def onboarding_app(parser): group = parser.add_argument_group("Onboarding App") group.add_argument( "--noapp", - action="store_false", dest="app", default=True, + action="store_false", dest="app", help="Disable the mitmproxy onboarding app." ) group.add_argument( "--app-host", - action="store", dest="app_host", default=options.APP_HOST, metavar="host", + action="store", dest="app_host", help=""" Domain to serve the onboarding app from. For transparent mode, use an IP when a DNS entry for the app domain is not present. Default: @@ -590,7 +589,6 @@ def onboarding_app(parser): "--app-port", action="store", dest="app_port", - default=options.APP_PORT, type=int, metavar="80", help="Port to serve the onboarding app from." @@ -601,7 +599,7 @@ def client_replay(parser): group = parser.add_argument_group("Client Replay") group.add_argument( "-c", "--client-replay", - action="append", dest="client_replay", default=[], metavar="PATH", + action="append", dest="client_replay", metavar="PATH", help="Replay client requests from a saved file." ) @@ -610,23 +608,23 @@ def server_replay(parser): group = parser.add_argument_group("Server Replay") group.add_argument( "-S", "--server-replay", - action="append", dest="server_replay", default=[], metavar="PATH", + action="append", dest="server_replay", metavar="PATH", help="Replay server responses from a saved file." ) group.add_argument( "-k", "--replay-kill-extra", - action="store_true", dest="replay_kill_extra", default=False, + action="store_true", dest="replay_kill_extra", help="Kill extra requests during replay." ) group.add_argument( "--server-replay-use-header", - action="append", dest="server_replay_use_headers", type=str, default=[], + action="append", dest="server_replay_use_headers", type=str, help="Request headers to be considered during replay. " "Can be passed multiple times." ) group.add_argument( "--norefresh", - action="store_true", dest="norefresh", default=False, + action="store_true", dest="norefresh", help=""" Disable response refresh, which updates times in cookies and headers for replayed responses. @@ -634,21 +632,21 @@ def server_replay(parser): ) group.add_argument( "--no-pop", - action="store_true", dest="server_replay_nopop", default=False, + action="store_true", dest="server_replay_nopop", help="Disable response pop from response flow. " "This makes it possible to replay same response multiple times." ) payload = group.add_mutually_exclusive_group() payload.add_argument( "--replay-ignore-content", - action="store_true", dest="server_replay_ignore_content", default=False, + action="store_true", dest="server_replay_ignore_content", help=""" Ignore request's content while searching for a saved flow to replay """ ) payload.add_argument( "--replay-ignore-payload-param", - action="append", dest="server_replay_ignore_payload_params", type=str, default=[], + action="append", dest="server_replay_ignore_payload_params", type=str, help=""" Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to be ignored while searching for a saved flow to replay. @@ -658,7 +656,7 @@ def server_replay(parser): group.add_argument( "--replay-ignore-param", - action="append", dest="server_replay_ignore_params", type=str, default=[], + action="append", dest="server_replay_ignore_params", type=str, help=""" Request's parameters to be ignored while searching for a saved flow to replay. Can be passed multiple times. @@ -668,7 +666,6 @@ def server_replay(parser): "--replay-ignore-host", action="store_true", dest="server_replay_ignore_host", - default=False, help="Ignore request's destination host while searching for a saved flow to replay") @@ -683,13 +680,13 @@ def replacements(parser): ) group.add_argument( "--replace", - action="append", type=str, dest="replace", default=[], + action="append", type=str, dest="replace", metavar="PATTERN", help="Replacement pattern." ) group.add_argument( "--replace-from-file", - action="append", type=str, dest="replace_file", default=[], + action="append", type=str, dest="replace_file", metavar="PATH", help=""" Replacement pattern, where the replacement clause is a path to a @@ -709,7 +706,7 @@ def set_headers(parser): ) group.add_argument( "--setheader", - action="append", type=str, dest="setheader", default=[], + action="append", type=str, dest="setheader", metavar="PATTERN", help="Header set pattern." ) @@ -747,6 +744,15 @@ def proxy_authentication(parser): def common_options(parser): + parser.add_argument( + "--conf", + type=str, dest="conf", default=CONFIG_PATH, + metavar="PATH", + help=""" + Configuration file + """ + ) + basic_options(parser) proxy_modes(parser) proxy_options(parser) @@ -764,26 +770,17 @@ def mitmproxy(): # platforms. from .console import palettes - parser = configargparse.ArgumentParser( - usage="%(prog)s [options]", - args_for_setting_config_path=["--conf"], - default_config_files=[ - os.path.join(options.CA_DIR, "common.conf"), - os.path.join(options.CA_DIR, "mitmproxy.conf") - ], - add_config_file_help=True, - add_env_var_help=True - ) + parser = argparse.ArgumentParser(usage="%(prog)s [options]") common_options(parser) parser.add_argument( - "--palette", type=str, default=palettes.DEFAULT, + "--palette", type=str, action="store", dest="palette", choices=sorted(palettes.palettes.keys()), help="Select color palette: " + ", ".join(palettes.palettes.keys()) ) parser.add_argument( "--palette-transparent", - action="store_true", dest="palette_transparent", default=False, + action="store_true", dest="palette_transparent", help="Set transparent background for palette." ) parser.add_argument( @@ -798,7 +795,7 @@ def mitmproxy(): ) parser.add_argument( "--order", - type=str, dest="order", default=None, + type=str, dest="order", choices=[o[1] for o in view.orders], help="Flow sort order." ) @@ -813,33 +810,24 @@ def mitmproxy(): ) group.add_argument( "-i", "--intercept", action="store", - type=str, dest="intercept", default=None, + type=str, dest="intercept", help="Intercept filter expression." ) group.add_argument( "-f", "--filter", action="store", - type=str, dest="filter", default=None, + type=str, dest="filter", help="Filter view expression." ) return parser def mitmdump(): - parser = configargparse.ArgumentParser( - usage="%(prog)s [options] [filter]", - args_for_setting_config_path=["--conf"], - default_config_files=[ - os.path.join(options.CA_DIR, "common.conf"), - os.path.join(options.CA_DIR, "mitmdump.conf") - ], - add_config_file_help=True, - add_env_var_help=True - ) + parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]") common_options(parser) parser.add_argument( "--keepserving", - action="store_true", dest="keepserving", default=False, + action="store_true", dest="keepserving", help=""" Continue serving after client playback or file read. We exit by default. @@ -847,7 +835,7 @@ def mitmdump(): ) parser.add_argument( "-d", "--detail", - action="count", dest="flow_detail", default=1, + action="count", dest="flow_detail", help="Increase flow detail display level. Can be passed multiple times." ) parser.add_argument( @@ -862,16 +850,7 @@ def mitmdump(): def mitmweb(): - parser = configargparse.ArgumentParser( - usage="%(prog)s [options]", - args_for_setting_config_path=["--conf"], - default_config_files=[ - os.path.join(options.CA_DIR, "common.conf"), - os.path.join(options.CA_DIR, "mitmweb.conf") - ], - add_config_file_help=True, - add_env_var_help=True - ) + parser = argparse.ArgumentParser(usage="%(prog)s [options]") group = parser.add_argument_group("Mitmweb") group.add_argument( @@ -881,13 +860,13 @@ def mitmweb(): ) group.add_argument( "--wport", - action="store", type=int, dest="wport", default=8081, + action="store", type=int, dest="wport", metavar="PORT", help="Mitmweb port." ) group.add_argument( "--wiface", - action="store", dest="wiface", default="127.0.0.1", + action="store", dest="wiface", metavar="IFACE", help="Mitmweb interface." ) @@ -904,7 +883,7 @@ def mitmweb(): ) group.add_argument( "-i", "--intercept", action="store", - type=str, dest="intercept", default=None, + type=str, dest="intercept", help="Intercept filter expression." ) return parser diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index 5e3f3d42..c7c05c35 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -164,7 +164,7 @@ class ScriptEditor(base.GridEditor): def is_error(self, col, val): try: script.parse_command(val) - except exceptions.AddonError as e: + except exceptions.OptionsError as e: return str(e) diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 5d0e0ef4..73d7adbd 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -11,7 +11,6 @@ import tempfile import traceback import urwid -from typing import Optional from mitmproxy import addons from mitmproxy import controller @@ -39,31 +38,9 @@ from mitmproxy.net import tcp EVENTLOG_SIZE = 500 -class Options(mitmproxy.options.Options): - def __init__( - self, - *, # all args are keyword-only. - eventlog: bool = False, - focus_follow: bool = False, - intercept: Optional[str] = None, - filter: Optional[str] = None, - palette: Optional[str] = None, - palette_transparent: bool = False, - no_mouse: bool = False, - order: Optional[str] = None, - order_reversed: bool = False, - **kwargs - ): - self.eventlog = eventlog - self.focus_follow = focus_follow - self.intercept = intercept - self.filter = filter - self.palette = palette - self.palette_transparent = palette_transparent - self.no_mouse = no_mouse - self.order = order - self.order_reversed = order_reversed - super().__init__(**kwargs) +class Logger: + def log(self, evt): + signals.add_log(evt.msg, evt.level) class ConsoleMaster(master.Master): @@ -72,14 +49,12 @@ class ConsoleMaster(master.Master): def __init__(self, options, server): super().__init__(options, server) self.view = view.View() # type: view.View + self.view.sig_view_update.connect(signals.flow_change.send) self.stream_path = None # This line is just for type hinting self.options = self.options # type: Options self.options.errored.connect(self.options_error) - self.palette = options.palette - self.palette_transparent = options.palette_transparent - self.logbuffer = urwid.SimpleListWalker([]) self.view_stack = [] @@ -89,6 +64,7 @@ class ConsoleMaster(master.Master): signals.replace_view_state.connect(self.sig_replace_view_state) signals.push_view_state.connect(self.sig_push_view_state) signals.sig_add_log.connect(self.sig_add_log) + self.addons.add(Logger()) self.addons.add(*addons.default_addons()) self.addons.add(intercept.Intercept(), self.view) @@ -253,10 +229,11 @@ class ConsoleMaster(master.Master): self.ui.start() os.unlink(name) - def set_palette(self, name): - self.palette = name + def set_palette(self, options, updated): self.ui.register_palette( - palettes.palettes[name].palette(self.palette_transparent) + palettes.palettes[options.palette].palette( + options.palette_transparent + ) ) self.ui.clear() @@ -269,7 +246,11 @@ class ConsoleMaster(master.Master): def run(self): self.ui = urwid.raw_display.Screen() self.ui.set_terminal_properties(256) - self.set_palette(self.palette) + self.set_palette(self.options, None) + self.options.subscribe( + self.set_palette, + ["palette", "palette_transparent"] + ) self.loop = urwid.MainLoop( urwid.SolidFill("x"), screen = self.ui, @@ -473,7 +454,3 @@ class ConsoleMaster(master.Master): direction=direction, ), "info") signals.add_log(strutils.bytes_to_escaped_str(message.content), "debug") - - @controller.handler - def log(self, evt): - signals.add_log(evt.msg, evt.level) diff --git a/mitmproxy/tools/console/options.py b/mitmproxy/tools/console/options.py index 824041dc..94483b3d 100644 --- a/mitmproxy/tools/console/options.py +++ b/mitmproxy/tools/console/options.py @@ -3,7 +3,6 @@ import urwid from mitmproxy import contentviews from mitmproxy.tools.console import common from mitmproxy.tools.console import grideditor -from mitmproxy.tools.console import palettes from mitmproxy.tools.console import select from mitmproxy.tools.console import signals @@ -26,6 +25,12 @@ def _mkhelp(): help_context = _mkhelp() +def checker(opt, options): + def _check(): + return options.has_changed(opt) + return _check + + class Options(urwid.WidgetWrap): def __init__(self, master): @@ -36,25 +41,25 @@ class Options(urwid.WidgetWrap): select.Option( "Header Set Patterns", "H", - lambda: len(master.options.setheaders), + checker("setheaders", master.options), self.setheaders ), select.Option( "Ignore Patterns", "I", - lambda: master.options.ignore_hosts, + checker("ignore_hosts", master.options), self.ignore_hosts ), select.Option( "Replacement Patterns", "R", - lambda: len(master.options.replacements), + checker("replacements", master.options), self.replacepatterns ), select.Option( "Scripts", "S", - lambda: master.options.scripts, + checker("scripts", master.options), self.scripts ), @@ -62,19 +67,19 @@ class Options(urwid.WidgetWrap): select.Option( "Default Display Mode", "M", - lambda: self.master.options.default_contentview != "auto", + checker("default_contentview", master.options), self.default_displaymode ), select.Option( "Palette", "P", - lambda: self.master.palette != palettes.DEFAULT, + checker("palette", master.options), self.palette ), select.Option( "Show Host", "w", - lambda: master.options.showhost, + checker("showhost", master.options), master.options.toggler("showhost") ), @@ -82,19 +87,19 @@ class Options(urwid.WidgetWrap): select.Option( "No Upstream Certs", "U", - lambda: master.options.no_upstream_cert, + checker("no_upstream_cert", master.options), master.options.toggler("no_upstream_cert") ), select.Option( "TCP Proxying", "T", - lambda: master.options.tcp_hosts, + checker("tcp_hosts", master.options), self.tcp_hosts ), select.Option( "Don't Verify SSL/TLS Certificates", "V", - lambda: master.options.ssl_insecure, + checker("ssl_insecure", master.options), master.options.toggler("ssl_insecure") ), @@ -102,37 +107,37 @@ class Options(urwid.WidgetWrap): select.Option( "Anti-Cache", "a", - lambda: master.options.anticache, + checker("anticache", master.options), master.options.toggler("anticache") ), select.Option( "Anti-Compression", "o", - lambda: master.options.anticomp, + checker("anticomp", master.options), master.options.toggler("anticomp") ), select.Option( "Kill Extra", "x", - lambda: master.options.replay_kill_extra, + checker("replay_kill_extra", master.options), master.options.toggler("replay_kill_extra") ), select.Option( "No Refresh", "f", - lambda: not master.options.refresh_server_playback, + checker("refresh_server_playback", master.options), master.options.toggler("refresh_server_playback") ), select.Option( "Sticky Auth", "A", - lambda: master.options.stickyauth, + checker("stickyauth", master.options), self.sticky_auth ), select.Option( "Sticky Cookies", "t", - lambda: master.options.stickycookie, + checker("stickycookie", master.options), self.sticky_cookie ), ] @@ -160,25 +165,10 @@ class Options(urwid.WidgetWrap): return super().keypress(size, key) def clearall(self): - self.master.options.update( - anticache = False, - anticomp = False, - ignore_hosts = (), - tcp_hosts = (), - replay_kill_extra = False, - no_upstream_cert = False, - refresh_server_playback = True, - replacements = [], - scripts = [], - setheaders = [], - showhost = False, - stickyauth = None, - stickycookie = None, - default_contentview = "auto", - ) + self.master.options.reset() signals.update_settings.send(self) signals.status_message.send( - message = "All select.Options cleared", + message = "Options cleared", expire = 1 ) diff --git a/mitmproxy/tools/console/palettepicker.py b/mitmproxy/tools/console/palettepicker.py index a3eb9b90..0d943baf 100644 --- a/mitmproxy/tools/console/palettepicker.py +++ b/mitmproxy/tools/console/palettepicker.py @@ -3,7 +3,6 @@ import urwid from mitmproxy.tools.console import common from mitmproxy.tools.console import palettes from mitmproxy.tools.console import select -from mitmproxy.tools.console import signals footer = [ ('heading_key', "enter/space"), ":select", @@ -43,8 +42,8 @@ class PalettePicker(urwid.WidgetWrap): return select.Option( i, None, - lambda: self.master.palette == name, - lambda: self.select(name) + lambda: self.master.options.palette == name, + lambda: setattr(self.master.options, "palette", name) ) for i in high: @@ -59,8 +58,8 @@ class PalettePicker(urwid.WidgetWrap): select.Option( "Transparent", "T", - lambda: master.palette_transparent, - self.toggle_palette_transparent + lambda: master.options.palette_transparent, + master.options.toggler("palette_transparent") ) ] ) @@ -73,15 +72,7 @@ class PalettePicker(urwid.WidgetWrap): self.lb, header = title ) - signals.update_settings.connect(self.sig_update_settings) + master.options.changed.connect(self.sig_options_changed) - def sig_update_settings(self, sender): + def sig_options_changed(self, options, updated): self.lb.walker._modified() - - def select(self, name): - self.master.set_palette(name) - - def toggle_palette_transparent(self): - self.master.palette_transparent = not self.master.palette_transparent - self.master.set_palette(self.master.palette) - signals.update_settings.send(self) diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index 3cd94c30..90332627 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -1,9 +1,8 @@ -from typing import Optional, IO +from typing import Optional from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import addons -from mitmproxy import io from mitmproxy import options from mitmproxy import master from mitmproxy.addons import dumper, termlog @@ -21,13 +20,11 @@ class Options(options.Options): keepserving: bool = False, filtstr: Optional[str] = None, flow_detail: int = 1, - tfile: Optional[IO[str]] = None, **kwargs ) -> None: self.filtstr = filtstr self.flow_detail = flow_detail self.keepserving = keepserving - self.tfile = tfile super().__init__(**kwargs) @@ -62,16 +59,6 @@ class DumpMaster(master.Master): self.add_log("Flow file corrupted.", "error") raise DumpError(v) - def _readflow(self, paths): - """ - Utitility function that reads a list of flows - or raises a DumpError if that fails. - """ - try: - return io.read_flows_from_paths(paths) - except exceptions.FlowReadException as e: - raise DumpError(str(e)) - @controller.handler def log(self, e): if e.level == "error": diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index c3b1e3a9..d07ae666 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -11,6 +11,7 @@ import signal # noqa from mitmproxy.tools import cmdline # noqa from mitmproxy import exceptions # noqa +from mitmproxy import options # noqa from mitmproxy.proxy import config # noqa from mitmproxy.proxy import server # noqa from mitmproxy.utils import version_check # noqa @@ -63,17 +64,21 @@ def mitmproxy(args=None): # pragma: no cover args = parser.parse_args(args) try: - console_options = console.master.Options( - **cmdline.get_common_options(args) + console_options = options.Options() + console_options.load_paths(args.conf) + console_options.merge(cmdline.get_common_options(args)) + console_options.merge( + dict( + palette = args.palette, + palette_transparent = args.palette_transparent, + eventlog = args.eventlog, + focus_follow = args.focus_follow, + intercept = args.intercept, + filter = args.filter, + no_mouse = args.no_mouse, + order = args.order, + ) ) - console_options.palette = args.palette - console_options.palette_transparent = args.palette_transparent - console_options.eventlog = args.eventlog - console_options.focus_follow = args.focus_follow - console_options.intercept = args.intercept - console_options.filter = args.filter - console_options.no_mouse = args.no_mouse - console_options.order = args.order server = process_options(parser, console_options, args) m = console.master.ConsoleMaster(console_options, server) @@ -98,10 +103,17 @@ def mitmdump(args=None): # pragma: no cover master = None try: - dump_options = dump.Options(**cmdline.get_common_options(args)) - dump_options.flow_detail = args.flow_detail - dump_options.keepserving = args.keepserving - dump_options.filtstr = " ".join(args.filter) if args.filter else None + dump_options = options.Options() + dump_options.load_paths(args.conf) + dump_options.merge(cmdline.get_common_options(args)) + dump_options.merge( + dict( + flow_detail = args.flow_detail, + keepserving = args.keepserving, + filtstr = " ".join(args.filter) if args.filter else None, + ) + ) + server = process_options(parser, dump_options, args) master = dump.DumpMaster(dump_options, server) @@ -130,13 +142,18 @@ def mitmweb(args=None): # pragma: no cover args = parser.parse_args(args) try: - web_options = web.master.Options(**cmdline.get_common_options(args)) - web_options.intercept = args.intercept - web_options.open_browser = args.open_browser - web_options.wdebug = args.wdebug - web_options.wiface = args.wiface - web_options.wport = args.wport - + web_options = options.Options() + web_options.load_paths(args.conf) + web_options.merge(cmdline.get_common_options(args)) + web_options.merge( + dict( + intercept = args.intercept, + open_browser = args.open_browser, + wdebug = args.wdebug, + wiface = args.wiface, + wport = args.wport, + ) + ) server = process_options(parser, web_options, args) m = web.master.WebMaster(web_options, server) except exceptions.OptionsError as e: diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 89ad698d..edb12467 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -1,5 +1,4 @@ import webbrowser -from typing import Optional, IO import tornado.httpserver import tornado.ioloop @@ -7,7 +6,6 @@ from mitmproxy import addons from mitmproxy import exceptions from mitmproxy import log from mitmproxy import master -from mitmproxy import options from mitmproxy.addons import eventstore from mitmproxy.addons import intercept from mitmproxy.addons import termlog @@ -15,27 +13,6 @@ from mitmproxy.addons import view from mitmproxy.tools.web import app -class Options(options.Options): - def __init__( - self, - *, # all args are keyword-only. - intercept: Optional[str] = None, - tfile: Optional[IO[str]] = None, - open_browser: bool = True, - wdebug: bool = False, - wport: int = 8081, - wiface: str = "127.0.0.1", - **kwargs - ) -> None: - self.intercept = intercept - self.tfile = tfile - self.open_browser = open_browser - self.wdebug = wdebug - self.wport = wport - self.wiface = wiface - super().__init__(**kwargs) - - class WebMaster(master.Master): def __init__(self, options, server): super().__init__(options, server) @@ -61,7 +61,6 @@ setup( "blinker>=1.4, <1.5", "click>=6.2, <7.0", "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! - "configargparse>=0.10, <0.12", "construct>=2.8, <2.9", "cryptography>=1.3, <1.7", "cssutils>=1.0.1, <1.1", @@ -78,6 +77,7 @@ setup( "pyparsing>=2.1.3, <2.2", "pyperclip>=1.5.22, <1.6", "requests>=2.9.1, <3", + "ruamel.yaml>=0.13.2, <0.14", "tornado>=4.3, <4.5", "urwid>=1.3.1, <1.4", "watchdog>=0.8.3, <0.9", diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index 0d61f800..760efa08 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -28,43 +28,40 @@ def test_configure(): def test_simple(): - d = dumper.Dumper() + sio = io.StringIO() + d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - sio = io.StringIO() - ctx.configure(d, tfile = sio, flow_detail = 0) + ctx.configure(d, flow_detail=0) d.response(tflow.tflow(resp=True)) assert not sio.getvalue() sio.truncate(0) - ctx.configure(d, tfile = sio, flow_detail = 1) + ctx.configure(d, flow_detail=1) d.response(tflow.tflow(resp=True)) assert sio.getvalue() sio.truncate(0) - ctx.configure(d, tfile = sio, flow_detail = 1) + ctx.configure(d, flow_detail=1) d.error(tflow.tflow(err=True)) assert sio.getvalue() sio.truncate(0) - ctx.configure(d, tfile = sio, flow_detail = 4) + ctx.configure(d, flow_detail=4) d.response(tflow.tflow(resp=True)) assert sio.getvalue() sio.truncate(0) - sio = io.StringIO() - ctx.configure(d, tfile = sio, flow_detail = 4) + ctx.configure(d, flow_detail=4) d.response(tflow.tflow(resp=True)) assert "<<" in sio.getvalue() sio.truncate(0) - sio = io.StringIO() - ctx.configure(d, tfile = sio, flow_detail = 4) + ctx.configure(d, flow_detail=4) d.response(tflow.tflow(err=True)) assert "<<" in sio.getvalue() sio.truncate(0) - sio = io.StringIO() - ctx.configure(d, tfile = sio, flow_detail = 4) + ctx.configure(d, flow_detail=4) flow = tflow.tflow() flow.request = tutils.treq() flow.request.stickycookie = True @@ -77,8 +74,7 @@ def test_simple(): assert sio.getvalue() sio.truncate(0) - sio = io.StringIO() - ctx.configure(d, tfile = sio, flow_detail = 4) + ctx.configure(d, flow_detail=4) flow = tflow.tflow(resp=tutils.tresp(content=b"{")) flow.response.headers["content-type"] = "application/json" flow.response.status_code = 400 @@ -86,8 +82,7 @@ def test_simple(): assert sio.getvalue() sio.truncate(0) - sio = io.StringIO() - ctx.configure(d, tfile = sio, flow_detail = 4) + ctx.configure(d, flow_detail=4) flow = tflow.tflow() flow.request.content = None flow.response = http.HTTPResponse.wrap(tutils.tresp()) @@ -102,20 +97,20 @@ def test_echo_body(): f.response.headers["content-type"] = "text/html" f.response.content = b"foo bar voing\n" * 100 - d = dumper.Dumper() sio = io.StringIO() + d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - ctx.configure(d, tfile=sio, flow_detail = 3) + ctx.configure(d, flow_detail=3) d._echo_message(f.response) t = sio.getvalue() assert "cut off" in t def test_echo_request_line(): - d = dumper.Dumper() sio = io.StringIO() + d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - ctx.configure(d, tfile=sio, flow_detail = 3, showhost = True) + ctx.configure(d, flow_detail=3, showhost=True) f = tflow.tflow(client_conn=None, server_conn=True, resp=True) f.request.is_replay = True d._echo_request_line(f) @@ -139,19 +134,19 @@ class TestContentView: @mock.patch("mitmproxy.contentviews.ViewAuto.__call__") def test_contentview(self, view_auto): view_auto.side_effect = exceptions.ContentViewException("") - d = dumper.Dumper() + sio = io.StringIO() + d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - sio = io.StringIO() - ctx.configure(d, flow_detail=4, verbosity=3, tfile=sio) + ctx.configure(d, flow_detail=4, verbosity=3) d.response(tflow.tflow()) assert "Content viewer failed" in ctx.master.event_log[0][1] def test_tcp(): - d = dumper.Dumper() sio = io.StringIO() + d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - ctx.configure(d, tfile=sio, flow_detail = 3, showhost = True) + ctx.configure(d, flow_detail=3, showhost=True) f = tflow.ttcpflow(client_conn=True, server_conn=True) d.tcp_message(f) assert "it's me" in sio.getvalue() diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index a41f6103..06463fa3 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -64,10 +64,10 @@ def test_reloadhandler(): class TestParseCommand: def test_empty_command(self): - with tutils.raises(exceptions.AddonError): + with tutils.raises(exceptions.OptionsError): script.parse_command("") - with tutils.raises(exceptions.AddonError): + with tutils.raises(exceptions.OptionsError): script.parse_command(" ") def test_no_script_file(self): diff --git a/test/mitmproxy/addons/test_termlog.py b/test/mitmproxy/addons/test_termlog.py index 880fcb51..d9e18134 100644 --- a/test/mitmproxy/addons/test_termlog.py +++ b/test/mitmproxy/addons/test_termlog.py @@ -7,9 +7,9 @@ from mitmproxy.tools import dump class TestTermLog: def test_simple(self): - t = termlog.TermLog() sio = io.StringIO() - t.configure(dump.Options(tfile = sio, verbosity = 2), set([])) + t = termlog.TermLog(outfile=sio) + t.configure(dump.Options(verbosity = 2), set([])) t.log(log.LogEntry("one", "info")) assert "one" in sio.getvalue() t.log(log.LogEntry("two", "debug")) diff --git a/test/mitmproxy/console/test_master.py b/test/mitmproxy/console/test_master.py index eb840187..fb3c2527 100644 --- a/test/mitmproxy/console/test_master.py +++ b/test/mitmproxy/console/test_master.py @@ -2,6 +2,7 @@ from mitmproxy.test import tflow import mitmproxy.test.tutils from mitmproxy.tools import console from mitmproxy import proxy +from mitmproxy import options from mitmproxy.tools.console import common from .. import mastertest @@ -20,14 +21,14 @@ def test_format_keyvals(): def test_options(): - assert console.master.Options(replay_kill_extra=True) + assert options.Options(replay_kill_extra=True) class TestMaster(mastertest.MasterTest): - def mkmaster(self, **options): - if "verbosity" not in options: - options["verbosity"] = 0 - o = console.master.Options(**options) + def mkmaster(self, **opts): + if "verbosity" not in opts: + opts["verbosity"] = 0 + o = options.Options(**opts) return console.master.ConsoleMaster(o, proxy.DummyServer()) def test_basic(self): diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py deleted file mode 100644 index c6b15c84..00000000 --- a/test/mitmproxy/test_dump.py +++ /dev/null @@ -1,163 +0,0 @@ -from mitmproxy.test import tflow -import os -import io - -from mitmproxy.tools import dump -from mitmproxy import exceptions -from mitmproxy import proxy -from mitmproxy.test import tutils -from . import mastertest - - -class TestDumpMaster(mastertest.MasterTest): - def dummy_cycle(self, master, n, content): - mastertest.MasterTest.dummy_cycle(self, master, n, content) - return master.options.tfile.getvalue() - - def mkmaster(self, flt, **options): - if "verbosity" not in options: - options["verbosity"] = 0 - if "flow_detail" not in options: - options["flow_detail"] = 0 - o = dump.Options(filtstr=flt, tfile=io.StringIO(), **options) - return dump.DumpMaster(o, proxy.DummyServer()) - - def test_basic(self): - for i in (1, 2, 3): - assert "GET" in self.dummy_cycle( - self.mkmaster("~s", flow_detail=i), - 1, - b"" - ) - assert "GET" in self.dummy_cycle( - self.mkmaster("~s", flow_detail=i), - 1, - b"\x00\x00\x00" - ) - assert "GET" in self.dummy_cycle( - self.mkmaster("~s", flow_detail=i), - 1, - b"ascii" - ) - - def test_error(self): - o = dump.Options( - tfile=io.StringIO(), - flow_detail=1 - ) - m = dump.DumpMaster(o, proxy.DummyServer()) - f = tflow.tflow(err=True) - m.error(f) - assert "error" in o.tfile.getvalue() - - def test_replay(self): - o = dump.Options(http2=False, server_replay=["nonexistent"], replay_kill_extra=True) - tutils.raises(exceptions.OptionsError, dump.DumpMaster, o, proxy.DummyServer()) - - with tutils.tmpdir() as t: - p = os.path.join(t, "rep") - self.flowfile(p) - - o = dump.Options(http2=False, server_replay=[p], replay_kill_extra=True) - o.verbosity = 0 - o.flow_detail = 0 - m = dump.DumpMaster(o, proxy.DummyServer()) - - self.cycle(m, b"content") - self.cycle(m, b"content") - - o = dump.Options(http2=False, server_replay=[p], replay_kill_extra=False) - o.verbosity = 0 - o.flow_detail = 0 - m = dump.DumpMaster(o, proxy.DummyServer()) - self.cycle(m, b"nonexistent") - - o = dump.Options(http2=False, client_replay=[p], replay_kill_extra=False) - o.verbosity = 0 - o.flow_detail = 0 - m = dump.DumpMaster(o, proxy.DummyServer()) - - def test_read(self): - with tutils.tmpdir() as t: - p = os.path.join(t, "read") - self.flowfile(p) - assert "GET" in self.dummy_cycle( - self.mkmaster(None, flow_detail=1, rfile=p), - 1, b"", - ) - tutils.raises( - dump.DumpError, - self.mkmaster, None, verbosity=1, rfile="/nonexistent" - ) - tutils.raises( - dump.DumpError, - self.mkmaster, None, verbosity=1, rfile="test_dump.py" - ) - - def test_options(self): - o = dump.Options(verbosity = 2) - assert o.verbosity == 2 - - def test_filter(self): - assert "GET" not in self.dummy_cycle( - self.mkmaster("~u foo", verbosity=1), 1, b"" - ) - - def test_replacements(self): - o = dump.Options( - replacements=[(".*", "content", "foo")], - tfile = io.StringIO(), - ) - o.verbosity = 0 - o.flow_detail = 0 - m = dump.DumpMaster(o, proxy.DummyServer()) - f = self.cycle(m, b"content") - assert f.request.content == b"foo" - - def test_setheader(self): - o = dump.Options( - setheaders=[(".*", "one", "two")], - tfile=io.StringIO() - ) - o.verbosity = 0 - o.flow_detail = 0 - m = dump.DumpMaster(o, proxy.DummyServer()) - f = self.cycle(m, b"content") - assert f.request.headers["one"] == "two" - - def test_script(self): - ret = self.dummy_cycle( - self.mkmaster( - None, - scripts=[tutils.test_data.path("mitmproxy/data/scripts/all.py")], - verbosity=2 - ), - 1, b"", - ) - assert "XCLIENTCONNECT" in ret - assert "XSERVERCONNECT" in ret - assert "XREQUEST" in ret - assert "XRESPONSE" in ret - assert "XCLIENTDISCONNECT" in ret - tutils.raises( - exceptions.AddonError, - self.mkmaster, - None, scripts=["nonexistent"] - ) - tutils.raises( - exceptions.AddonError, - self.mkmaster, - None, scripts=["starterr.py"] - ) - - def test_stickycookie(self): - self.dummy_cycle( - self.mkmaster(None, stickycookie = ".*"), - 1, b"" - ) - - def test_stickyauth(self): - self.dummy_cycle( - self.mkmaster(None, stickyauth = ".*"), - 1, b"" - ) diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 3c845707..9e938703 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -1,5 +1,7 @@ import copy +import os +from mitmproxy import options from mitmproxy import optmanager from mitmproxy import exceptions from mitmproxy.test import tutils @@ -12,6 +14,51 @@ class TO(optmanager.OptManager): super().__init__() +class TD(optmanager.OptManager): + def __init__(self, *, one="done", two="dtwo", three="error"): + self.one = one + self.two = two + self.three = three + super().__init__() + + +class TD2(TD): + def __init__(self, *, three="dthree", four="dfour", **kwargs): + self.three = three + self.four = four + super().__init__(three=three, **kwargs) + + +def test_defaults(): + assert TD2.default("one") == "done" + assert TD2.default("two") == "dtwo" + assert TD2.default("three") == "dthree" + assert TD2.default("four") == "dfour" + + o = TD2() + assert o._defaults == { + "one": "done", + "two": "dtwo", + "three": "dthree", + "four": "dfour", + } + assert not o.has_changed("one") + newvals = dict( + one="xone", + two="xtwo", + three="xthree", + four="xfour", + ) + o.update(**newvals) + assert o.has_changed("one") + for k, v in newvals.items(): + assert v == getattr(o, k) + o.reset() + assert not o.has_changed("one") + for k, v in o._defaults.items(): + assert v == getattr(o, k) + + def test_options(): o = TO(two="three") assert o.keys() == set(["one", "two"]) @@ -64,6 +111,29 @@ def test_toggler(): o.toggler("nonexistent") +class Rec(): + def __init__(self): + self.called = None + + def __call__(self, *args, **kwargs): + self.called = (args, kwargs) + + +def test_subscribe(): + o = TO() + r = Rec() + o.subscribe(r, ["two"]) + o.one = "foo" + assert not r.called + o.two = "foo" + assert r.called + + assert len(o.changed.receivers) == 1 + del r + o.two = "bar" + assert len(o.changed.receivers) == 0 + + def test_rollback(): o = TO(one="two") @@ -99,3 +169,72 @@ def test_repr(): 'one': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', 'two': None })""" + + +def test_serialize(): + o = TD2() + o.three = "set" + assert "dfour" in o.serialize(None, defaults=True) + + data = o.serialize(None) + assert "dfour" not in data + + o2 = TD2() + o2.load(data) + assert o2 == o + + t = """ + unknown: foo + """ + data = o.serialize(t) + o2 = TD2() + o2.load(data) + assert o2 == o + + t = "invalid: foo\ninvalid" + tutils.raises("config error", o2.load, t) + + t = "invalid" + tutils.raises("config error", o2.load, t) + + t = "" + o2.load(t) + + +def test_serialize_defaults(): + o = options.Options() + assert o.serialize(None, defaults=True) + + +def test_saving(): + o = TD2() + o.three = "set" + with tutils.tmpdir() as tdir: + dst = os.path.join(tdir, "conf") + o.save(dst, defaults=True) + + o2 = TD2() + o2.load_paths(dst) + o2.three = "foo" + o2.save(dst, defaults=True) + + o.load_paths(dst) + assert o.three == "foo" + + +class TM(optmanager.OptManager): + def __init__(self, one="one", two=["foo"], three=None): + self.one = one + self.two = two + self.three = three + super().__init__() + + +def test_merge(): + m = TM() + m.merge(dict(one="two")) + assert m.one == "two" + m.merge(dict(one=None)) + assert m.one == "two" + m.merge(dict(two=["bar"])) + assert m.two == ["foo", "bar"] diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 177bac1f..0d63f147 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -75,8 +75,9 @@ class TestProcessProxyOptions: parser = MockParser() cmdline.common_options(parser) args = parser.parse_args(args=args) - opts = cmdline.get_common_options(args) - pconf = config.ProxyConfig(options.Options(**opts)) + opts = options.Options() + opts.merge(cmdline.get_common_options(args)) + pconf = config.ProxyConfig(opts) return parser, pconf def assert_err(self, err, *args): diff --git a/test/mitmproxy/test_tools_dump.py b/test/mitmproxy/test_tools_dump.py new file mode 100644 index 00000000..1488f33b --- /dev/null +++ b/test/mitmproxy/test_tools_dump.py @@ -0,0 +1,38 @@ +import os + +from mitmproxy.tools import dump +from mitmproxy import proxy +from mitmproxy.test import tutils +from mitmproxy import log +from mitmproxy import controller +from . import mastertest + + +class TestDumpMaster(mastertest.MasterTest): + def mkmaster(self, flt, **options): + o = dump.Options(filtstr=flt, verbosity=-1, flow_detail=0, **options) + return dump.DumpMaster(o, proxy.DummyServer()) + + def test_read(self): + with tutils.tmpdir() as t: + p = os.path.join(t, "read") + self.flowfile(p) + self.dummy_cycle( + self.mkmaster(None, rfile=p), + 1, b"", + ) + tutils.raises( + dump.DumpError, + self.mkmaster, None, rfile="/nonexistent" + ) + tutils.raises( + dump.DumpError, + self.mkmaster, None, rfile="test_dump.py" + ) + + def test_has_error(self): + m = self.mkmaster(None) + ent = log.LogEntry("foo", "error") + ent.reply = controller.DummyReply() + m.log(ent) + assert m.has_errored diff --git a/test/mitmproxy/test_web_app.py b/test/mitmproxy/test_web_app.py index be195528..2cab5bf4 100644 --- a/test/mitmproxy/test_web_app.py +++ b/test/mitmproxy/test_web_app.py @@ -4,6 +4,7 @@ import mock import tornado.testing from mitmproxy import exceptions from mitmproxy import proxy +from mitmproxy import options from mitmproxy.test import tflow from mitmproxy.tools.web import app from mitmproxy.tools.web import master as webmaster @@ -17,7 +18,7 @@ def json(resp: httpclient.HTTPResponse): class TestApp(tornado.testing.AsyncHTTPTestCase): def get_app(self): - o = webmaster.Options() + o = options.Options() m = webmaster.WebMaster(o, proxy.DummyServer()) f = tflow.tflow(resp=True) f.id = "42" diff --git a/test/mitmproxy/test_web_master.py b/test/mitmproxy/test_web_master.py index 298b14eb..08dce8f3 100644 --- a/test/mitmproxy/test_web_master.py +++ b/test/mitmproxy/test_web_master.py @@ -1,11 +1,12 @@ from mitmproxy.tools.web import master from mitmproxy import proxy +from mitmproxy import options from . import mastertest class TestWebMaster(mastertest.MasterTest): - def mkmaster(self, **options): - o = master.Options(**options) + def mkmaster(self, **opts): + o = options.Options(**opts) return master.WebMaster(o, proxy.DummyServer(o)) def test_basic(self): |