From 67381ae550a5d57c1f2841cd7118550afdfaa736 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 5 Mar 2017 14:55:46 +1300 Subject: Revamp options - Options are now explicitly initialized with an add_option method - We have one canonical Options class - ditch dump.Options --- mitmproxy/options.py | 254 ++++++++++--------------------- mitmproxy/optmanager.py | 147 ++++++++++-------- mitmproxy/test/taddons.py | 14 +- mitmproxy/tools/dump.py | 19 +-- mitmproxy/utils/typecheck.py | 19 +-- test/helper_tools/dumperview.py | 4 +- test/mitmproxy/addons/test_dumper.py | 16 +- test/mitmproxy/addons/test_intercept.py | 8 +- test/mitmproxy/addons/test_streamfile.py | 4 +- test/mitmproxy/addons/test_termlog.py | 2 +- test/mitmproxy/addons/test_view.py | 25 +-- test/mitmproxy/test_optmanager.py | 120 ++++++++------- test/mitmproxy/tools/test_dump.py | 9 +- test/mitmproxy/utils/test_typecheck.py | 6 - 14 files changed, 266 insertions(+), 381 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index ff17fbbf..16009316 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -21,186 +21,94 @@ DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA class Options(optmanager.OptManager): - def __init__( - self, - *, # all args are keyword-only. - onboarding: bool = True, - onboarding_host: str = APP_HOST, - onboarding_port: int = APP_PORT, - anticache: bool = False, - anticomp: bool = False, - client_replay: Sequence[str] = [], - replay_kill_extra: bool = False, - keepserving: bool = True, - no_server: bool = False, - server_replay_nopop: bool = False, - refresh_server_playback: bool = True, - rfile: Optional[str] = None, - scripts: Sequence[str] = [], - showhost: bool = False, - replacements: Sequence[Union[Tuple[str, str, str], str]] = [], - replacement_files: Sequence[Union[Tuple[str, str, str], str]] = [], - server_replay_use_headers: Sequence[str] = [], - setheaders: Sequence[Union[Tuple[str, str, str], str]] = [], - server_replay: Sequence[str] = [], - stickycookie: Optional[str] = None, - stickyauth: Optional[str] = None, - stream_large_bodies: Optional[int] = None, - verbosity: int = 2, - default_contentview: str = "auto", - streamfile: Optional[str] = None, - streamfile_append: bool = False, - server_replay_ignore_content: bool = False, - server_replay_ignore_params: Sequence[str] = [], - server_replay_ignore_payload_params: Sequence[str] = [], - server_replay_ignore_host: bool = False, - - # Proxy options - auth_nonanonymous: bool = False, - auth_singleuser: Optional[str] = None, - auth_htpasswd: Optional[str] = None, - add_upstream_certs_to_client_chain: bool = False, - body_size_limit: Optional[int] = None, - cadir: str = CA_DIR, - certs: Sequence[Tuple[str, str]] = [], - ciphers_client: str=DEFAULT_CLIENT_CIPHERS, - ciphers_server: Optional[str]=None, - clientcerts: Optional[str] = None, - ignore_hosts: Sequence[str] = [], - listen_host: str = "", - listen_port: int = LISTEN_PORT, - upstream_bind_address: str = "", - mode: str = "regular", - no_upstream_cert: bool = False, - keep_host_header: bool = False, - - http2: bool = True, - http2_priority: bool = False, - websocket: bool = True, - rawtcp: bool = False, - - spoof_source_address: bool = False, - upstream_server: Optional[str] = None, - upstream_auth: Optional[str] = None, - ssl_version_client: str = "secure", - ssl_version_server: str = "secure", - ssl_insecure: bool = False, - ssl_verify_upstream_trusted_cadir: Optional[str] = None, - ssl_verify_upstream_trusted_ca: Optional[str] = None, - tcp_hosts: Sequence[str] = [], - - intercept: Optional[str] = None, - - # Console options - console_eventlog: bool = False, - console_focus_follow: bool = False, - console_palette: Optional[str] = "dark", - console_palette_transparent: bool = False, - console_no_mouse: bool = False, - console_order: Optional[str] = None, - console_order_reversed: bool = False, - - filter: Optional[str] = None, - - # Web options - web_open_browser: bool = True, - web_debug: bool = False, - web_port: int = 8081, - web_iface: str = "127.0.0.1", - - # Dump options - filtstr: Optional[str] = None, - flow_detail: int = 1 - ) -> None: - # We could replace all assignments with clever metaprogramming, - # but type hints are a much more valueable asset. - - self.onboarding = onboarding - self.onboarding_host = onboarding_host - self.onboarding_port = onboarding_port - self.anticache = anticache - self.anticomp = anticomp - self.client_replay = client_replay - self.keepserving = keepserving - self.replay_kill_extra = replay_kill_extra - self.no_server = no_server - self.server_replay_nopop = server_replay_nopop - self.refresh_server_playback = refresh_server_playback - self.rfile = rfile - self.scripts = scripts - self.showhost = showhost - self.replacements = replacements - self.replacement_files = replacement_files - self.server_replay_use_headers = server_replay_use_headers - self.setheaders = setheaders - self.server_replay = server_replay - self.stickycookie = stickycookie - self.stickyauth = stickyauth - self.stream_large_bodies = stream_large_bodies - self.verbosity = verbosity - self.default_contentview = default_contentview - self.streamfile = streamfile - self.streamfile_append = streamfile_append - self.server_replay_ignore_content = server_replay_ignore_content - self.server_replay_ignore_params = server_replay_ignore_params - self.server_replay_ignore_payload_params = server_replay_ignore_payload_params - self.server_replay_ignore_host = server_replay_ignore_host + def __init__(self, **kwargs) -> None: + super().__init__() + self.add_option("onboarding", True, bool) + self.add_option("onboarding_host", APP_HOST, str) + self.add_option("onboarding_port", APP_PORT, int) + self.add_option("anticache", False, bool) + self.add_option("anticomp", False, bool) + self.add_option("client_replay", [], Sequence[str]) + self.add_option("replay_kill_extra", False, bool) + self.add_option("keepserving", True, bool) + self.add_option("no_server", False, bool) + self.add_option("server_replay_nopop", False, bool) + self.add_option("refresh_server_playback", True, bool) + self.add_option("rfile", None, Optional[str]) + self.add_option("scripts", [], Sequence[str]) + self.add_option("showhost", False, bool) + self.add_option("replacements", [], Sequence[Union[Tuple[str, str, str], str]]) + self.add_option("replacement_files", [], Sequence[Union[Tuple[str, str, str], str]]) + self.add_option("server_replay_use_headers", [], Sequence[str]) + self.add_option("setheaders", [], Sequence[Union[Tuple[str, str, str], str]]) + self.add_option("server_replay", [], Sequence[str]) + self.add_option("stickycookie", None, Optional[str]) + self.add_option("stickyauth", None, Optional[str]) + self.add_option("stream_large_bodies", None, Optional[int]) + self.add_option("verbosity", 2, int) + self.add_option("default_contentview", "auto", str) + self.add_option("streamfile", None, Optional[str]) + self.add_option("streamfile_append", False, bool) + self.add_option("server_replay_ignore_content", False, bool) + self.add_option("server_replay_ignore_params", [], Sequence[str]) + self.add_option("server_replay_ignore_payload_params", [], Sequence[str]) + self.add_option("server_replay_ignore_host", False, bool) # Proxy options - self.auth_nonanonymous = auth_nonanonymous - self.auth_singleuser = auth_singleuser - self.auth_htpasswd = auth_htpasswd - self.add_upstream_certs_to_client_chain = add_upstream_certs_to_client_chain - self.body_size_limit = body_size_limit - self.cadir = cadir - self.certs = certs - self.ciphers_client = ciphers_client - self.ciphers_server = ciphers_server - self.clientcerts = clientcerts - self.ignore_hosts = ignore_hosts - self.listen_host = listen_host - self.listen_port = listen_port - self.upstream_bind_address = upstream_bind_address - self.mode = mode - self.no_upstream_cert = no_upstream_cert - self.keep_host_header = keep_host_header - - self.http2 = http2 - self.http2_priority = http2_priority - self.websocket = websocket - self.rawtcp = rawtcp - - self.spoof_source_address = spoof_source_address - self.upstream_server = upstream_server - self.upstream_auth = upstream_auth - self.ssl_version_client = ssl_version_client - self.ssl_version_server = ssl_version_server - self.ssl_insecure = ssl_insecure - self.ssl_verify_upstream_trusted_cadir = ssl_verify_upstream_trusted_cadir - self.ssl_verify_upstream_trusted_ca = ssl_verify_upstream_trusted_ca - self.tcp_hosts = tcp_hosts - - self.intercept = intercept + self.add_option("auth_nonanonymous", False, bool) + self.add_option("auth_singleuser", None, Optional[str]) + self.add_option("auth_htpasswd", None, Optional[str]) + self.add_option("add_upstream_certs_to_client_chain", False, bool) + self.add_option("body_size_limit", None, Optional[int]) + self.add_option("cadir", CA_DIR, str) + self.add_option("certs", [], Sequence[Tuple[str, str]]) + self.add_option("ciphers_client", DEFAULT_CLIENT_CIPHERS, str) + self.add_option("ciphers_server", None, Optional[str]) + self.add_option("clientcerts", None, Optional[str]) + self.add_option("ignore_hosts", [], Sequence[str]) + self.add_option("listen_host", "", str) + self.add_option("listen_port", LISTEN_PORT, int) + self.add_option("upstream_bind_address", "", str) + self.add_option("mode", "regular", str) + self.add_option("no_upstream_cert", False, bool) + self.add_option("keep_host_header", False, bool) + + self.add_option("http2", True, bool) + self.add_option("http2_priority", False, bool) + self.add_option("websocket", True, bool) + self.add_option("rawtcp", False, bool) + + self.add_option("spoof_source_address", False, bool) + self.add_option("upstream_server", None, Optional[str]) + self.add_option("upstream_auth", None, Optional[str]) + self.add_option("ssl_version_client", "secure", str) + self.add_option("ssl_version_server", "secure", str) + self.add_option("ssl_insecure", False, bool) + self.add_option("ssl_verify_upstream_trusted_cadir", None, Optional[str]) + self.add_option("ssl_verify_upstream_trusted_ca", None, Optional[str]) + self.add_option("tcp_hosts", [], Sequence[str]) + + self.add_option("intercept", None, Optional[str]) # Console options - self.console_eventlog = console_eventlog - self.console_focus_follow = console_focus_follow - self.console_palette = console_palette - self.console_palette_transparent = console_palette_transparent - self.console_no_mouse = console_no_mouse - self.console_order = console_order - self.console_order_reversed = console_order_reversed + self.add_option("console_eventlog", False, bool) + self.add_option("console_focus_follow", False, bool) + self.add_option("console_palette", "dark", Optional[str]) + self.add_option("console_palette_transparent", False, bool) + self.add_option("console_no_mouse", False, bool) + self.add_option("console_order", None, Optional[str]) + self.add_option("console_order_reversed", False, bool) - self.filter = filter + self.add_option("filter", None, Optional[str]) # Web options - self.web_open_browser = web_open_browser - self.web_debug = web_debug - self.web_port = web_port - self.web_iface = web_iface + self.add_option("web_open_browser", True, bool) + self.add_option("web_debug", False, bool) + self.add_option("web_port", 8081, int) + self.add_option("web_iface", "127.0.0.1", str) # Dump options - self.filtstr = filtstr - self.flow_detail = flow_detail + self.add_option("filtstr", None, Optional[str]) + self.add_option("flow_detail", 1, int) - super().__init__() + self.update(**kwargs) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index f95ce836..4b5a710e 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -1,11 +1,11 @@ import contextlib import blinker import pprint -import inspect import copy import functools import weakref import os +import typing import ruamel.yaml @@ -17,21 +17,62 @@ from mitmproxy.utils import typecheck The base implementation for Options. """ +unset = object() -class _DefaultsMeta(type): - def __new__(cls, name, bases, namespace, **kwds): - ret = type.__new__(cls, name, bases, dict(namespace)) - defaults = {} - for klass in reversed(inspect.getmro(ret)): - for p in inspect.signature(klass.__init__).parameters.values(): - if p.kind in (p.KEYWORD_ONLY, p.POSITIONAL_OR_KEYWORD): - if not p.default == p.empty: - defaults[p.name] = p.default - ret._defaults = defaults - return ret +class _Option: + __slots__ = ("name", "typespec", "value", "_default") -class OptManager(metaclass=_DefaultsMeta): + def __init__( + self, + name: str, + default: typing.Any, + typespec: typing.Type + ) -> None: + typecheck.check_type(name, default, typespec) + self.name = name + self._default = default + self.typespec = typespec + self.value = unset + + def __repr__(self): + return "{value} [{type}]".format(value=self.current(), type=self.typespec) + + @property + def default(self): + return copy.deepcopy(self._default) + + def current(self) -> typing.Any: + if self.value is unset: + v = self.default + else: + v = self.value + return copy.deepcopy(v) + + def set(self, value: typing.Any) -> None: + typecheck.check_type(self.name, value, self.typespec) + self.value = value + + def reset(self) -> None: + self.value = unset + + def has_changed(self) -> bool: + return self.value is not unset + + def __eq__(self, other) -> bool: + for i in self.__slots__: + if getattr(self, i) != getattr(other, i): + return False + return True + + def __deepcopy__(self, _): + o = _Option(self.name, self.default, self.typespec) + if self.has_changed(): + o.value = self.current() + return o + + +class OptManager: """ OptManager is the base class from which Options objects are derived. Note that the __init__ method of all child classes must force all @@ -45,32 +86,26 @@ class OptManager(metaclass=_DefaultsMeta): Optmanager always returns a deep copy of options to ensure that mutation doesn't change the option state inadvertently. """ - _initialized = False - attributes = [] - - def __new__(cls, *args, **kwargs): - # Initialize instance._opts before __init__ is called. - # This allows us to call super().__init__() last, which then sets - # ._initialized = True as the final operation. - instance = super().__new__(cls) - instance.__dict__["_opts"] = {} - return instance - def __init__(self): + self.__dict__["_options"] = {} self.__dict__["changed"] = blinker.Signal() self.__dict__["errored"] = blinker.Signal() - self.__dict__["_initialized"] = True + + def add_option(self, name: str, default: typing.Any, typespec: typing.Type) -> None: + if name in self._options: + raise ValueError("Option %s already exists" % name) + self._options[name] = _Option(name, default, typespec) @contextlib.contextmanager def rollback(self, updated): - old = self._opts.copy() + old = copy.deepcopy(self._options) try: yield except exceptions.OptionsError as e: # Notify error handlers self.errored.send(self, exc=e) # Rollback - self.__dict__["_opts"] = old + self.__dict__["_options"] = old self.changed.send(self, updated=updated) def subscribe(self, func, opts): @@ -95,61 +130,48 @@ class OptManager(metaclass=_DefaultsMeta): self.changed.connect(_call, weak=False) def __eq__(self, other): - return self._opts == other._opts + return self._options == other._options def __copy__(self): - return self.__class__(**self._opts) + o = OptManager() + o.__dict__["_options"] = copy.deepcopy(self._options) + return o def __getattr__(self, attr): - if attr in self._opts: - return copy.deepcopy(self._opts[attr]) + if attr in self._options: + return self._options[attr].current() else: raise AttributeError("No such option: %s" % attr) def __setattr__(self, attr, value): - if not self._initialized: - self._typecheck(attr, value) - self._opts[attr] = value - return self.update(**{attr: value}) - def _typecheck(self, attr, value): - expected_type = typecheck.get_arg_type_from_constructor_annotation( - type(self), attr - ) - if expected_type is None: - return # no type info :( - typecheck.check_type(attr, value, expected_type) - def keys(self): - return set(self._opts.keys()) + return set(self._options.keys()) def reset(self): """ Restore defaults for all options. """ - self.update(**self._defaults) - - @classmethod - def default(klass, opt): - return copy.deepcopy(klass._defaults[opt]) + for o in self._options.values(): + o.reset() def update(self, **kwargs): updated = set(kwargs.keys()) - for k, v in kwargs.items(): - if k not in self._opts: - raise KeyError("No such option: %s" % k) - self._typecheck(k, v) with self.rollback(updated): - self._opts.update(kwargs) + for k, v in kwargs.items(): + if k not in self._options: + raise KeyError("No such option: %s" % k) + self._options[k].set(v) self.changed.send(self, updated=updated) + return self def setter(self, attr): """ Generate a setter for a given attribute. This returns a callable taking a single argument. """ - if attr not in self._opts: + if attr not in self._options: raise KeyError("No such option: %s" % attr) def setter(x): @@ -161,19 +183,24 @@ class OptManager(metaclass=_DefaultsMeta): Generate a toggler for a boolean attribute. This returns a callable that takes no arguments. """ - if attr not in self._opts: + if attr not in self._options: raise KeyError("No such option: %s" % attr) + o = self._options[attr] + if o.typespec != bool: + raise ValueError("Toggler can only be used with boolean options") def toggle(): setattr(self, attr, not getattr(self, attr)) return toggle + def default(self, option: str) -> typing.Any: + return self._options[option].default + def has_changed(self, option): """ Has the option changed from the default? """ - if getattr(self, option) != self._defaults[option]: - return True + return self._options[option].has_changed() def save(self, path, defaults=False): """ @@ -204,7 +231,7 @@ class OptManager(metaclass=_DefaultsMeta): if defaults or self.has_changed(k): data[k] = getattr(self, k) for k in list(data.keys()): - if k not in self._opts: + if k not in self._options: del data[k] return ruamel.yaml.round_trip_dump(data) @@ -268,7 +295,7 @@ class OptManager(metaclass=_DefaultsMeta): self.update(**toset) def __repr__(self): - options = pprint.pformat(self._opts, indent=4).strip(" {}") + options = pprint.pformat(self._options, indent=4).strip(" {}") if "\n" in options: options = "\n " + options + "\n" return "{mod}.{cls}({{{options}}})".format( diff --git a/mitmproxy/test/taddons.py b/mitmproxy/test/taddons.py index bb8daa02..c3e19cc7 100644 --- a/mitmproxy/test/taddons.py +++ b/mitmproxy/test/taddons.py @@ -4,7 +4,6 @@ import mitmproxy.master import mitmproxy.options from mitmproxy import proxy from mitmproxy import eventsequence -from mitmproxy import exceptions class RecordingMaster(mitmproxy.master.Master): @@ -43,14 +42,6 @@ class context: return False @contextlib.contextmanager - def _rollback(self, opts, updates): - old = opts._opts.copy() - try: - yield - except exceptions.OptionsError as e: - opts.__dict__["_opts"] = old - raise - def cycle(self, addon, f): """ Cycles the flow through the events for the flow. Stops if a reply @@ -70,6 +61,5 @@ class context: Options object with the given keyword arguments, then calls the configure method on the addon with the updated value. """ - with self._rollback(self.options, kwargs): - self.options.update(**kwargs) - addon.configure(self.options, kwargs.keys()) + self.options.update(**kwargs) + addon.configure(self.options, kwargs.keys()) diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index fefbddfb..6b862475 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -1,5 +1,3 @@ -from typing import Optional - from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import addons @@ -12,26 +10,11 @@ class DumpError(Exception): pass -class Options(options.Options): - def __init__( - self, - *, # all args are keyword-only. - keepserving: bool = False, - filtstr: Optional[str] = None, - flow_detail: int = 1, - **kwargs - ) -> None: - self.filtstr = filtstr - self.flow_detail = flow_detail - self.keepserving = keepserving - super().__init__(**kwargs) - - class DumpMaster(master.Master): def __init__( self, - options: Options, + options: options.Options, server, with_termlog=True, with_dumper=True, diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py index 2cdf7f51..bdd83ee6 100644 --- a/mitmproxy/utils/typecheck.py +++ b/mitmproxy/utils/typecheck.py @@ -1,7 +1,7 @@ import typing -def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: +def check_type(name: str, value: typing.Any, typeinfo: type) -> None: """ This function checks if the provided value is an instance of typeinfo and raises a TypeError otherwise. @@ -17,7 +17,7 @@ def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: e = TypeError("Expected {} for {}, but got {}.".format( typeinfo, - attr_name, + name, type(value) )) @@ -32,7 +32,7 @@ def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: for T in types: try: - check_type(attr_name, value, T) + check_type(name, value, T) except TypeError: pass else: @@ -50,7 +50,7 @@ def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: if len(types) != len(value): raise e for i, (x, T) in enumerate(zip(value, types)): - check_type("{}[{}]".format(attr_name, i), x, T) + check_type("{}[{}]".format(name, i), x, T) return elif typename.startswith("typing.Sequence"): try: @@ -62,7 +62,7 @@ def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: if not isinstance(value, (tuple, list)): raise e for v in value: - check_type(attr_name, v, T) + check_type(name, v, T) elif typename.startswith("typing.IO"): if hasattr(value, "read"): return @@ -70,12 +70,3 @@ def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: raise e elif not isinstance(value, typeinfo): raise e - - -def get_arg_type_from_constructor_annotation(cls: type, attr: str) -> typing.Optional[type]: - """ - Returns the first type annotation for attr in the class hierarchy. - """ - for c in cls.mro(): - if attr in getattr(c.__init__, "__annotations__", ()): - return c.__init__.__annotations__[attr] diff --git a/test/helper_tools/dumperview.py b/test/helper_tools/dumperview.py index be56fe14..d417d767 100755 --- a/test/helper_tools/dumperview.py +++ b/test/helper_tools/dumperview.py @@ -4,12 +4,12 @@ import click from mitmproxy.addons import dumper from mitmproxy.test import tflow from mitmproxy.test import taddons -from mitmproxy.tools import dump +from mitmproxy.tools import options def show(flow_detail, flows): d = dumper.Dumper() - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=flow_detail) for f in flows: ctx.cycle(d, f) diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index 22d2c2c6..47374617 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -9,13 +9,13 @@ from mitmproxy.test import tutils from mitmproxy.addons import dumper from mitmproxy import exceptions -from mitmproxy.tools import dump from mitmproxy import http +from mitmproxy import options def test_configure(): d = dumper.Dumper() - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, filtstr="~b foo") assert d.filter @@ -34,7 +34,7 @@ def test_configure(): def test_simple(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=0) d.response(tflow.tflow(resp=True)) assert not sio.getvalue() @@ -103,7 +103,7 @@ def test_echo_body(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=3) d._echo_message(f.response) t = sio.getvalue() @@ -113,7 +113,7 @@ def test_echo_body(): def test_echo_request_line(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.tflow(client_conn=None, server_conn=True, resp=True) f.request.is_replay = True @@ -148,7 +148,7 @@ class TestContentView: view_auto.side_effect = exceptions.ContentViewException("") sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=4, verbosity=3) d.response(tflow.tflow()) assert "Content viewer failed" in ctx.master.event_log[0][1] @@ -157,7 +157,7 @@ class TestContentView: def test_tcp(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.ttcpflow() d.tcp_message(f) @@ -172,7 +172,7 @@ def test_tcp(): def test_websocket(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=dump.Options()) as ctx: + with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.twebsocketflow() d.websocket_message(f) diff --git a/test/mitmproxy/addons/test_intercept.py b/test/mitmproxy/addons/test_intercept.py index cf5ba6e8..465e6433 100644 --- a/test/mitmproxy/addons/test_intercept.py +++ b/test/mitmproxy/addons/test_intercept.py @@ -7,15 +7,9 @@ from mitmproxy.test import taddons from mitmproxy.test import tflow -class Options(options.Options): - def __init__(self, *, intercept=None, **kwargs): - self.intercept = intercept - super().__init__(**kwargs) - - def test_simple(): r = intercept.Intercept() - with taddons.context(options=Options()) as tctx: + with taddons.context(options=options.Options()) as tctx: assert not r.filt tctx.configure(r, intercept="~q") assert r.filt diff --git a/test/mitmproxy/addons/test_streamfile.py b/test/mitmproxy/addons/test_streamfile.py index 4922fc0b..89dc2af3 100644 --- a/test/mitmproxy/addons/test_streamfile.py +++ b/test/mitmproxy/addons/test_streamfile.py @@ -7,13 +7,13 @@ from mitmproxy.test import taddons from mitmproxy import io from mitmproxy import exceptions -from mitmproxy.tools import dump +from mitmproxy import options from mitmproxy.addons import streamfile def test_configure(): sa = streamfile.StreamFile() - with taddons.context(options=dump.Options()) as tctx: + with taddons.context(options=options.Options()) as tctx: with tutils.tmpdir() as tdir: p = os.path.join(tdir, "foo") with pytest.raises(exceptions.OptionsError): diff --git a/test/mitmproxy/addons/test_termlog.py b/test/mitmproxy/addons/test_termlog.py index 70c3a7f2..2133b74d 100644 --- a/test/mitmproxy/addons/test_termlog.py +++ b/test/mitmproxy/addons/test_termlog.py @@ -3,7 +3,7 @@ import pytest from mitmproxy.addons import termlog from mitmproxy import log -from mitmproxy.tools.dump import Options +from mitmproxy.options import Options from mitmproxy.test import taddons diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index a063416f..b7842314 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -15,23 +15,6 @@ def tft(*, method="get", start=0): return f -class Options(options.Options): - def __init__( - self, - *, - filter=None, - console_order=None, - console_order_reversed=False, - console_focus_follow=False, - **kwargs - ): - self.filter = filter - self.console_order = console_order - self.console_order_reversed = console_order_reversed - self.console_focus_follow = console_focus_follow - super().__init__(**kwargs) - - def test_order_refresh(): v = view.View() sargs = [] @@ -42,7 +25,7 @@ def test_order_refresh(): v.sig_view_refresh.connect(save) tf = tflow.tflow(resp=True) - with taddons.context(options=Options()) as tctx: + with taddons.context(options=options.Options()) as tctx: tctx.configure(v, console_order="time") v.add(tf) tf.request.timestamp_start = 1 @@ -149,7 +132,7 @@ def test_filter(): def test_order(): v = view.View() - with taddons.context(options=Options()) as tctx: + with taddons.context(options=options.Options()) as tctx: v.request(tft(method="get", start=1)) v.request(tft(method="put", start=2)) v.request(tft(method="get", start=3)) @@ -280,7 +263,7 @@ def test_signals(): def test_focus_follow(): v = view.View() - with taddons.context(options=Options()) as tctx: + with taddons.context(options=options.Options()) as tctx: tctx.configure(v, console_focus_follow=True, filter="~m get") v.add(tft(start=5)) @@ -394,7 +377,7 @@ def test_settings(): def test_configure(): v = view.View() - with taddons.context(options=Options()) as tctx: + with taddons.context(options=options.Options()) as tctx: tctx.configure(v, filter="~q") with pytest.raises(Exception, match="Invalid interception filter"): tctx.configure(v, filter="~~") diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 161b0dcf..3fba304a 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -1,6 +1,7 @@ import copy import os import pytest +import typing from mitmproxy import options from mitmproxy import optmanager @@ -9,48 +10,45 @@ from mitmproxy.test import tutils class TO(optmanager.OptManager): - def __init__(self, one=None, two=None): - self.one = one - self.two = two + def __init__(self): super().__init__() + self.add_option("one", None, typing.Optional[int]) + self.add_option("two", 2, typing.Optional[int]) + self.add_option("bool", False, bool) class TD(optmanager.OptManager): - def __init__(self, *, one="done", two="dtwo", three="error"): - self.one = one - self.two = two - self.three = three + def __init__(self): super().__init__() + self.add_option("one", "done", str) + self.add_option("two", "dtwo", str) class TD2(TD): - def __init__(self, *, three="dthree", four="dfour", **kwargs): - self.three = three - self.four = four - super().__init__(three=three, **kwargs) + def __init__(self): + super().__init__() + self.add_option("three", "dthree", str) + self.add_option("four", "dfour", str) class TM(optmanager.OptManager): - def __init__(self, one="one", two=["foo"], three=None): - self.one = one - self.two = two - self.three = three + def __init__(self): super().__init__() + self.add_option("two", ["foo"], typing.Sequence[str]) + self.add_option("one", None, typing.Optional[str]) def test_defaults(): - assert TD2.default("one") == "done" - assert TD2.default("two") == "dtwo" - assert TD2.default("three") == "dthree" - assert TD2.default("four") == "dfour" - o = TD2() - assert o._defaults == { + defaults = { "one": "done", "two": "dtwo", "three": "dthree", "four": "dfour", } + for k, v in defaults.items(): + assert o.default(k) == v + assert not o.has_changed("one") newvals = dict( one="xone", @@ -64,18 +62,19 @@ def test_defaults(): assert v == getattr(o, k) o.reset() assert not o.has_changed("one") - for k, v in o._defaults.items(): - assert v == getattr(o, k) + + for k in o.keys(): + assert not o.has_changed(k) def test_options(): - o = TO(two="three") - assert o.keys() == set(["one", "two"]) + o = TO() + assert o.keys() == set(["bool", "one", "two"]) assert o.one is None - assert o.two == "three" - o.one = "one" - assert o.one == "one" + assert o.two == 2 + o.one = 1 + assert o.one == 1 with pytest.raises(TypeError): TO(nonexistent = "value") @@ -91,34 +90,38 @@ def test_options(): o.changed.connect(sub) - o.one = "ninety" + o.one = 90 assert len(rec) == 1 - assert rec[-1].one == "ninety" + assert rec[-1].one == 90 - o.update(one="oink") + o.update(one=3) assert len(rec) == 2 - assert rec[-1].one == "oink" + assert rec[-1].one == 3 def test_setter(): - o = TO(two="three") + o = TO() f = o.setter("two") - f("xxx") - assert o.two == "xxx" + f(99) + assert o.two == 99 with pytest.raises(Exception, match="No such option"): o.setter("nonexistent") def test_toggler(): - o = TO(two=True) - f = o.toggler("two") + o = TO() + f = o.toggler("bool") + assert o.bool is False f() - assert o.two is False + assert o.bool is True f() - assert o.two is True + assert o.bool is False with pytest.raises(Exception, match="No such option"): o.toggler("nonexistent") + with pytest.raises(Exception, match="boolean options"): + o.toggler("one") + class Rec(): def __init__(self): @@ -132,19 +135,19 @@ def test_subscribe(): o = TO() r = Rec() o.subscribe(r, ["two"]) - o.one = "foo" + o.one = 2 assert not r.called - o.two = "foo" + o.two = 3 assert r.called assert len(o.changed.receivers) == 1 del r - o.two = "bar" + o.two = 4 assert len(o.changed.receivers) == 0 def test_rollback(): - o = TO(one="two") + o = TO() rec = [] @@ -157,27 +160,24 @@ def test_rollback(): recerr.append(kwargs) def err(opts, updated): - if opts.one == "ten": + if opts.one == 10: raise exceptions.OptionsError() o.changed.connect(sub) o.changed.connect(err) o.errored.connect(errsub) - o.one = "ten" + assert o.one is None + o.one = 10 assert isinstance(recerr[0]["exc"], exceptions.OptionsError) - assert o.one == "two" + assert o.one is None assert len(rec) == 2 - assert rec[0].one == "ten" - assert rec[1].one == "two" + assert rec[0].one == 10 + assert rec[1].one is None def test_repr(): - assert repr(TO()) == "test.mitmproxy.test_optmanager.TO({'one': None, 'two': None})" - assert repr(TO(one='x' * 60)) == """test.mitmproxy.test_optmanager.TO({ - 'one': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', - 'two': None -})""" + assert repr(TO()) def test_serialize(): @@ -249,3 +249,17 @@ def test_merge(): assert m.one == "two" m.merge(dict(two=["bar"])) assert m.two == ["foo", "bar"] + + +def test_option(): + o = optmanager._Option("test", 1, int) + assert o.current() == 1 + with pytest.raises(TypeError): + o.set("foo") + with pytest.raises(TypeError): + optmanager._Option("test", 1, str) + + o2 = optmanager._Option("test", 1, int) + assert o2 == o + o2.set(5) + assert o2 != o diff --git a/test/mitmproxy/tools/test_dump.py b/test/mitmproxy/tools/test_dump.py index b4183725..3210b0bb 100644 --- a/test/mitmproxy/tools/test_dump.py +++ b/test/mitmproxy/tools/test_dump.py @@ -5,6 +5,7 @@ from unittest import mock from mitmproxy import proxy from mitmproxy import log from mitmproxy import controller +from mitmproxy import options from mitmproxy.tools import dump from mitmproxy.test import tutils @@ -12,8 +13,8 @@ from .. import tservers class TestDumpMaster(tservers.MasterTest): - def mkmaster(self, flt, **options): - o = dump.Options(filtstr=flt, verbosity=-1, flow_detail=0, **options) + def mkmaster(self, flt, **opts): + o = options.Options(filtstr=flt, verbosity=-1, flow_detail=0, **opts) m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=False, with_dumper=False) return m @@ -40,13 +41,13 @@ class TestDumpMaster(tservers.MasterTest): @pytest.mark.parametrize("termlog", [False, True]) def test_addons_termlog(self, termlog): with mock.patch('sys.stdout'): - o = dump.Options() + o = options.Options() m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=termlog) assert (m.addons.get('termlog') is not None) == termlog @pytest.mark.parametrize("dumper", [False, True]) def test_addons_dumper(self, dumper): with mock.patch('sys.stdout'): - o = dump.Options() + o = options.Options() m = dump.DumpMaster(o, proxy.DummyServer(), with_dumper=dumper) assert (m.addons.get('dumper') is not None) == dumper diff --git a/test/mitmproxy/utils/test_typecheck.py b/test/mitmproxy/utils/test_typecheck.py index 67981be4..d99a914f 100644 --- a/test/mitmproxy/utils/test_typecheck.py +++ b/test/mitmproxy/utils/test_typecheck.py @@ -16,12 +16,6 @@ class T(TBase): super(T, self).__init__(42) -def test_get_arg_type_from_constructor_annotation(): - assert typecheck.get_arg_type_from_constructor_annotation(T, "foo") == str - assert typecheck.get_arg_type_from_constructor_annotation(T, "bar") == int - assert not typecheck.get_arg_type_from_constructor_annotation(T, "baz") - - def test_check_type(): typecheck.check_type("foo", 42, int) with pytest.raises(TypeError): -- cgit v1.2.3 From f15a6285613540e031b004b726799dd6edee5a27 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 5 Mar 2017 20:10:06 +1300 Subject: Start unifying options and the command-line: booleans This commit: - Adds a help field to options - Adds a function to generate parser definitions from options - Uses this to migrate all boolean flags over to the new system - Makes all booleans consistently follow the --foo/--not-foo convention There are a number of things left to be done here: - Argparse doesn't give us a nice way to format --foo --not-foo help. Click does, and moving to click is a goal down the track. - For now, we remove all short aliases. I want to re-evaluate these systematically once we have the new structure in place. --- mitmproxy/options.py | 151 ++++++++++++--- mitmproxy/optmanager.py | 39 +++- mitmproxy/proxy/protocol/tls.py | 4 +- mitmproxy/tools/cmdline.py | 243 ++++++------------------ mitmproxy/tools/console/master.py | 2 +- mitmproxy/tools/console/options.py | 6 +- mitmproxy/tools/console/statusbar.py | 2 +- mitmproxy/tools/main.py | 15 +- mitmproxy/tools/web/app.py | 4 +- test/mitmproxy/proxy/protocol/test_http2.py | 2 +- test/mitmproxy/proxy/protocol/test_websocket.py | 2 +- test/mitmproxy/proxy/test_server.py | 4 +- test/mitmproxy/test_optmanager.py | 6 +- test/mitmproxy/test_proxy.py | 6 +- test/mitmproxy/tools/test_cmdline.py | 23 ++- tox.ini | 1 + 16 files changed, 255 insertions(+), 255 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 16009316..1d879ac8 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -23,20 +23,50 @@ DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA class Options(optmanager.OptManager): def __init__(self, **kwargs) -> None: super().__init__() - self.add_option("onboarding", True, bool) + self.add_option( + "onboarding", True, bool, + "Toggle the mitmproxy onboarding app." + ) self.add_option("onboarding_host", APP_HOST, str) self.add_option("onboarding_port", APP_PORT, int) - self.add_option("anticache", False, bool) - self.add_option("anticomp", False, bool) + self.add_option( + "anticache", False, bool, + """ + Strip out request headers that might cause the server to return + 304-not-modified. + """ + ) + self.add_option( + "anticomp", False, bool, + "Try to convince servers to send us un-compressed data." + ) self.add_option("client_replay", [], Sequence[str]) - self.add_option("replay_kill_extra", False, bool) - self.add_option("keepserving", True, bool) - self.add_option("no_server", False, bool) - self.add_option("server_replay_nopop", False, bool) - self.add_option("refresh_server_playback", True, bool) + self.add_option( + "replay_kill_extra", False, bool, + "Kill extra requests during replay." + ) + self.add_option( + "keepserving", True, bool, + "Continue serving after client playback or file read." + ) + self.add_option( + "no_server", False, bool, + "Don't start a proxy server." + ) + self.add_option( + "server_replay_nopop", False, bool, + "Disable response pop from response flow. " + "This makes it possible to replay same response multiple times." + ) + self.add_option( + "refresh_server_playback", True, bool, + ) self.add_option("rfile", None, Optional[str]) self.add_option("scripts", [], Sequence[str]) - self.add_option("showhost", False, bool) + self.add_option( + "showhost", False, bool, + "Use the Host header to construct URLs for display." + ) self.add_option("replacements", [], Sequence[Union[Tuple[str, str, str], str]]) self.add_option("replacement_files", [], Sequence[Union[Tuple[str, str, str], str]]) self.add_option("server_replay_use_headers", [], Sequence[str]) @@ -49,16 +79,30 @@ class Options(optmanager.OptManager): self.add_option("default_contentview", "auto", str) self.add_option("streamfile", None, Optional[str]) self.add_option("streamfile_append", False, bool) - self.add_option("server_replay_ignore_content", False, bool) + self.add_option( + "server_replay_ignore_content", False, bool, + "Ignore request's content while searching for a saved flow to replay." + ) self.add_option("server_replay_ignore_params", [], Sequence[str]) self.add_option("server_replay_ignore_payload_params", [], Sequence[str]) - self.add_option("server_replay_ignore_host", False, bool) + self.add_option( + "server_replay_ignore_host", False, bool, + "Ignore request's destination host while searching for a saved" + " flow to replay" + ) # Proxy options - self.add_option("auth_nonanonymous", False, bool) + self.add_option( + "auth_nonanonymous", False, bool, + "Allow access to any user long as a credentials are specified." + ) self.add_option("auth_singleuser", None, Optional[str]) self.add_option("auth_htpasswd", None, Optional[str]) - self.add_option("add_upstream_certs_to_client_chain", False, bool) + self.add_option( + "add_upstream_certs_to_client_chain", False, bool, + "Add all certificates of the upstream server to the certificate chain " + "that will be served to the proxy client, as extras." + ) self.add_option("body_size_limit", None, Optional[int]) self.add_option("cadir", CA_DIR, str) self.add_option("certs", [], Sequence[Tuple[str, str]]) @@ -70,20 +114,51 @@ class Options(optmanager.OptManager): self.add_option("listen_port", LISTEN_PORT, int) self.add_option("upstream_bind_address", "", str) self.add_option("mode", "regular", str) - self.add_option("no_upstream_cert", False, bool) - self.add_option("keep_host_header", False, bool) + self.add_option( + "upstream_cert", True, bool, + "Connect to upstream server to look up certificate details." + ) + self.add_option( + "keep_host_header", False, bool, + "Reverse Proxy: Keep the original host header instead of rewriting it" + " to the reverse proxy target." + ) - self.add_option("http2", True, bool) - self.add_option("http2_priority", False, bool) - self.add_option("websocket", True, bool) - self.add_option("rawtcp", False, bool) + self.add_option( + "http2", True, bool, + "Enable/disable HTTP/2 support. " + "HTTP/2 support is enabled by default.", + ) + self.add_option( + "http2_priority", False, bool, + "Enable/disable PRIORITY forwarding for HTTP/2 connections. " + "PRIORITY forwarding is disabled by default, " + "because some webservers fail to implement the RFC properly.", + ) + self.add_option( + "websocket", True, bool, + "Enable/disable WebSocket support. " + "WebSocket support is enabled by default.", + ) + self.add_option( + "rawtcp", False, bool, + "Enable/disable experimental raw TCP support. " + "Disabled by default. " + ) - self.add_option("spoof_source_address", False, bool) + self.add_option( + "spoof_source_address", False, bool, + "Use the client's IP for server-side connections. " + "Combine with --upstream-bind-address to spoof a fixed source address." + ) self.add_option("upstream_server", None, Optional[str]) self.add_option("upstream_auth", None, Optional[str]) self.add_option("ssl_version_client", "secure", str) self.add_option("ssl_version_server", "secure", str) - self.add_option("ssl_insecure", False, bool) + self.add_option( + "ssl_insecure", False, bool, + "Do not verify upstream server SSL/TLS certificates." + ) self.add_option("ssl_verify_upstream_trusted_cadir", None, Optional[str]) self.add_option("ssl_verify_upstream_trusted_ca", None, Optional[str]) self.add_option("tcp_hosts", [], Sequence[str]) @@ -91,19 +166,39 @@ class Options(optmanager.OptManager): self.add_option("intercept", None, Optional[str]) # Console options - self.add_option("console_eventlog", False, bool) - self.add_option("console_focus_follow", False, bool) + self.add_option( + "console_eventlog", False, bool, + help="Show event log." + ) + self.add_option( + "console_focus_follow", False, bool, + "Focus follows new flows." + ) self.add_option("console_palette", "dark", Optional[str]) - self.add_option("console_palette_transparent", False, bool) - self.add_option("console_no_mouse", False, bool) + self.add_option( + "console_palette_transparent", False, bool, + "Set transparent background for palette." + ) + self.add_option( + "console_mouse", True, bool, + "Console mouse interaction." + ) self.add_option("console_order", None, Optional[str]) - self.add_option("console_order_reversed", False, bool) + self.add_option( + "console_order_reversed", False, bool, + ) self.add_option("filter", None, Optional[str]) # Web options - self.add_option("web_open_browser", True, bool) - self.add_option("web_debug", False, bool) + self.add_option( + "web_open_browser", True, bool, + "Start a browser" + ) + self.add_option( + "web_debug", False, bool, + "Mitmweb debugging" + ) self.add_option("web_port", 8081, int) self.add_option("web_iface", "127.0.0.1", str) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 4b5a710e..240e3642 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -21,19 +21,21 @@ unset = object() class _Option: - __slots__ = ("name", "typespec", "value", "_default") + __slots__ = ("name", "typespec", "value", "_default", "help") def __init__( self, name: str, default: typing.Any, - typespec: typing.Type + typespec: typing.Type, + help: typing.Optional[str] ) -> None: typecheck.check_type(name, default, typespec) self.name = name self._default = default self.typespec = typespec self.value = unset + self.help = help def __repr__(self): return "{value} [{type}]".format(value=self.current(), type=self.typespec) @@ -66,7 +68,7 @@ class _Option: return True def __deepcopy__(self, _): - o = _Option(self.name, self.default, self.typespec) + o = _Option(self.name, self.default, self.typespec, self.help) if self.has_changed(): o.value = self.current() return o @@ -91,10 +93,16 @@ class OptManager: self.__dict__["changed"] = blinker.Signal() self.__dict__["errored"] = blinker.Signal() - def add_option(self, name: str, default: typing.Any, typespec: typing.Type) -> None: + def add_option( + self, + name: str, + default: typing.Any, + typespec: typing.Type, + help: str = None + ) -> None: if name in self._options: raise ValueError("Option %s already exists" % name) - self._options[name] = _Option(name, default, typespec) + self._options[name] = _Option(name, default, typespec, help) @contextlib.contextmanager def rollback(self, updated): @@ -303,3 +311,24 @@ class OptManager: cls=type(self).__name__, options=options ) + + def make_parser(self, parser, option): + o = self._options[option] + f = option.replace("_", "-") + if o.typespec == bool: + g = parser.add_mutually_exclusive_group(required=False) + g.add_argument( + "--%s" % f, + action="store_true", + dest=option, + help=o.help + ) + g.add_argument( + "--no-%s" % f, + action="store_false", + dest=option, + help=o.help + ) + parser.set_defaults(**{option: o.default}) + else: + raise ValueError("Unsupported option type: %s", o.typespec) diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index 7d15130f..103d96cc 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -358,7 +358,7 @@ class TlsLayer(base.Layer): # 2.5 The client did not sent a SNI value, we don't know the certificate subject. client_tls_requires_server_connection = ( self._server_tls and - not self.config.options.no_upstream_cert and + self.config.options.upstream_cert and ( self.config.options.add_upstream_certs_to_client_chain or self._client_tls and ( @@ -574,7 +574,7 @@ class TlsLayer(base.Layer): use_upstream_cert = ( self.server_conn and self.server_conn.tls_established and - (not self.config.options.no_upstream_cert) + self.config.options.upstream_cert ) if use_upstream_cert: upstream_cert = self.server_conn.cert diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 11558cc3..41f4cedb 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -85,7 +85,7 @@ def get_common_options(args): "are mutually exclusive. Read the docs on proxy modes " "to understand why." ) - if args.add_upstream_certs_to_client_chain and args.no_upstream_cert: + if args.add_upstream_certs_to_client_chain and not args.upstream_cert: raise exceptions.OptionsError( "The no-upstream-cert and add-upstream-certs-to-client-chain " "options are mutually exclusive. If no-upstream-cert is enabled " @@ -106,7 +106,7 @@ def get_common_options(args): client_replay=args.client_replay, replay_kill_extra=args.replay_kill_extra, no_server=args.no_server, - refresh_server_playback=not args.norefresh, + refresh_server_playback=args.refresh_server_playback, server_replay_use_headers=args.server_replay_use_headers, rfile=args.rfile, replacements=args.replacements, @@ -143,7 +143,7 @@ def get_common_options(args): listen_port = args.port, upstream_bind_address = args.upstream_bind_address, mode = mode, - no_upstream_cert = args.no_upstream_cert, + upstream_cert = args.upstream_cert, spoof_source_address = args.spoof_source_address, http2 = args.http2, @@ -162,7 +162,7 @@ def get_common_options(args): ) -def basic_options(parser): +def basic_options(parser, opts): parser.add_argument( '--version', action='store_true', @@ -174,24 +174,13 @@ def basic_options(parser): help="show program's short version number and exit", version=version.VERSION ) - parser.add_argument( - "--anticache", - action="store_true", dest="anticache", - help=""" - Strip out request headers that might cause the server to return - 304-not-modified. - """ - ) + opts.make_parser(parser, "anticache") parser.add_argument( "--cadir", action="store", type=str, dest="cadir", help="Location of the default mitmproxy CA files. (%s)" % options.CA_DIR ) - parser.add_argument( - "--host", - action="store_true", dest="showhost", - help="Use the Host header to construct URLs for display." - ) + opts.make_parser(parser, "showhost") parser.add_argument( "-q", "--quiet", action="store_true", dest="quiet", @@ -239,11 +228,7 @@ def basic_options(parser): action="store", dest="streamfile", type=lambda f: (f, "a"), help="Append flows to file." ) - parser.add_argument( - "-z", "--anticomp", - action="store_true", dest="anticomp", - help="Try to convince servers to send us un-compressed data." - ) + opts.make_parser(parser, "anticomp") parser.add_argument( "-Z", "--body-size-limit", action="store", dest="body_size_limit", @@ -263,7 +248,7 @@ def basic_options(parser): ) -def proxy_modes(parser): +def proxy_modes(parser, opts): group = parser.add_argument_group("Proxy Modes") group.add_argument( "-R", "--reverse", @@ -296,7 +281,7 @@ def proxy_modes(parser): ) -def proxy_options(parser): +def proxy_options(parser, opts): group = parser.add_argument_group("Proxy Options") group.add_argument( "-b", "--bind-address", @@ -326,11 +311,7 @@ def proxy_options(parser): communication contents are printed to the log in verbose mode. """ ) - group.add_argument( - "-n", "--no-server", - action="store_true", dest="no_server", - help="Don't start a proxy server." - ) + opts.make_parser(group, "no_server") group.add_argument( "-p", "--port", action="store", type=int, dest="port", @@ -338,26 +319,11 @@ def proxy_options(parser): ) http2 = group.add_mutually_exclusive_group() - http2.add_argument("--no-http2", action="store_false", dest="http2") - http2.add_argument("--http2", action="store_true", dest="http2", - help="Explicitly enable/disable HTTP/2 support. " - "HTTP/2 support is enabled by default.", - ) - - http2_priority = group.add_mutually_exclusive_group() - http2_priority.add_argument("--http2-priority", action="store_true", dest="http2_priority") - http2_priority.add_argument("--no-http2-priority", action="store_false", dest="http2_priority", - help="Explicitly enable/disable PRIORITY forwarding for HTTP/2 connections. " - "PRIORITY forwarding is disabled by default, " - "because some webservers fail at implementing the RFC properly.", - ) + opts.make_parser(http2, "http2") + opts.make_parser(http2, "http2_priority") websocket = group.add_mutually_exclusive_group() - websocket.add_argument("--no-websocket", action="store_false", dest="websocket") - websocket.add_argument("--websocket", action="store_true", dest="websocket", - help="Explicitly enable/disable WebSocket support. " - "WebSocket support is enabled by default.", - ) + opts.make_parser(websocket, "websocket") parser.add_argument( "--upstream-auth", @@ -369,33 +335,18 @@ def proxy_options(parser): """ ) - rawtcp = group.add_mutually_exclusive_group() - rawtcp.add_argument("--raw-tcp", action="store_true", dest="rawtcp") - rawtcp.add_argument("--no-raw-tcp", action="store_false", dest="rawtcp", - help="Explicitly enable/disable experimental raw tcp support. " - "Disabled by default. " - "Default value will change in a future version." - ) + opts.make_parser(group, "rawtcp") - group.add_argument( - "--spoof-source-address", - action="store_true", dest="spoof_source_address", - help="Use the client's IP for server-side connections. " - "Combine with --upstream-bind-address to spoof a fixed source address." - ) + opts.make_parser(group, "spoof_source_address") group.add_argument( "--upstream-bind-address", action="store", type=str, dest="upstream_bind_address", help="Address to bind upstream requests to (defaults to none)" ) - group.add_argument( - "--keep-host-header", - action="store_true", dest="keep_host_header", - help="Reverse Proxy: Keep the original host header instead of rewriting it to the reverse proxy target." - ) + opts.make_parser(group, "keep_host_header") -def proxy_ssl_options(parser): +def proxy_ssl_options(parser, opts): # TODO: Agree to consistently either use "upstream" or "server". group = parser.add_argument_group("SSL") group.add_argument( @@ -425,22 +376,9 @@ def proxy_ssl_options(parser): type=str, dest="clientcerts", help="Client certificate file or directory." ) - group.add_argument( - "--no-upstream-cert", - action="store_true", dest="no_upstream_cert", - help="Don't connect to upstream server to look up certificate details." - ) - group.add_argument( - "--add-upstream-certs-to-client-chain", - action="store_true", dest="add_upstream_certs_to_client_chain", - help="Add all certificates of the upstream server to the certificate chain " - "that will be served to the proxy client, as extras." - ) - group.add_argument( - "--insecure", - action="store_true", dest="ssl_insecure", - help="Do not verify upstream server SSL/TLS certificates." - ) + opts.make_parser(group, "upstream_cert") + opts.make_parser(group, "add_upstream_certs_to_client_chain") + opts.make_parser(group, "ssl_insecure") group.add_argument( "--upstream-trusted-cadir", action="store", dest="ssl_verify_upstream_trusted_cadir", @@ -468,13 +406,9 @@ def proxy_ssl_options(parser): ) -def onboarding_app(parser): +def onboarding_app(parser, opts): group = parser.add_argument_group("Onboarding App") - group.add_argument( - "--no-onboarding", - action="store_false", dest="onboarding", - help="Disable the mitmproxy onboarding app." - ) + opts.make_parser(parser, "onboarding") group.add_argument( "--onboarding-host", action="store", dest="onboarding_host", @@ -494,7 +428,7 @@ def onboarding_app(parser): ) -def client_replay(parser): +def client_replay(parser, opts): group = parser.add_argument_group("Client Replay") group.add_argument( "-c", "--client-replay", @@ -503,46 +437,24 @@ def client_replay(parser): ) -def server_replay(parser): +def server_replay(parser, opts): group = parser.add_argument_group("Server Replay") group.add_argument( "-S", "--server-replay", action="append", dest="server_replay", metavar="PATH", help="Replay server responses from a saved file." ) - group.add_argument( - "-k", "--replay-kill-extra", - action="store_true", dest="replay_kill_extra", - help="Kill extra requests during replay." - ) + opts.make_parser(parser, "replay_kill_extra") group.add_argument( "--server-replay-use-header", action="append", dest="server_replay_use_headers", type=str, help="Request headers to be considered during replay. " "Can be passed multiple times." ) - group.add_argument( - "--norefresh", - action="store_true", dest="norefresh", - help=""" - Disable response refresh, which updates times in cookies and headers - for replayed responses. - """ - ) - group.add_argument( - "--no-pop", - action="store_true", dest="server_replay_nopop", - help="Disable response pop from response flow. " - "This makes it possible to replay same response multiple times." - ) + opts.make_parser(group, "refresh_server_playback") + opts.make_parser(group, "server_replay_nopop") payload = group.add_mutually_exclusive_group() - payload.add_argument( - "--replay-ignore-content", - action="store_true", dest="server_replay_ignore_content", - help=""" - Ignore request's content while searching for a saved flow to replay - """ - ) + opts.make_parser(payload, "server_replay_ignore_content") payload.add_argument( "--replay-ignore-payload-param", action="append", dest="server_replay_ignore_payload_params", type=str, @@ -561,14 +473,10 @@ def server_replay(parser): to replay. Can be passed multiple times. """ ) - group.add_argument( - "--replay-ignore-host", - action="store_true", - dest="server_replay_ignore_host", - help="Ignore request's destination host while searching for a saved flow to replay") + opts.make_parser(parser, "server_replay_ignore_host") -def replacements(parser): +def replacements(parser, opts): group = parser.add_argument_group( "Replacements", """ @@ -594,7 +502,7 @@ def replacements(parser): ) -def set_headers(parser): +def set_headers(parser, opts): group = parser.add_argument_group( "Set Headers", """ @@ -611,7 +519,7 @@ def set_headers(parser): ) -def proxy_authentication(parser): +def proxy_authentication(parser, opts): group = parser.add_argument_group( "Proxy Authentication", """ @@ -619,12 +527,7 @@ def proxy_authentication(parser): used for authenticating them. """ ).add_mutually_exclusive_group() - group.add_argument( - "--nonanonymous", - action="store_true", dest="auth_nonanonymous", - help="Allow access to any user long as a credentials are specified." - ) - + opts.make_parser(group, "auth_nonanonymous") group.add_argument( "--singleuser", action="store", dest="auth_singleuser", type=str, @@ -642,7 +545,7 @@ def proxy_authentication(parser): ) -def common_options(parser): +def common_options(parser, opts): parser.add_argument( "--conf", type=str, dest="conf", default=CONFIG_PATH, @@ -651,58 +554,41 @@ def common_options(parser): Configuration file """ ) - - basic_options(parser) - proxy_modes(parser) - proxy_options(parser) - proxy_ssl_options(parser) - onboarding_app(parser) - client_replay(parser) - server_replay(parser) - replacements(parser) - set_headers(parser) - proxy_authentication(parser) + basic_options(parser, opts) + proxy_modes(parser, opts) + proxy_options(parser, opts) + proxy_ssl_options(parser, opts) + onboarding_app(parser, opts) + client_replay(parser, opts) + server_replay(parser, opts) + replacements(parser, opts) + set_headers(parser, opts) + proxy_authentication(parser, opts) -def mitmproxy(): +def mitmproxy(opts): # Don't import mitmproxy.tools.console for mitmdump, urwid is not available # on all platforms. from .console import palettes parser = argparse.ArgumentParser(usage="%(prog)s [options]") - common_options(parser) + common_options(parser, opts) parser.add_argument( "--palette", type=str, action="store", dest="console_palette", choices=sorted(palettes.palettes.keys()), help="Select color palette: " + ", ".join(palettes.palettes.keys()) ) - parser.add_argument( - "--palette-transparent", - action="store_true", dest="console_palette_transparent", - help="Set transparent background for palette." - ) - parser.add_argument( - "-e", "--eventlog", - action="store_true", dest="console_eventlog", - help="Show event log." - ) - parser.add_argument( - "--follow", - action="store_true", dest="console_focus_follow", - help="Focus follows new flows." - ) + opts.make_parser(parser, "console_palette_transparent") + opts.make_parser(parser, "console_eventlog") + opts.make_parser(parser, "console_focus_follow") parser.add_argument( "--order", type=str, dest="console_order", choices=[o[1] for o in view.orders], help="Flow sort order." ) - parser.add_argument( - "--no-mouse", - action="store_true", dest="console_no_mouse", - help="Disable mouse interaction." - ) + opts.make_parser(parser, "console_mouse") group = parser.add_argument_group( "Filters", "See help in mitmproxy for filter expression syntax." @@ -720,18 +606,11 @@ def mitmproxy(): return parser -def mitmdump(): +def mitmdump(opts): parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]") - common_options(parser) - parser.add_argument( - "--keepserving", - action="store_true", dest="keepserving", - help=""" - Continue serving after client playback or file read. We exit by - default. - """ - ) + common_options(parser, opts) + opts.make_parser(parser, "keepserving") parser.add_argument( "-d", "--detail", action="count", dest="flow_detail", @@ -748,15 +627,11 @@ def mitmdump(): return parser -def mitmweb(): +def mitmweb(opts): parser = argparse.ArgumentParser(usage="%(prog)s [options]") group = parser.add_argument_group("Mitmweb") - group.add_argument( - "--no-browser", - action="store_false", dest="web_open_browser", - help="Don't start a browser" - ) + opts.make_parser(group, "web_open_browser") group.add_argument( "--web-port", action="store", type=int, dest="web_port", @@ -769,13 +644,9 @@ def mitmweb(): metavar="IFACE", help="Mitmweb interface." ) - group.add_argument( - "--web-debug", - action="store_true", dest="web_debug", - help="Turn on mitmweb debugging" - ) + opts.make_parser(group, "web_debug") - common_options(parser) + common_options(parser, opts) group = parser.add_argument_group( "Filters", "See help in mitmproxy for filter expression syntax." diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index d68dc93c..e75105cf 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -252,7 +252,7 @@ class ConsoleMaster(master.Master): self.loop = urwid.MainLoop( urwid.SolidFill("x"), screen = self.ui, - handle_mouse = not self.options.console_no_mouse, + handle_mouse = self.options.console_mouse, ) self.ab = statusbar.ActionBar() diff --git a/mitmproxy/tools/console/options.py b/mitmproxy/tools/console/options.py index 4115bd18..33e3ec38 100644 --- a/mitmproxy/tools/console/options.py +++ b/mitmproxy/tools/console/options.py @@ -90,10 +90,10 @@ class Options(urwid.WidgetWrap): select.Heading("Network"), select.Option( - "No Upstream Certs", + "Upstream Certs", "U", - checker("no_upstream_cert", master.options), - master.options.toggler("no_upstream_cert") + checker("upstream_cert", master.options), + master.options.toggler("upstream_cert") ), select.Option( "TCP Proxying", diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index d90d932b..a5611b28 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -220,7 +220,7 @@ class StatusBar(urwid.WidgetWrap): opts.append("norefresh") if self.master.options.replay_kill_extra: opts.append("killextra") - if self.master.options.no_upstream_cert: + if not self.master.options.upstream_cert: opts.append("no-upstream-cert") if self.master.options.console_focus_follow: opts.append("following") diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index ce78cd13..c0293f28 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -61,11 +61,11 @@ def mitmproxy(args=None): # pragma: no cover version_check.check_pyopenssl_version() assert_utf8_env() - parser = cmdline.mitmproxy() + console_options = options.Options() + parser = cmdline.mitmproxy(console_options) args = parser.parse_args(args) try: - console_options = options.Options() console_options.load_paths(args.conf) console_options.merge(cmdline.get_common_options(args)) console_options.merge( @@ -74,7 +74,7 @@ def mitmproxy(args=None): # pragma: no cover console_palette_transparent = args.console_palette_transparent, console_eventlog = args.console_eventlog, console_focus_follow = args.console_focus_follow, - console_no_mouse = args.console_no_mouse, + console_mouse = args.console_mouse, console_order = args.console_order, filter = args.filter, @@ -98,14 +98,14 @@ def mitmdump(args=None): # pragma: no cover version_check.check_pyopenssl_version() - parser = cmdline.mitmdump() + dump_options = options.Options() + parser = cmdline.mitmdump(dump_options) args = parser.parse_args(args) if args.quiet: args.flow_detail = 0 master = None try: - dump_options = options.Options() dump_options.load_paths(args.conf) dump_options.merge(cmdline.get_common_options(args)) dump_options.merge( @@ -139,12 +139,11 @@ def mitmweb(args=None): # pragma: no cover version_check.check_pyopenssl_version() - parser = cmdline.mitmweb() - + web_options = options.Options() + parser = cmdline.mitmweb(web_options) args = parser.parse_args(args) try: - web_options = options.Options() web_options.load_paths(args.conf) web_options.merge(cmdline.get_common_options(args)) web_options.merge( diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 35b549ee..eddaa3e1 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -408,7 +408,7 @@ class Settings(RequestHandler): mode=str(self.master.options.mode), intercept=self.master.options.intercept, showhost=self.master.options.showhost, - no_upstream_cert=self.master.options.no_upstream_cert, + upstream_cert=self.master.options.upstream_cert, rawtcp=self.master.options.rawtcp, http2=self.master.options.http2, websocket=self.master.options.websocket, @@ -425,7 +425,7 @@ class Settings(RequestHandler): def put(self): update = self.json option_whitelist = { - "intercept", "showhost", "no_upstream_cert", + "intercept", "showhost", "upstream_cert", "rawtcp", "http2", "websocket", "anticache", "anticomp", "stickycookie", "stickyauth", "stream_large_bodies" } diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index 871d02fe..770c6550 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -100,7 +100,7 @@ class _Http2TestBase: def get_options(cls): opts = options.Options( listen_port=0, - no_upstream_cert=False, + upstream_cert=True, ssl_insecure=True ) opts.cadir = os.path.join(tempfile.gettempdir(), "mitmproxy") diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py index bac0e527..486e9d64 100644 --- a/test/mitmproxy/proxy/protocol/test_websocket.py +++ b/test/mitmproxy/proxy/protocol/test_websocket.py @@ -64,7 +64,7 @@ class _WebSocketTestBase: def get_options(cls): opts = options.Options( listen_port=0, - no_upstream_cert=False, + upstream_cert=True, ssl_insecure=True, websocket=True, ) diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 56b09b9a..eb40dd14 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -870,11 +870,11 @@ class TestServerConnect(tservers.HTTPProxyTest): @classmethod def get_options(cls): opts = tservers.HTTPProxyTest.get_options() - opts.no_upstream_cert = True + opts.upstream_cert = False return opts def test_unnecessary_serverconnect(self): - """A replayed/fake response with no_upstream_cert should not connect to an upstream server""" + """A replayed/fake response with no upstream_cert should not connect to an upstream server""" assert self.pathod("200").status_code == 200 for msg in self.proxy.tmaster.tlog: assert "serverconnect" not in msg diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 3fba304a..44c757af 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -252,14 +252,14 @@ def test_merge(): def test_option(): - o = optmanager._Option("test", 1, int) + o = optmanager._Option("test", 1, int, None) assert o.current() == 1 with pytest.raises(TypeError): o.set("foo") with pytest.raises(TypeError): - optmanager._Option("test", 1, str) + optmanager._Option("test", 1, str, None) - o2 = optmanager._Option("test", 1, int) + o2 = optmanager._Option("test", 1, int, None) assert o2 == o o2.set(5) assert o2 != o diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 37cec57a..6e360875 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -30,9 +30,9 @@ class TestProcessProxyOptions: def p(self, *args): parser = MockParser() - cmdline.common_options(parser) - args = parser.parse_args(args=args) opts = options.Options() + cmdline.common_options(parser, opts) + args = parser.parse_args(args=args) opts.merge(cmdline.get_common_options(args)) pconf = config.ProxyConfig(opts) return parser, pconf @@ -91,7 +91,7 @@ class TestProcessProxyOptions: self.p("--cert", "nonexistent") def test_insecure(self): - p = self.assert_noerr("--insecure") + p = self.assert_noerr("--ssl-insecure") assert p.openssl_verification_mode_server == SSL.VERIFY_NONE def test_upstream_trusted_cadir(self): diff --git a/test/mitmproxy/tools/test_cmdline.py b/test/mitmproxy/tools/test_cmdline.py index 96d5ae31..b9f9d00d 100644 --- a/test/mitmproxy/tools/test_cmdline.py +++ b/test/mitmproxy/tools/test_cmdline.py @@ -1,31 +1,36 @@ import argparse from mitmproxy.tools import cmdline +from mitmproxy import options def test_common(): parser = argparse.ArgumentParser() - cmdline.common_options(parser) - opts = parser.parse_args(args=[]) + opts = options.Options() + cmdline.common_options(parser, opts) + args = parser.parse_args(args=[]) - assert cmdline.get_common_options(opts) + assert cmdline.get_common_options(args) - opts.stickycookie_filt = "foo" - opts.stickyauth_filt = "foo" - v = cmdline.get_common_options(opts) + args.stickycookie_filt = "foo" + args.stickyauth_filt = "foo" + v = cmdline.get_common_options(args) assert v["stickycookie"] == "foo" assert v["stickyauth"] == "foo" def test_mitmproxy(): - ap = cmdline.mitmproxy() + opts = options.Options() + ap = cmdline.mitmproxy(opts) assert ap def test_mitmdump(): - ap = cmdline.mitmdump() + opts = options.Options() + ap = cmdline.mitmdump(opts) assert ap def test_mitmweb(): - ap = cmdline.mitmweb() + opts = options.Options() + ap = cmdline.mitmweb(opts) assert ap diff --git a/tox.ini b/tox.ini index 4994b119..a9904e87 100644 --- a/tox.ini +++ b/tox.ini @@ -30,6 +30,7 @@ commands = mypy --ignore-missing-imports --follow-imports=skip \ mitmproxy/addons/ \ mitmproxy/addonmanager.py \ + mitmproxy/optmanager.py \ mitmproxy/proxy/protocol/ \ mitmproxy/log.py \ mitmproxy/tools/dump.py \ -- cgit v1.2.3 From 45d18ac8cba462eb4f4f73e3e63ea539b44c6f83 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 5 Mar 2017 20:45:55 +1300 Subject: Start unifying options and the command-line: ints Moves all integer options apart from a few tricky ones like verbosity over to auto generation. Also add a metavar argument to parser generation to support this. --- mitmproxy/options.py | 25 ++++++++++++++++++++----- mitmproxy/optmanager.py | 11 ++++++++++- mitmproxy/tools/cmdline.py | 34 +++++++--------------------------- 3 files changed, 37 insertions(+), 33 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 1d879ac8..c91e2f88 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -28,7 +28,10 @@ class Options(optmanager.OptManager): "Toggle the mitmproxy onboarding app." ) self.add_option("onboarding_host", APP_HOST, str) - self.add_option("onboarding_port", APP_PORT, int) + self.add_option( + "onboarding_port", APP_PORT, int, + help="Port to serve the onboarding app from." + ) self.add_option( "anticache", False, bool, """ @@ -75,7 +78,10 @@ class Options(optmanager.OptManager): self.add_option("stickycookie", None, Optional[str]) self.add_option("stickyauth", None, Optional[str]) self.add_option("stream_large_bodies", None, Optional[int]) - self.add_option("verbosity", 2, int) + self.add_option( + "verbosity", 2, int, + "Log verbosity." + ) self.add_option("default_contentview", "auto", str) self.add_option("streamfile", None, Optional[str]) self.add_option("streamfile_append", False, bool) @@ -111,7 +117,10 @@ class Options(optmanager.OptManager): self.add_option("clientcerts", None, Optional[str]) self.add_option("ignore_hosts", [], Sequence[str]) self.add_option("listen_host", "", str) - self.add_option("listen_port", LISTEN_PORT, int) + self.add_option( + "listen_port", LISTEN_PORT, int, + "Proxy service port." + ) self.add_option("upstream_bind_address", "", str) self.add_option("mode", "regular", str) self.add_option( @@ -199,11 +208,17 @@ class Options(optmanager.OptManager): "web_debug", False, bool, "Mitmweb debugging" ) - self.add_option("web_port", 8081, int) + self.add_option( + "web_port", 8081, int, + "Mitmweb port." + ) self.add_option("web_iface", "127.0.0.1", str) # Dump options self.add_option("filtstr", None, Optional[str]) - self.add_option("flow_detail", 1, int) + self.add_option( + "flow_detail", 1, int, + "Flow detail display level" + ) self.update(**kwargs) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 240e3642..c16e2999 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -312,7 +312,7 @@ class OptManager: options=options ) - def make_parser(self, parser, option): + def make_parser(self, parser, option, metavar=None): o = self._options[option] f = option.replace("_", "-") if o.typespec == bool: @@ -330,5 +330,14 @@ class OptManager: help=o.help ) parser.set_defaults(**{option: o.default}) + elif o.typespec == int: + parser.add_argument( + "--%s" % f, + action="store", + type=int, + dest=option, + help=o.help, + metavar=metavar + ) else: raise ValueError("Unsupported option type: %s", o.typespec) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 41f4cedb..a2c7e2b8 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -139,8 +139,8 @@ def get_common_options(args): ciphers_server = args.ciphers_server, clientcerts = args.clientcerts, ignore_hosts = args.ignore_hosts, - listen_host = args.addr, - listen_port = args.port, + listen_host = args.listen_addr, + listen_port = args.listen_port, upstream_bind_address = args.upstream_bind_address, mode = mode, upstream_cert = args.upstream_cert, @@ -285,7 +285,7 @@ def proxy_options(parser, opts): group = parser.add_argument_group("Proxy Options") group.add_argument( "-b", "--bind-address", - action="store", type=str, dest="addr", + action="store", type=str, dest="listen_addr", help="Address to bind proxy to (defaults to all interfaces)" ) group.add_argument( @@ -312,11 +312,7 @@ def proxy_options(parser, opts): """ ) opts.make_parser(group, "no_server") - group.add_argument( - "-p", "--port", - action="store", type=int, dest="port", - help="Proxy service port." - ) + opts.make_parser(group, "listen_port", metavar="PORT") http2 = group.add_mutually_exclusive_group() opts.make_parser(http2, "http2") @@ -418,14 +414,7 @@ def onboarding_app(parser, opts): %s """ % options.APP_HOST ) - group.add_argument( - "--onboarding-port", - action="store", - dest="onboarding_port", - type=int, - metavar="80", - help="Port to serve the onboarding app from." - ) + opts.make_parser(group, "onboarding_port", metavar="PORT") def client_replay(parser, opts): @@ -611,11 +600,7 @@ def mitmdump(opts): common_options(parser, opts) opts.make_parser(parser, "keepserving") - parser.add_argument( - "-d", "--detail", - action="count", dest="flow_detail", - help="Increase flow detail display level. Can be passed multiple times." - ) + opts.make_parser(parser, "flow_detail", metavar = "LEVEL") parser.add_argument( 'filter', nargs="...", @@ -632,12 +617,7 @@ def mitmweb(opts): group = parser.add_argument_group("Mitmweb") opts.make_parser(group, "web_open_browser") - group.add_argument( - "--web-port", - action="store", type=int, dest="web_port", - metavar="PORT", - help="Mitmweb port." - ) + opts.make_parser(parser, "web_port", metavar="PORT") group.add_argument( "--web-iface", action="store", dest="web_iface", -- cgit v1.2.3 From 201c65960eb19b97e6c499f22082aa176bdaa6b1 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 6 Mar 2017 08:59:35 +1300 Subject: Options unification: sizes Start dealing with corner cases: - Sizes are always stored in options as strings - Add a new core addon that's responsible for verifying settings that don't belong to an addon - Add a _processed scratch space on the Options object for processed core values to be stored in. This is pretty dirty, but less dirty than re-parsing values every time. We'll come up with something better down the track. --- mitmproxy/addons/__init__.py | 2 ++ mitmproxy/addons/core.py | 20 ++++++++++++++++ mitmproxy/addons/streambodies.py | 7 +++++- mitmproxy/options.py | 15 ++++++++++-- mitmproxy/optmanager.py | 13 ++++++++-- mitmproxy/proxy/protocol/http1.py | 4 ++-- mitmproxy/proxy/protocol/http2.py | 2 +- mitmproxy/proxy/protocol/http_replay.py | 5 ++-- mitmproxy/tools/cmdline.py | 37 ++++------------------------- test/mitmproxy/addons/test_core.py | 13 ++++++++++ test/mitmproxy/addons/test_streambodies.py | 7 ++++-- test/mitmproxy/proxy/protocol/test_http2.py | 3 ++- 12 files changed, 82 insertions(+), 46 deletions(-) create mode 100644 mitmproxy/addons/core.py create mode 100644 test/mitmproxy/addons/test_core.py diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py index 97fa2dcd..1bf89bbb 100644 --- a/mitmproxy/addons/__init__.py +++ b/mitmproxy/addons/__init__.py @@ -3,6 +3,7 @@ from mitmproxy.addons import anticomp from mitmproxy.addons import check_alpn from mitmproxy.addons import check_ca from mitmproxy.addons import clientplayback +from mitmproxy.addons import core from mitmproxy.addons import disable_h2c_upgrade from mitmproxy.addons import onboarding from mitmproxy.addons import proxyauth @@ -19,6 +20,7 @@ from mitmproxy.addons import upstream_auth def default_addons(): return [ + core.Core(), anticache.AntiCache(), anticomp.AntiComp(), check_alpn.CheckALPN(), diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py new file mode 100644 index 00000000..5d2cf57b --- /dev/null +++ b/mitmproxy/addons/core.py @@ -0,0 +1,20 @@ +""" + The core addon is responsible for verifying core settings that are not + checked by other addons. +""" +from mitmproxy import exceptions +from mitmproxy.utils import human + + +class Core: + def configure(self, options, updated): + if "body_size_limit" in updated and options.body_size_limit: + try: + options._processed["body_size_limit"] = human.parse_size( + options.body_size_limit + ) + except ValueError as e: + raise exceptions.OptionsError( + "Invalid body size limit specification: %s" % + options.body_size_limit + ) diff --git a/mitmproxy/addons/streambodies.py b/mitmproxy/addons/streambodies.py index 3c2a153b..a10bdb93 100644 --- a/mitmproxy/addons/streambodies.py +++ b/mitmproxy/addons/streambodies.py @@ -1,6 +1,7 @@ from mitmproxy.net.http import http1 from mitmproxy import exceptions from mitmproxy import ctx +from mitmproxy.utils import human class StreamBodies: @@ -8,7 +9,11 @@ class StreamBodies: self.max_size = None def configure(self, options, updated): - self.max_size = options.stream_large_bodies + if "stream_large_bodies" in updated and options.stream_large_bodies: + try: + self.max_size = human.parse_size(options.stream_large_bodies) + except ValueError as e: + raise exceptions.OptionsError(e) def run(self, f, is_request): if self.max_size: diff --git a/mitmproxy/options.py b/mitmproxy/options.py index c91e2f88..58907ef3 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -77,7 +77,14 @@ class Options(optmanager.OptManager): self.add_option("server_replay", [], Sequence[str]) self.add_option("stickycookie", None, Optional[str]) self.add_option("stickyauth", None, Optional[str]) - self.add_option("stream_large_bodies", None, Optional[int]) + self.add_option( + "stream_large_bodies", None, Optional[str], + """ + Stream data to the client if response body exceeds the given + threshold. If streamed, the body will not be stored in any way. + Understands k/m/g suffixes, i.e. 3m for 3 megabytes. + """ + ) self.add_option( "verbosity", 2, int, "Log verbosity." @@ -109,7 +116,11 @@ class Options(optmanager.OptManager): "Add all certificates of the upstream server to the certificate chain " "that will be served to the proxy client, as extras." ) - self.add_option("body_size_limit", None, Optional[int]) + self.add_option( + "body_size_limit", None, Optional[str], + "Byte size limit of HTTP request and response bodies." + " Understands k/m/g suffixes, i.e. 3m for 3 megabytes." + ) self.add_option("cadir", CA_DIR, str) self.add_option("certs", [], Sequence[Tuple[str, str]]) self.add_option("ciphers_client", DEFAULT_CLIENT_CIPHERS, str) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index c16e2999..b9aaf6c8 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -12,7 +12,6 @@ import ruamel.yaml from mitmproxy import exceptions from mitmproxy.utils import typecheck - """ The base implementation for Options. """ @@ -92,6 +91,7 @@ class OptManager: self.__dict__["_options"] = {} self.__dict__["changed"] = blinker.Signal() self.__dict__["errored"] = blinker.Signal() + self.__dict__["_processed"] = {} def add_option( self, @@ -330,7 +330,7 @@ class OptManager: help=o.help ) parser.set_defaults(**{option: o.default}) - elif o.typespec == int: + elif o.typespec in (int, typing.Optional[int]): parser.add_argument( "--%s" % f, action="store", @@ -339,5 +339,14 @@ class OptManager: help=o.help, metavar=metavar ) + elif o.typespec in (str, typing.Optional[str]): + parser.add_argument( + "--%s" % f, + action="store", + type=str, + dest=option, + help=o.help, + metavar=metavar + ) else: raise ValueError("Unsupported option type: %s", o.typespec) diff --git a/mitmproxy/proxy/protocol/http1.py b/mitmproxy/proxy/protocol/http1.py index b1fd0ecd..cafc2682 100644 --- a/mitmproxy/proxy/protocol/http1.py +++ b/mitmproxy/proxy/protocol/http1.py @@ -19,7 +19,7 @@ class Http1Layer(httpbase._HttpTransmissionLayer): return http1.read_body( self.client_conn.rfile, expected_size, - self.config.options.body_size_limit + self.config.options._processed.get("body_size_limit") ) def send_request(self, request): @@ -35,7 +35,7 @@ class Http1Layer(httpbase._HttpTransmissionLayer): return http1.read_body( self.server_conn.rfile, expected_size, - self.config.options.body_size_limit + self.config.options._processed.get("body_size_limit") ) def send_response_headers(self, response): diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index 01406798..a6e8a4dd 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -183,7 +183,7 @@ class Http2Layer(base.Layer): return True def _handle_data_received(self, eid, event, source_conn): - bsl = self.config.options.body_size_limit + bsl = self.config.options._processed.get("body_size_limit") if bsl and self.streams[eid].queued_data_length > bsl: self.streams[eid].kill() self.connections[source_conn].safe_reset_stream( diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py index 38f511c4..c47f2e8f 100644 --- a/mitmproxy/proxy/protocol/http_replay.py +++ b/mitmproxy/proxy/protocol/http_replay.py @@ -31,6 +31,7 @@ class RequestReplayThread(basethread.BaseThread): def run(self): r = self.f.request + bsl = self.config.options._processed.get("body_size_limit") first_line_format_backup = r.first_line_format server = None try: @@ -55,7 +56,7 @@ class RequestReplayThread(basethread.BaseThread): resp = http1.read_response( server.rfile, connect_request, - body_size_limit=self.config.options.body_size_limit + body_size_limit=bsl ) if resp.status_code != 200: raise exceptions.ReplayException("Upstream server refuses CONNECT request") @@ -87,7 +88,7 @@ class RequestReplayThread(basethread.BaseThread): http1.read_response( server.rfile, r, - body_size_limit=self.config.options.body_size_limit + body_size_limit=bsl ) ) if self.channel: diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index a2c7e2b8..def2ae31 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -4,7 +4,6 @@ import os from mitmproxy import exceptions from mitmproxy import options from mitmproxy import platform -from mitmproxy.utils import human from mitmproxy.net import tcp from mitmproxy import version from mitmproxy.addons import view @@ -25,10 +24,6 @@ def get_common_options(args): if args.stickyauth_filt: stickyauth = args.stickyauth_filt - stream_large_bodies = args.stream_large_bodies - if stream_large_bodies: - stream_large_bodies = human.parse_size(stream_large_bodies) - if args.streamfile and args.streamfile[0] == args.rfile: if args.streamfile[1] == "wb": raise exceptions.OptionsError( @@ -49,15 +44,6 @@ def get_common_options(args): parts = ["*", parts[0]] certs.append(parts) - body_size_limit = args.body_size_limit - if body_size_limit: - try: - body_size_limit = human.parse_size(body_size_limit) - except ValueError as e: - raise exceptions.OptionsError( - "Invalid body size limit specification: %s" % body_size_limit - ) - # Establish proxy mode c = 0 mode, upstream_server = "regular", None @@ -117,7 +103,7 @@ def get_common_options(args): scripts=args.scripts, stickycookie=stickycookie, stickyauth=stickyauth, - stream_large_bodies=stream_large_bodies, + stream_large_bodies=args.stream_large_bodies, showhost=args.showhost, streamfile=args.streamfile[0] if args.streamfile else None, streamfile_append=True if args.streamfile and args.streamfile[1] == "a" else False, @@ -132,7 +118,7 @@ def get_common_options(args): auth_singleuser = args.auth_singleuser, auth_htpasswd = args.auth_htpasswd, add_upstream_certs_to_client_chain = args.add_upstream_certs_to_client_chain, - body_size_limit = body_size_limit, + body_size_limit = args.body_size_limit, cadir = args.cadir, certs = certs, ciphers_client = args.ciphers_client, @@ -229,23 +215,8 @@ def basic_options(parser, opts): help="Append flows to file." ) opts.make_parser(parser, "anticomp") - parser.add_argument( - "-Z", "--body-size-limit", - action="store", dest="body_size_limit", - metavar="SIZE", - help="Byte size limit of HTTP request and response bodies." - " Understands k/m/g suffixes, i.e. 3m for 3 megabytes." - ) - parser.add_argument( - "--stream", - action="store", dest="stream_large_bodies", - metavar="SIZE", - help=""" - Stream data to the client if response body exceeds the given - threshold. If streamed, the body will not be stored in any way. - Understands k/m/g suffixes, i.e. 3m for 3 megabytes. - """ - ) + opts.make_parser(parser, "body_size_limit", metavar="SIZE") + opts.make_parser(parser, "stream_large_bodies") def proxy_modes(parser, opts): diff --git a/test/mitmproxy/addons/test_core.py b/test/mitmproxy/addons/test_core.py new file mode 100644 index 00000000..95272716 --- /dev/null +++ b/test/mitmproxy/addons/test_core.py @@ -0,0 +1,13 @@ +from mitmproxy import exceptions +from mitmproxy.addons import core +from mitmproxy.test import taddons +import pytest + + +def test_simple(): + sa = core.Core() + with taddons.context() as tctx: + with pytest.raises(exceptions.OptionsError): + tctx.configure(sa, body_size_limit = "invalid") + tctx.configure(sa, body_size_limit = "1m") + assert tctx.options._processed["body_size_limit"] diff --git a/test/mitmproxy/addons/test_streambodies.py b/test/mitmproxy/addons/test_streambodies.py index b982c66d..c6ce5e81 100644 --- a/test/mitmproxy/addons/test_streambodies.py +++ b/test/mitmproxy/addons/test_streambodies.py @@ -1,13 +1,16 @@ +from mitmproxy import exceptions from mitmproxy.test import tflow from mitmproxy.test import taddons - from mitmproxy.addons import streambodies +import pytest def test_simple(): sa = streambodies.StreamBodies() with taddons.context() as tctx: - tctx.configure(sa, stream_large_bodies = 10) + with pytest.raises(exceptions.OptionsError): + tctx.configure(sa, stream_large_bodies = "invalid") + tctx.configure(sa, stream_large_bodies = "10") f = tflow.tflow() f.request.content = b"" diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index 770c6550..1f695cc5 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -499,7 +499,8 @@ class TestBodySizeLimit(_Http2Test): return True def test_body_size_limit(self): - self.config.options.body_size_limit = 20 + self.config.options.body_size_limit = "20" + self.config.options._processed["body_size_limit"] = 20 client, h2_conn = self._setup_connection() -- cgit v1.2.3 From 18a6b66ba4510ad2c597ae3224ae67172fdd461e Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 6 Mar 2017 11:39:19 +1300 Subject: Options unification: simple strings Move all simple string options to the new scheme. Also regularise some names. --- mitmproxy/options.py | 103 ++++++++++++++++++---- mitmproxy/proxy/config.py | 12 +-- mitmproxy/proxy/protocol/http_replay.py | 4 +- mitmproxy/proxy/protocol/tls.py | 2 +- mitmproxy/tools/cmdline.py | 152 ++++++-------------------------- test/mitmproxy/proxy/test_server.py | 8 +- test/mitmproxy/test_proxy.py | 4 +- test/mitmproxy/tools/test_cmdline.py | 4 +- 8 files changed, 126 insertions(+), 163 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 58907ef3..ec270076 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -27,7 +27,14 @@ class Options(optmanager.OptManager): "onboarding", True, bool, "Toggle the mitmproxy onboarding app." ) - self.add_option("onboarding_host", APP_HOST, str) + self.add_option( + "onboarding_host", APP_HOST, str, + """ + Domain to serve the onboarding app from. For transparent mode, use + an IP when a DNS entry for the app domain is not present. Default: + %s + """ % APP_HOST + ) self.add_option( "onboarding_port", APP_PORT, int, help="Port to serve the onboarding app from." @@ -64,7 +71,10 @@ class Options(optmanager.OptManager): self.add_option( "refresh_server_playback", True, bool, ) - self.add_option("rfile", None, Optional[str]) + self.add_option( + "rfile", None, Optional[str], + "Read flows from file." + ) self.add_option("scripts", [], Sequence[str]) self.add_option( "showhost", False, bool, @@ -75,8 +85,14 @@ class Options(optmanager.OptManager): self.add_option("server_replay_use_headers", [], Sequence[str]) self.add_option("setheaders", [], Sequence[Union[Tuple[str, str, str], str]]) self.add_option("server_replay", [], Sequence[str]) - self.add_option("stickycookie", None, Optional[str]) - self.add_option("stickyauth", None, Optional[str]) + self.add_option( + "stickycookie", None, Optional[str], + "Set sticky cookie filter. Matched against requests." + ) + self.add_option( + "stickyauth", None, Optional[str], + "Set sticky auth filter. Matched against requests." + ) self.add_option( "stream_large_bodies", None, Optional[str], """ @@ -109,8 +125,17 @@ class Options(optmanager.OptManager): "auth_nonanonymous", False, bool, "Allow access to any user long as a credentials are specified." ) - self.add_option("auth_singleuser", None, Optional[str]) - self.add_option("auth_htpasswd", None, Optional[str]) + self.add_option( + "auth_singleuser", None, Optional[str], + """ + Allows access to a a single user, specified in the form + username:password. + """ + ) + self.add_option( + "auth_htpasswd", None, Optional[str], + "Allow access to users specified in an Apache htpasswd file." + ) self.add_option( "add_upstream_certs_to_client_chain", False, bool, "Add all certificates of the upstream server to the certificate chain " @@ -121,18 +146,36 @@ class Options(optmanager.OptManager): "Byte size limit of HTTP request and response bodies." " Understands k/m/g suffixes, i.e. 3m for 3 megabytes." ) - self.add_option("cadir", CA_DIR, str) + self.add_option( + "cadir", CA_DIR, str, + "Location of the default mitmproxy CA files. (%s)" % CA_DIR + ) self.add_option("certs", [], Sequence[Tuple[str, str]]) - self.add_option("ciphers_client", DEFAULT_CLIENT_CIPHERS, str) - self.add_option("ciphers_server", None, Optional[str]) - self.add_option("clientcerts", None, Optional[str]) + self.add_option( + "ciphers_client", DEFAULT_CLIENT_CIPHERS, str, + "Set supported ciphers for client connections. (OpenSSL Syntax)" + ) + self.add_option( + "ciphers_server", None, Optional[str], + "Set supported ciphers for server connections. (OpenSSL Syntax)" + ) + self.add_option( + "client_certs", None, Optional[str], + "Client certificate file or directory." + ) self.add_option("ignore_hosts", [], Sequence[str]) - self.add_option("listen_host", "", str) + self.add_option( + "listen_host", "", str, + "Address to bind proxy to (defaults to all interfaces)" + ) self.add_option( "listen_port", LISTEN_PORT, int, "Proxy service port." ) - self.add_option("upstream_bind_address", "", str) + self.add_option( + "upstream_bind_address", "", str, + "Address to bind upstream requests to (defaults to none)" + ) self.add_option("mode", "regular", str) self.add_option( "upstream_cert", True, bool, @@ -172,23 +215,39 @@ class Options(optmanager.OptManager): "Combine with --upstream-bind-address to spoof a fixed source address." ) self.add_option("upstream_server", None, Optional[str]) - self.add_option("upstream_auth", None, Optional[str]) + self.add_option( + "upstream_auth", None, Optional[str], + """ + Add HTTP Basic authentcation to upstream proxy and reverse proxy + requests. Format: username:password + """ + ) self.add_option("ssl_version_client", "secure", str) self.add_option("ssl_version_server", "secure", str) self.add_option( "ssl_insecure", False, bool, "Do not verify upstream server SSL/TLS certificates." ) - self.add_option("ssl_verify_upstream_trusted_cadir", None, Optional[str]) - self.add_option("ssl_verify_upstream_trusted_ca", None, Optional[str]) + self.add_option( + "ssl_verify_upstream_trusted_cadir", None, Optional[str], + "Path to a directory of trusted CA certificates for upstream " + "server verification prepared using the c_rehash tool." + ) + self.add_option( + "ssl_verify_upstream_trusted_ca", None, Optional[str], + "Path to a PEM formatted trusted CA certificate." + ) self.add_option("tcp_hosts", [], Sequence[str]) - self.add_option("intercept", None, Optional[str]) + self.add_option( + "intercept", None, Optional[str], + "Intercept filter expression." + ) # Console options self.add_option( "console_eventlog", False, bool, - help="Show event log." + "Show event log." ) self.add_option( "console_focus_follow", False, bool, @@ -208,7 +267,10 @@ class Options(optmanager.OptManager): "console_order_reversed", False, bool, ) - self.add_option("filter", None, Optional[str]) + self.add_option( + "filter", None, Optional[str], + "Filter view expression." + ) # Web options self.add_option( @@ -223,7 +285,10 @@ class Options(optmanager.OptManager): "web_port", 8081, int, "Mitmweb port." ) - self.add_option("web_iface", "127.0.0.1", str) + self.add_option( + "web_iface", "127.0.0.1", str, + "Mitmweb interface." + ) # Dump options self.add_option("filtstr", None, Optional[str]) diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index ea2f7c7f..778fd306 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -59,7 +59,7 @@ class ProxyConfig: self.check_ignore = None self.check_tcp = None self.certstore = None - self.clientcerts = None + self.client_certs = None self.openssl_verification_mode_server = None self.configure(options, set(options.keys())) options.changed.connect(self.configure) @@ -96,14 +96,14 @@ class ProxyConfig: CONF_BASENAME ) - if options.clientcerts: - clientcerts = os.path.expanduser(options.clientcerts) - if not os.path.exists(clientcerts): + if options.client_certs: + client_certs = os.path.expanduser(options.client_certs) + if not os.path.exists(client_certs): raise exceptions.OptionsError( "Client certificate path does not exist: %s" % - options.clientcerts + options.client_certs ) - self.clientcerts = clientcerts + self.client_certs = client_certs for spec, cert in options.certs: cert = os.path.expanduser(cert) diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py index c47f2e8f..161816e7 100644 --- a/mitmproxy/proxy/protocol/http_replay.py +++ b/mitmproxy/proxy/protocol/http_replay.py @@ -61,7 +61,7 @@ class RequestReplayThread(basethread.BaseThread): if resp.status_code != 200: raise exceptions.ReplayException("Upstream server refuses CONNECT request") server.establish_ssl( - self.config.clientcerts, + self.config.client_certs, sni=self.f.server_conn.sni ) r.first_line_format = "relative" @@ -76,7 +76,7 @@ class RequestReplayThread(basethread.BaseThread): server.connect() if r.scheme == "https": server.establish_ssl( - self.config.clientcerts, + self.config.client_certs, sni=self.f.server_conn.sni ) r.first_line_format = "relative" diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index 103d96cc..acc0c6e3 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -527,7 +527,7 @@ class TlsLayer(base.Layer): ciphers_server = ':'.join(ciphers_server) self.server_conn.establish_ssl( - self.config.clientcerts, + self.config.client_certs, self.server_sni, method=self.config.openssl_method_server, options=self.config.openssl_options_server, diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index def2ae31..fc584869 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -17,13 +17,6 @@ class ParseException(Exception): def get_common_options(args): - stickycookie, stickyauth = None, None - if args.stickycookie_filt: - stickycookie = args.stickycookie_filt - - if args.stickyauth_filt: - stickyauth = args.stickyauth_filt - if args.streamfile and args.streamfile[0] == args.rfile: if args.streamfile[1] == "wb": raise exceptions.OptionsError( @@ -101,8 +94,8 @@ def get_common_options(args): keep_host_header=args.keep_host_header, server_replay=args.server_replay, scripts=args.scripts, - stickycookie=stickycookie, - stickyauth=stickyauth, + stickycookie=args.stickycookie, + stickyauth=args.stickyauth, stream_large_bodies=args.stream_large_bodies, showhost=args.showhost, streamfile=args.streamfile[0] if args.streamfile else None, @@ -123,9 +116,9 @@ def get_common_options(args): certs = certs, ciphers_client = args.ciphers_client, ciphers_server = args.ciphers_server, - clientcerts = args.clientcerts, + client_certs = args.client_certs, ignore_hosts = args.ignore_hosts, - listen_host = args.listen_addr, + listen_host = args.listen_host, listen_port = args.listen_port, upstream_bind_address = args.upstream_bind_address, mode = mode, @@ -161,22 +154,14 @@ def basic_options(parser, opts): version=version.VERSION ) opts.make_parser(parser, "anticache") - parser.add_argument( - "--cadir", - action="store", type=str, dest="cadir", - help="Location of the default mitmproxy CA files. (%s)" % options.CA_DIR - ) + opts.make_parser(parser, "cadir") opts.make_parser(parser, "showhost") parser.add_argument( "-q", "--quiet", action="store_true", dest="quiet", help="Quiet." ) - parser.add_argument( - "-r", "--read-flows", - action="store", dest="rfile", - help="Read flows from file." - ) + opts.make_parser(parser, "rfile") parser.add_argument( "-s", "--script", action="append", type=str, dest="scripts", @@ -186,18 +171,8 @@ def basic_options(parser, opts): passed multiple times. """ ) - parser.add_argument( - "-t", "--stickycookie", - action="store", - dest="stickycookie_filt", - metavar="FILTER", - help="Set sticky cookie filter. Matched against requests." - ) - parser.add_argument( - "-u", "--stickyauth", - action="store", dest="stickyauth_filt", metavar="FILTER", - help="Set sticky auth filter. Matched against requests." - ) + opts.make_parser(parser, "stickycookie", metavar="FILTER") + opts.make_parser(parser, "stickyauth", metavar="FILTER") parser.add_argument( "-v", "--verbose", action="store_const", dest="verbose", const=3, @@ -254,11 +229,7 @@ def proxy_modes(parser, opts): def proxy_options(parser, opts): group = parser.add_argument_group("Proxy Options") - group.add_argument( - "-b", "--bind-address", - action="store", type=str, dest="listen_addr", - help="Address to bind proxy to (defaults to all interfaces)" - ) + opts.make_parser(group, "listen_host") group.add_argument( "-I", "--ignore", action="append", type=str, dest="ignore_hosts", @@ -292,24 +263,10 @@ def proxy_options(parser, opts): websocket = group.add_mutually_exclusive_group() opts.make_parser(websocket, "websocket") - parser.add_argument( - "--upstream-auth", - action="store", dest="upstream_auth", - type=str, - help=""" - Add HTTP Basic authentcation to upstream proxy and reverse proxy - requests. Format: username:password - """ - ) - + opts.make_parser(group, "upstream_auth", metavar="USER:PASS") opts.make_parser(group, "rawtcp") - opts.make_parser(group, "spoof_source_address") - group.add_argument( - "--upstream-bind-address", - action="store", type=str, dest="upstream_bind_address", - help="Address to bind upstream requests to (defaults to none)" - ) + opts.make_parser(group, "upstream_bind_address", metavar="ADDR") opts.make_parser(group, "keep_host_header") @@ -328,35 +285,14 @@ def proxy_ssl_options(parser, opts): 'in the PEM, it is used, else the default key in the conf dir is used. ' 'The PEM file should contain the full certificate chain, with the leaf certificate ' 'as the first entry. Can be passed multiple times.') - group.add_argument( - "--ciphers-client", action="store", - type=str, dest="ciphers_client", - help="Set supported ciphers for client connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--ciphers-server", action="store", - type=str, dest="ciphers_server", - help="Set supported ciphers for server connections. (OpenSSL Syntax)" - ) - group.add_argument( - "--client-certs", action="store", - type=str, dest="clientcerts", - help="Client certificate file or directory." - ) + opts.make_parser(group, "ciphers_server", metavar="CIPHERS") + opts.make_parser(group, "ciphers_client", metavar="CIPHERS") + opts.make_parser(group, "client_certs") opts.make_parser(group, "upstream_cert") opts.make_parser(group, "add_upstream_certs_to_client_chain") opts.make_parser(group, "ssl_insecure") - group.add_argument( - "--upstream-trusted-cadir", action="store", - dest="ssl_verify_upstream_trusted_cadir", - help="Path to a directory of trusted CA certificates for upstream " - "server verification prepared using the c_rehash tool." - ) - group.add_argument( - "--upstream-trusted-ca", action="store", - dest="ssl_verify_upstream_trusted_ca", - help="Path to a PEM formatted trusted CA certificate." - ) + opts.make_parser(group, "ssl_verify_upstream_trusted_cadir", metavar="PATH") + opts.make_parser(group, "ssl_verify_upstream_trusted_ca", metavar="PATH") group.add_argument( "--ssl-version-client", dest="ssl_version_client", action="store", @@ -375,16 +311,8 @@ def proxy_ssl_options(parser, opts): def onboarding_app(parser, opts): group = parser.add_argument_group("Onboarding App") - opts.make_parser(parser, "onboarding") - group.add_argument( - "--onboarding-host", - action="store", dest="onboarding_host", - help=""" - Domain to serve the onboarding app from. For transparent mode, use - an IP when a DNS entry for the app domain is not present. Default: - %s - """ % options.APP_HOST - ) + opts.make_parser(group, "onboarding") + opts.make_parser(group, "onboarding_host", metavar="HOST") opts.make_parser(group, "onboarding_port", metavar="PORT") @@ -488,21 +416,8 @@ def proxy_authentication(parser, opts): """ ).add_mutually_exclusive_group() opts.make_parser(group, "auth_nonanonymous") - group.add_argument( - "--singleuser", - action="store", dest="auth_singleuser", type=str, - metavar="USER", - help=""" - Allows access to a a single user, specified in the form - username:password. - """ - ) - group.add_argument( - "--htpasswd", - action="store", dest="auth_htpasswd", type=str, - metavar="PATH", - help="Allow access to users specified in an Apache htpasswd file." - ) + opts.make_parser(group, "auth_singleuser", metavar="USER:PASS") + opts.make_parser(group, "auth_htpasswd", metavar="PATH") def common_options(parser, opts): @@ -553,16 +468,8 @@ def mitmproxy(opts): "Filters", "See help in mitmproxy for filter expression syntax." ) - group.add_argument( - "-i", "--intercept", action="store", - type=str, dest="intercept", - help="Intercept filter expression." - ) - group.add_argument( - "-f", "--filter", action="store", - type=str, dest="filter", - help="Filter view expression." - ) + opts.make_parser(group, "intercept", metavar="FILTER") + opts.make_parser(group, "filter", metavar="FILTER") return parser @@ -588,13 +495,8 @@ def mitmweb(opts): group = parser.add_argument_group("Mitmweb") opts.make_parser(group, "web_open_browser") - opts.make_parser(parser, "web_port", metavar="PORT") - group.add_argument( - "--web-iface", - action="store", dest="web_iface", - metavar="IFACE", - help="Mitmweb interface." - ) + opts.make_parser(group, "web_port", metavar="PORT") + opts.make_parser(group, "web_iface", metavar="INTERFACE") opts.make_parser(group, "web_debug") common_options(parser, opts) @@ -602,9 +504,5 @@ def mitmweb(opts): "Filters", "See help in mitmproxy for filter expression syntax." ) - group.add_argument( - "-i", "--intercept", action="store", - type=str, dest="intercept", - help="Intercept filter expression." - ) + opts.make_parser(group, "intercept", metavar="FILTER") return parser diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index eb40dd14..bcfecf6f 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -342,22 +342,22 @@ class TestHTTPS(tservers.HTTPProxyTest, CommonMixin, TcpMixin): def test_clientcert_file(self): try: - self.config.clientcerts = os.path.join( + self.config.client_certs = os.path.join( tutils.test_data.path("mitmproxy/data/clientcert"), "client.pem") f = self.pathod("304") assert f.status_code == 304 assert self.server.last_log()["request"]["clientcert"]["keyinfo"] finally: - self.config.clientcerts = None + self.config.client_certs = None def test_clientcert_dir(self): try: - self.config.clientcerts = tutils.test_data.path("mitmproxy/data/clientcert") + self.config.client_certs = tutils.test_data.path("mitmproxy/data/clientcert") f = self.pathod("304") assert f.status_code == 304 assert self.server.last_log()["request"]["clientcert"]["keyinfo"] finally: - self.config.clientcerts = None + self.config.client_certs = None def test_error_post_connect(self): p = self.pathoc() diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 6e360875..5dd4a9e3 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -96,12 +96,12 @@ class TestProcessProxyOptions: def test_upstream_trusted_cadir(self): expected_dir = "/path/to/a/ca/dir" - p = self.assert_noerr("--upstream-trusted-cadir", expected_dir) + p = self.assert_noerr("--ssl-verify-upstream-trusted-cadir", expected_dir) assert p.options.ssl_verify_upstream_trusted_cadir == expected_dir def test_upstream_trusted_ca(self): expected_file = "/path/to/a/cert/file" - p = self.assert_noerr("--upstream-trusted-ca", expected_file) + p = self.assert_noerr("--ssl-verify-upstream-trusted-ca", expected_file) assert p.options.ssl_verify_upstream_trusted_ca == expected_file diff --git a/test/mitmproxy/tools/test_cmdline.py b/test/mitmproxy/tools/test_cmdline.py index b9f9d00d..bae68f83 100644 --- a/test/mitmproxy/tools/test_cmdline.py +++ b/test/mitmproxy/tools/test_cmdline.py @@ -11,8 +11,8 @@ def test_common(): assert cmdline.get_common_options(args) - args.stickycookie_filt = "foo" - args.stickyauth_filt = "foo" + args.stickycookie = "foo" + args.stickyauth = "foo" v = cmdline.get_common_options(args) assert v["stickycookie"] == "foo" assert v["stickyauth"] == "foo" -- cgit v1.2.3 From e70b46672c9bc7a1923d121f954224930bb8af7f Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 6 Mar 2017 11:56:11 +1300 Subject: Options unification: simple string sequences --- mitmproxy/options.py | 61 ++++++++++++++++++++++++++++++----- mitmproxy/optmanager.py | 9 ++++++ mitmproxy/tools/cmdline.py | 79 +++++++--------------------------------------- 3 files changed, 73 insertions(+), 76 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index ec270076..9d2d8e19 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -50,7 +50,10 @@ class Options(optmanager.OptManager): "anticomp", False, bool, "Try to convince servers to send us un-compressed data." ) - self.add_option("client_replay", [], Sequence[str]) + self.add_option( + "client_replay", [], Sequence[str], + "Replay client requests from a saved file." + ) self.add_option( "replay_kill_extra", False, bool, "Kill extra requests during replay." @@ -75,16 +78,29 @@ class Options(optmanager.OptManager): "rfile", None, Optional[str], "Read flows from file." ) - self.add_option("scripts", [], Sequence[str]) + self.add_option( + "scripts", [], Sequence[str], + """ + Run a script. Surround with quotes to pass script arguments. Can + be passed multiple times. + """ + ) self.add_option( "showhost", False, bool, "Use the Host header to construct URLs for display." ) self.add_option("replacements", [], Sequence[Union[Tuple[str, str, str], str]]) self.add_option("replacement_files", [], Sequence[Union[Tuple[str, str, str], str]]) - self.add_option("server_replay_use_headers", [], Sequence[str]) + self.add_option( + "server_replay_use_headers", [], Sequence[str], + "Request headers to be considered during replay. " + "Can be passed multiple times." + ) self.add_option("setheaders", [], Sequence[Union[Tuple[str, str, str], str]]) - self.add_option("server_replay", [], Sequence[str]) + self.add_option( + "server_replay", [], Sequence[str], + "Replay server responses from a saved file." + ) self.add_option( "stickycookie", None, Optional[str], "Set sticky cookie filter. Matched against requests." @@ -112,8 +128,21 @@ class Options(optmanager.OptManager): "server_replay_ignore_content", False, bool, "Ignore request's content while searching for a saved flow to replay." ) - self.add_option("server_replay_ignore_params", [], Sequence[str]) - self.add_option("server_replay_ignore_payload_params", [], Sequence[str]) + self.add_option( + "server_replay_ignore_params", [], Sequence[str], + """ + Request's parameters to be ignored while searching for a saved flow + to replay. Can be passed multiple times. + """ + ) + self.add_option( + "server_replay_ignore_payload_params", [], Sequence[str], + """ + Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to + be ignored while searching for a saved flow to replay. + Can be passed multiple times. + """ + ) self.add_option( "server_replay_ignore_host", False, bool, "Ignore request's destination host while searching for a saved" @@ -163,7 +192,16 @@ class Options(optmanager.OptManager): "client_certs", None, Optional[str], "Client certificate file or directory." ) - self.add_option("ignore_hosts", [], Sequence[str]) + self.add_option( + "ignore_hosts", [], Sequence[str], + """ + Ignore host and forward all traffic without processing it. In + transparent mode, it is recommended to use an IP address (range), + not the hostname. In regular mode, only SSL traffic is ignored and + the hostname should be used. The supplied value is interpreted as a + regular expression and matched on the ip or the hostname. + """ + ) self.add_option( "listen_host", "", str, "Address to bind proxy to (defaults to all interfaces)" @@ -237,7 +275,14 @@ class Options(optmanager.OptManager): "ssl_verify_upstream_trusted_ca", None, Optional[str], "Path to a PEM formatted trusted CA certificate." ) - self.add_option("tcp_hosts", [], Sequence[str]) + self.add_option( + "tcp_hosts", [], Sequence[str], + """ + Generic TCP SSL proxy mode for all hosts that match the pattern. + Similar to --ignore, but SSL connections are intercepted. The + communication contents are printed to the log in verbose mode. + """ + ) self.add_option( "intercept", None, Optional[str], diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index b9aaf6c8..24ef8f0c 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -348,5 +348,14 @@ class OptManager: help=o.help, metavar=metavar ) + elif o.typespec == typing.Sequence[str]: + parser.add_argument( + "--%s" % f, + action="append", + type=str, + dest=option, + help=o.help, + metavar=metavar + ) else: raise ValueError("Unsupported option type: %s", o.typespec) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index fc584869..0d6212bf 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -162,15 +162,7 @@ def basic_options(parser, opts): help="Quiet." ) opts.make_parser(parser, "rfile") - parser.add_argument( - "-s", "--script", - action="append", type=str, dest="scripts", - metavar='"script.py --bar"', - help=""" - Run a script. Surround with quotes to pass script arguments. Can be - passed multiple times. - """ - ) + opts.make_parser(parser, "scripts", metavar="SCRIPT") opts.make_parser(parser, "stickycookie", metavar="FILTER") opts.make_parser(parser, "stickyauth", metavar="FILTER") parser.add_argument( @@ -230,29 +222,8 @@ def proxy_modes(parser, opts): def proxy_options(parser, opts): group = parser.add_argument_group("Proxy Options") opts.make_parser(group, "listen_host") - group.add_argument( - "-I", "--ignore", - action="append", type=str, dest="ignore_hosts", - metavar="HOST", - help=""" - Ignore host and forward all traffic without processing it. In - transparent mode, it is recommended to use an IP address (range), - not the hostname. In regular mode, only SSL traffic is ignored and - the hostname should be used. The supplied value is interpreted as a - regular expression and matched on the ip or the hostname. Can be - passed multiple times. - """ - ) - group.add_argument( - "--tcp", - action="append", type=str, dest="tcp_hosts", - metavar="HOST", - help=""" - Generic TCP SSL proxy mode for all hosts that match the pattern. - Similar to --ignore, but SSL connections are intercepted. The - communication contents are printed to the log in verbose mode. - """ - ) + opts.make_parser(group, "ignore_hosts", metavar="HOST") + opts.make_parser(group, "tcp_hosts", metavar="HOST") opts.make_parser(group, "no_server") opts.make_parser(group, "listen_port", metavar="PORT") @@ -318,50 +289,22 @@ def onboarding_app(parser, opts): def client_replay(parser, opts): group = parser.add_argument_group("Client Replay") - group.add_argument( - "-c", "--client-replay", - action="append", dest="client_replay", metavar="PATH", - help="Replay client requests from a saved file." - ) + opts.make_parser(group, "client_replay", metavar="PATH") def server_replay(parser, opts): group = parser.add_argument_group("Server Replay") - group.add_argument( - "-S", "--server-replay", - action="append", dest="server_replay", metavar="PATH", - help="Replay server responses from a saved file." - ) - opts.make_parser(parser, "replay_kill_extra") - group.add_argument( - "--server-replay-use-header", - action="append", dest="server_replay_use_headers", type=str, - help="Request headers to be considered during replay. " - "Can be passed multiple times." - ) + opts.make_parser(group, "server_replay", metavar="PATH") + opts.make_parser(group, "replay_kill_extra") + opts.make_parser(group, "server_replay_use_headers", metavar="HEADER") opts.make_parser(group, "refresh_server_playback") opts.make_parser(group, "server_replay_nopop") + payload = group.add_mutually_exclusive_group() opts.make_parser(payload, "server_replay_ignore_content") - payload.add_argument( - "--replay-ignore-payload-param", - action="append", dest="server_replay_ignore_payload_params", type=str, - help=""" - Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to - be ignored while searching for a saved flow to replay. - Can be passed multiple times. - """ - ) - - group.add_argument( - "--replay-ignore-param", - action="append", dest="server_replay_ignore_params", type=str, - help=""" - Request's parameters to be ignored while searching for a saved flow - to replay. Can be passed multiple times. - """ - ) - opts.make_parser(parser, "server_replay_ignore_host") + opts.make_parser(payload, "server_replay_ignore_payload_params") + opts.make_parser(payload, "server_replay_ignore_params") + opts.make_parser(payload, "server_replay_ignore_host") def replacements(parser, opts): -- cgit v1.2.3 From 71a830c83668aaabf182a03bec0194bde1a36665 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 6 Mar 2017 13:17:53 +1300 Subject: Options unification: string choices --- mitmproxy/options.py | 48 +++++++++++++++++++++++++++++++++++---- mitmproxy/optmanager.py | 23 ++++++++++++------- mitmproxy/tools/cmdline.py | 35 ++++------------------------ test/mitmproxy/test_optmanager.py | 6 ++--- 4 files changed, 66 insertions(+), 46 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 9d2d8e19..59c57f3e 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -1,7 +1,27 @@ from typing import Tuple, Optional, Sequence, Union +from mitmproxy.net import tcp from mitmproxy import optmanager + +# We redefine these here for now to avoid importing Urwid-related guff on +# platforms that don't support it, and circular imports. We can do better using +# a lazy checker down the track. +console_palettes = [ + "lowlight", + "lowdark", + "light", + "dark", + "solarized_light", + "solarized_dark" +] +view_orders = [ + "time", + "method", + "url", + "size", +] + APP_HOST = "mitm.it" APP_PORT = 80 CA_DIR = "~/.mitmproxy" @@ -260,8 +280,20 @@ class Options(optmanager.OptManager): requests. Format: username:password """ ) - self.add_option("ssl_version_client", "secure", str) - self.add_option("ssl_version_server", "secure", str) + self.add_option( + "ssl_version_client", "secure", str, + "Set supported SSL/TLS versions for client connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which " + "is TLS1.0+.", + choices=tcp.sslversion_choices.keys(), + ) + self.add_option( + "ssl_version_server", "secure", str, + "Set supported SSL/TLS versions for server connections. " + "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, " + "which is TLS1.0+.", + choices=tcp.sslversion_choices.keys(), + ) self.add_option( "ssl_insecure", False, bool, "Do not verify upstream server SSL/TLS certificates." @@ -298,7 +330,11 @@ class Options(optmanager.OptManager): "console_focus_follow", False, bool, "Focus follows new flows." ) - self.add_option("console_palette", "dark", Optional[str]) + self.add_option( + "console_palette", "dark", Optional[str], + help="Select color palette: " + ", ".join(console_palettes), + choices=sorted(console_palettes), + ) self.add_option( "console_palette_transparent", False, bool, "Set transparent background for palette." @@ -307,10 +343,12 @@ class Options(optmanager.OptManager): "console_mouse", True, bool, "Console mouse interaction." ) - self.add_option("console_order", None, Optional[str]) self.add_option( - "console_order_reversed", False, bool, + "console_order", None, Optional[str], + "Flow sort order.", + choices=view_orders, ) + self.add_option("console_order_reversed", False, bool) self.add_option( "filter", None, Optional[str], diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 24ef8f0c..21e366c3 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -20,14 +20,15 @@ unset = object() class _Option: - __slots__ = ("name", "typespec", "value", "_default", "help") + __slots__ = ("name", "typespec", "value", "_default", "choices", "help") def __init__( self, name: str, default: typing.Any, typespec: typing.Type, - help: typing.Optional[str] + help: typing.Optional[str], + choices: typing.Optional[typing.Sequence[str]] ) -> None: typecheck.check_type(name, default, typespec) self.name = name @@ -35,6 +36,7 @@ class _Option: self.typespec = typespec self.value = unset self.help = help + self.choices = choices def __repr__(self): return "{value} [{type}]".format(value=self.current(), type=self.typespec) @@ -67,7 +69,9 @@ class _Option: return True def __deepcopy__(self, _): - o = _Option(self.name, self.default, self.typespec, self.help) + o = _Option( + self.name, self.default, self.typespec, self.help, self.choices + ) if self.has_changed(): o.value = self.current() return o @@ -98,11 +102,12 @@ class OptManager: name: str, default: typing.Any, typespec: typing.Type, - help: str = None + help: typing.Optional[str] = None, + choices: typing.Optional[typing.Sequence[str]] = None ) -> None: if name in self._options: raise ValueError("Option %s already exists" % name) - self._options[name] = _Option(name, default, typespec, help) + self._options[name] = _Option(name, default, typespec, help, choices) @contextlib.contextmanager def rollback(self, updated): @@ -337,7 +342,7 @@ class OptManager: type=int, dest=option, help=o.help, - metavar=metavar + metavar=metavar, ) elif o.typespec in (str, typing.Optional[str]): parser.add_argument( @@ -346,7 +351,8 @@ class OptManager: type=str, dest=option, help=o.help, - metavar=metavar + metavar=metavar, + choices=o.choices ) elif o.typespec == typing.Sequence[str]: parser.add_argument( @@ -355,7 +361,8 @@ class OptManager: type=str, dest=option, help=o.help, - metavar=metavar + metavar=metavar, + choices=o.choices, ) else: raise ValueError("Unsupported option type: %s", o.typespec) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 0d6212bf..2290086c 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -4,9 +4,7 @@ import os from mitmproxy import exceptions from mitmproxy import options from mitmproxy import platform -from mitmproxy.net import tcp from mitmproxy import version -from mitmproxy.addons import view CONFIG_PATH = os.path.join(options.CA_DIR, "config.yaml") @@ -264,20 +262,8 @@ def proxy_ssl_options(parser, opts): opts.make_parser(group, "ssl_insecure") opts.make_parser(group, "ssl_verify_upstream_trusted_cadir", metavar="PATH") opts.make_parser(group, "ssl_verify_upstream_trusted_ca", metavar="PATH") - group.add_argument( - "--ssl-version-client", dest="ssl_version_client", - action="store", - choices=tcp.sslversion_choices.keys(), - help="Set supported SSL/TLS versions for client connections. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." - ) - group.add_argument( - "--ssl-version-server", dest="ssl_version_server", - action="store", - choices=tcp.sslversion_choices.keys(), - help="Set supported SSL/TLS versions for server connections. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." - ) + opts.make_parser(group, "ssl_version_client", metavar="VERSION") + opts.make_parser(group, "ssl_version_server", metavar="VERSION") def onboarding_app(parser, opts): @@ -387,25 +373,14 @@ def common_options(parser, opts): def mitmproxy(opts): # Don't import mitmproxy.tools.console for mitmdump, urwid is not available # on all platforms. - from .console import palettes - parser = argparse.ArgumentParser(usage="%(prog)s [options]") common_options(parser, opts) - parser.add_argument( - "--palette", type=str, - action="store", dest="console_palette", - choices=sorted(palettes.palettes.keys()), - help="Select color palette: " + ", ".join(palettes.palettes.keys()) - ) + + opts.make_parser(parser, "console_palette") opts.make_parser(parser, "console_palette_transparent") opts.make_parser(parser, "console_eventlog") opts.make_parser(parser, "console_focus_follow") - parser.add_argument( - "--order", - type=str, dest="console_order", - choices=[o[1] for o in view.orders], - help="Flow sort order." - ) + opts.make_parser(parser, "console_order") opts.make_parser(parser, "console_mouse") group = parser.add_argument_group( "Filters", diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 44c757af..1989cc0d 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -252,14 +252,14 @@ def test_merge(): def test_option(): - o = optmanager._Option("test", 1, int, None) + o = optmanager._Option("test", 1, int, None, None) assert o.current() == 1 with pytest.raises(TypeError): o.set("foo") with pytest.raises(TypeError): - optmanager._Option("test", 1, str, None) + optmanager._Option("test", 1, str, None, None) - o2 = optmanager._Option("test", 1, int, None) + o2 = optmanager._Option("test", 1, int, None, None) assert o2 == o o2.set(5) assert o2 != o -- cgit v1.2.3 From 9b1f40da370860afbf75a34ba437092413e872ec Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 6 Mar 2017 13:42:11 +1300 Subject: Options unification: streamfile We now have one option to control this. If the path is prefixed with a "+" we append, otherwise we overwrite. --- mitmproxy/addons/streamfile.py | 6 ++++-- mitmproxy/options.py | 6 ++++-- mitmproxy/tools/cmdline.py | 27 ++------------------------- test/mitmproxy/addons/test_streamfile.py | 2 +- 4 files changed, 11 insertions(+), 30 deletions(-) diff --git a/mitmproxy/addons/streamfile.py b/mitmproxy/addons/streamfile.py index 5517e9dc..624297f2 100644 --- a/mitmproxy/addons/streamfile.py +++ b/mitmproxy/addons/streamfile.py @@ -35,11 +35,13 @@ class StreamFile: if self.stream: self.done() if options.streamfile: - if options.streamfile_append: + if options.streamfile.startswith("+"): + path = options.streamfile[1:] mode = "ab" else: + path = options.streamfile mode = "wb" - self.start_stream_to_path(options.streamfile, mode, self.filt) + self.start_stream_to_path(path, mode, self.filt) def tcp_start(self, flow): if self.stream: diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 59c57f3e..c0ac3d67 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -142,8 +142,10 @@ class Options(optmanager.OptManager): "Log verbosity." ) self.add_option("default_contentview", "auto", str) - self.add_option("streamfile", None, Optional[str]) - self.add_option("streamfile_append", False, bool) + self.add_option( + "streamfile", None, Optional[str], + help="Write flows to file. Prefix path with + to append." + ) self.add_option( "server_replay_ignore_content", False, bool, "Ignore request's content while searching for a saved flow to replay." diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 2290086c..d48fc737 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -15,18 +15,6 @@ class ParseException(Exception): def get_common_options(args): - if args.streamfile and args.streamfile[0] == args.rfile: - if args.streamfile[1] == "wb": - raise exceptions.OptionsError( - "Cannot use '{}' for both reading and writing flows. " - "Are you looking for --afile?".format(args.rfile) - ) - else: - raise exceptions.OptionsError( - "Cannot use '{}' for both reading and appending flows. " - "That would trigger an infinite loop." - ) - # Proxy config certs = [] for i in args.certs or []: @@ -96,8 +84,7 @@ def get_common_options(args): stickyauth=args.stickyauth, stream_large_bodies=args.stream_large_bodies, showhost=args.showhost, - streamfile=args.streamfile[0] if args.streamfile else None, - streamfile_append=True if args.streamfile and args.streamfile[1] == "a" else False, + streamfile=args.streamfile, verbosity=args.verbose, server_replay_nopop=args.server_replay_nopop, server_replay_ignore_content=args.server_replay_ignore_content, @@ -168,17 +155,7 @@ def basic_options(parser, opts): action="store_const", dest="verbose", const=3, help="Increase log verbosity." ) - streamfile = parser.add_mutually_exclusive_group() - streamfile.add_argument( - "-w", "--wfile", - action="store", dest="streamfile", type=lambda f: (f, "w"), - help="Write flows to file." - ) - streamfile.add_argument( - "-a", "--afile", - action="store", dest="streamfile", type=lambda f: (f, "a"), - help="Append flows to file." - ) + opts.make_parser(parser, "streamfile") opts.make_parser(parser, "anticomp") opts.make_parser(parser, "body_size_limit", metavar="SIZE") opts.make_parser(parser, "stream_large_bodies") diff --git a/test/mitmproxy/addons/test_streamfile.py b/test/mitmproxy/addons/test_streamfile.py index 89dc2af3..4105c1fc 100644 --- a/test/mitmproxy/addons/test_streamfile.py +++ b/test/mitmproxy/addons/test_streamfile.py @@ -59,7 +59,7 @@ def test_simple(): tctx.configure(sa, streamfile=None) assert rd(p)[0].response - tctx.configure(sa, streamfile=p, streamfile_append=True) + tctx.configure(sa, streamfile="+" + p) f = tflow.tflow() sa.request(f) tctx.configure(sa, streamfile=None) -- cgit v1.2.3 From f5fb6972aac9b4f17e0d0d34b2a5c347141f476c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 6 Mar 2017 15:36:52 +1300 Subject: Options unification: certs - Regularise to Sequence[str] - Move conversion and checking into proxy config object --- mitmproxy/options.py | 13 ++++++++++++- mitmproxy/proxy/config.py | 10 +++++++--- mitmproxy/tools/cmdline.py | 30 ++++-------------------------- 3 files changed, 23 insertions(+), 30 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index c0ac3d67..84ab1ecf 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -201,7 +201,18 @@ class Options(optmanager.OptManager): "cadir", CA_DIR, str, "Location of the default mitmproxy CA files. (%s)" % CA_DIR ) - self.add_option("certs", [], Sequence[Tuple[str, str]]) + self.add_option( + "certs", [], Sequence[str], + """ + Add an SSL certificate. SPEC is of the form "[domain=]path". The + domain may include a wildcard, and is equal to "*" if not specified. + The file at path is a certificate in PEM format. If a private key is + included in the PEM, it is used, else the default key in the conf + dir is used. The PEM file should contain the full certificate chain, + with the leaf certificate as the first entry. Can be passed multiple + times. + """ + ) self.add_option( "ciphers_client", DEFAULT_CLIENT_CIPHERS, str, "Set supported ciphers for client connections. (OpenSSL Syntax)" diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 778fd306..61d8e1b7 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -105,14 +105,18 @@ class ProxyConfig: ) self.client_certs = client_certs - for spec, cert in options.certs: - cert = os.path.expanduser(cert) + for c in options.certs: + parts = c.split("=", 1) + if len(parts) == 1: + parts = ["*", parts[0]] + + cert = os.path.expanduser(parts[1]) if not os.path.exists(cert): raise exceptions.OptionsError( "Certificate file does not exist: %s" % cert ) try: - self.certstore.add_cert_file(spec, cert) + self.certstore.add_cert_file(parts[0], cert) except crypto.Error: raise exceptions.OptionsError( "Invalid certificate format: %s" % cert diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index d48fc737..5e83e828 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -15,14 +15,6 @@ class ParseException(Exception): def get_common_options(args): - # Proxy config - certs = [] - for i in args.certs or []: - parts = i.split("=", 1) - if len(parts) == 1: - parts = ["*", parts[0]] - certs.append(parts) - # Establish proxy mode c = 0 mode, upstream_server = "regular", None @@ -50,6 +42,7 @@ def get_common_options(args): "are mutually exclusive. Read the docs on proxy modes " "to understand why." ) + if args.add_upstream_certs_to_client_chain and not args.upstream_cert: raise exceptions.OptionsError( "The no-upstream-cert and add-upstream-certs-to-client-chain " @@ -98,7 +91,7 @@ def get_common_options(args): add_upstream_certs_to_client_chain = args.add_upstream_certs_to_client_chain, body_size_limit = args.body_size_limit, cadir = args.cadir, - certs = certs, + certs = args.certs, ciphers_client = args.ciphers_client, ciphers_server = args.ciphers_server, client_certs = args.client_certs, @@ -219,18 +212,7 @@ def proxy_options(parser, opts): def proxy_ssl_options(parser, opts): # TODO: Agree to consistently either use "upstream" or "server". group = parser.add_argument_group("SSL") - group.add_argument( - "--cert", - dest='certs', - type=str, - metavar="SPEC", - action="append", - help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' - 'The domain may include a wildcard, and is equal to "*" if not specified. ' - 'The file at path is a certificate in PEM format. If a private key is included ' - 'in the PEM, it is used, else the default key in the conf dir is used. ' - 'The PEM file should contain the full certificate chain, with the leaf certificate ' - 'as the first entry. Can be passed multiple times.') + opts.make_parser(group, "certs", metavar="SPEC") opts.make_parser(group, "ciphers_server", metavar="CIPHERS") opts.make_parser(group, "ciphers_client", metavar="CIPHERS") opts.make_parser(group, "client_certs") @@ -331,9 +313,7 @@ def common_options(parser, opts): "--conf", type=str, dest="conf", default=CONFIG_PATH, metavar="PATH", - help=""" - Configuration file - """ + help="Configuration file" ) basic_options(parser, opts) proxy_modes(parser, opts) @@ -348,8 +328,6 @@ def common_options(parser, opts): def mitmproxy(opts): - # Don't import mitmproxy.tools.console for mitmdump, urwid is not available - # on all platforms. parser = argparse.ArgumentParser(usage="%(prog)s [options]") common_options(parser, opts) -- cgit v1.2.3 From 82163a1e680d608cdf2fb33f30dfcc6e142547a7 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 6 Mar 2017 20:30:49 +1300 Subject: Unify mode specification We now have: --mode regular (the default) --mode transparent --mode socks5 --mode reverse:SPEC --mode upstream:SPEC Where SPEC is a host specification. --- mitmproxy/addons/core.py | 29 +++++++++++--- mitmproxy/master.py | 2 +- mitmproxy/options.py | 15 +++++++- mitmproxy/proxy/config.py | 8 ++-- mitmproxy/proxy/protocol/http.py | 2 +- mitmproxy/proxy/protocol/http_replay.py | 2 +- mitmproxy/proxy/server.py | 4 +- mitmproxy/tools/cmdline.py | 67 +-------------------------------- mitmproxy/tools/console/statusbar.py | 10 +---- test/mitmproxy/addons/test_core.py | 18 +++++++++ test/mitmproxy/proxy/test_server.py | 4 +- test/mitmproxy/test_flow.py | 3 +- test/mitmproxy/test_proxy.py | 37 +++--------------- test/mitmproxy/tservers.py | 8 ++-- 14 files changed, 80 insertions(+), 129 deletions(-) diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index 5d2cf57b..8a2ea525 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -3,18 +3,37 @@ checked by other addons. """ from mitmproxy import exceptions +from mitmproxy import options +from mitmproxy import platform from mitmproxy.utils import human class Core: - def configure(self, options, updated): - if "body_size_limit" in updated and options.body_size_limit: + def configure(self, opts, updated): + if "body_size_limit" in updated and opts.body_size_limit: try: - options._processed["body_size_limit"] = human.parse_size( - options.body_size_limit + opts._processed["body_size_limit"] = human.parse_size( + opts.body_size_limit ) except ValueError as e: raise exceptions.OptionsError( "Invalid body size limit specification: %s" % - options.body_size_limit + opts.body_size_limit + ) + if "mode" in updated: + mode = opts.mode + if mode.startswith("reverse:") or mode.startswith("upstream:"): + spec = options.get_mode_spec(mode) + if not spec: + raise exceptions.OptionsError( + "Invalid mode specification: %s" % mode + ) + elif mode == "transparent": + if not platform.original_addr: + raise exceptions.OptionsError( + "Transparent mode not supported on this platform." + ) + elif mode not in ["regular", "socks5"]: + raise exceptions.OptionsError( + "Invalid mode specification: %s" % mode ) diff --git a/mitmproxy/master.py b/mitmproxy/master.py index 633f32aa..8855452c 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -148,7 +148,7 @@ class Master: Loads a flow """ if isinstance(f, http.HTTPFlow): - if self.server and self.options.mode == "reverse": + if self.server and self.options.mode.startswith("reverse:"): f.request.host = self.server.config.upstream_server.address[0] f.request.port = self.server.config.upstream_server.address[1] f.request.scheme = self.server.config.upstream_server.scheme diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 84ab1ecf..41e02031 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -22,6 +22,11 @@ view_orders = [ "size", ] + +def get_mode_spec(m): + return m.split(":", maxsplit=1)[1] + + APP_HOST = "mitm.it" APP_PORT = 80 CA_DIR = "~/.mitmproxy" @@ -247,7 +252,14 @@ class Options(optmanager.OptManager): "upstream_bind_address", "", str, "Address to bind upstream requests to (defaults to none)" ) - self.add_option("mode", "regular", str) + self.add_option( + "mode", "regular", str, + """ + Mode can be "regular", "transparent", "socks5", "reverse:SPEC", + or "upstream:SPEC". For reverse and upstream proxy modes, SPEC + is proxy specification in the form of "http[s]://host[:port]". + """ + ) self.add_option( "upstream_cert", True, bool, "Connect to upstream server to look up certificate details." @@ -285,7 +297,6 @@ class Options(optmanager.OptManager): "Use the client's IP for server-side connections. " "Combine with --upstream-bind-address to spoof a fixed source address." ) - self.add_option("upstream_server", None, Optional[str]) self.add_option( "upstream_auth", None, Optional[str], """ diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 61d8e1b7..9cf2b00f 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -121,7 +121,7 @@ class ProxyConfig: raise exceptions.OptionsError( "Invalid certificate format: %s" % cert ) - - self.upstream_server = None - if options.upstream_server: - self.upstream_server = parse_server_spec(options.upstream_server) + m = options.mode + if m.startswith("upstream:") or m.startswith("reverse:"): + spec = moptions.get_mode_spec(options.mode) + self.upstream_server = parse_server_spec(spec) diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index d2f6d374..d9e53fed 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -290,7 +290,7 @@ class HttpLayer(base.Layer): request.first_line_format = "relative" # update host header in reverse proxy mode - if self.config.options.mode == "reverse" and not self.config.options.keep_host_header: + if self.config.options.mode.startswith("reverse:") and not self.config.options.keep_host_header: f.request.host_header = self.config.upstream_server.address[0] # Determine .scheme, .host and .port attributes for inline scripts. For diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py index 161816e7..25867871 100644 --- a/mitmproxy/proxy/protocol/http_replay.py +++ b/mitmproxy/proxy/protocol/http_replay.py @@ -45,7 +45,7 @@ class RequestReplayThread(basethread.BaseThread): if not self.f.response: # In all modes, we directly connect to the server displayed - if self.config.options.mode == "upstream": + if self.config.options.mode.startswith("upstream:"): server_address = self.config.upstream_server.address server = connections.ServerConnection(server_address, (self.config.options.listen_host, 0)) server.connect() diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 8082cb64..16692234 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -85,14 +85,14 @@ class ConnectionHandler: ) mode = self.config.options.mode - if mode == "upstream": + if mode.startswith("upstream:"): return modes.HttpUpstreamProxy( root_ctx, self.config.upstream_server.address ) elif mode == "transparent": return modes.TransparentProxy(root_ctx) - elif mode == "reverse": + elif mode.startswith("reverse:"): server_tls = self.config.upstream_server.scheme == "https" return modes.ReverseProxy( root_ctx, diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 5e83e828..1160e1e4 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -3,7 +3,6 @@ import os from mitmproxy import exceptions from mitmproxy import options -from mitmproxy import platform from mitmproxy import version @@ -15,34 +14,6 @@ class ParseException(Exception): def get_common_options(args): - # Establish proxy mode - c = 0 - mode, upstream_server = "regular", None - if args.transparent_proxy: - c += 1 - if not platform.original_addr: - raise exceptions.OptionsError( - "Transparent mode not supported on this platform." - ) - mode = "transparent" - if args.socks_proxy: - c += 1 - mode = "socks5" - if args.reverse_proxy: - c += 1 - mode = "reverse" - upstream_server = args.reverse_proxy - if args.upstream_proxy: - c += 1 - mode = "upstream" - upstream_server = args.upstream_proxy - if c > 1: - raise exceptions.OptionsError( - "Transparent, SOCKS5, reverse and upstream proxy mode " - "are mutually exclusive. Read the docs on proxy modes " - "to understand why." - ) - if args.add_upstream_certs_to_client_chain and not args.upstream_cert: raise exceptions.OptionsError( "The no-upstream-cert and add-upstream-certs-to-client-chain " @@ -99,7 +70,7 @@ def get_common_options(args): listen_host = args.listen_host, listen_port = args.listen_port, upstream_bind_address = args.upstream_bind_address, - mode = mode, + mode = args.mode, upstream_cert = args.upstream_cert, spoof_source_address = args.spoof_source_address, @@ -108,7 +79,6 @@ def get_common_options(args): websocket = args.websocket, rawtcp = args.rawtcp, - upstream_server = upstream_server, upstream_auth = args.upstream_auth, ssl_version_client = args.ssl_version_client, ssl_version_server = args.ssl_version_server, @@ -154,39 +124,6 @@ def basic_options(parser, opts): opts.make_parser(parser, "stream_large_bodies") -def proxy_modes(parser, opts): - group = parser.add_argument_group("Proxy Modes") - group.add_argument( - "-R", "--reverse", - action="store", - type=str, - dest="reverse_proxy", - help=""" - Forward all requests to upstream HTTP server: - http[s]://host[:port]. Clients can always connect both - via HTTPS and HTTP, the connection to the server is - determined by the specified scheme. - """ - ) - group.add_argument( - "--socks", - action="store_true", dest="socks_proxy", - help="Set SOCKS5 proxy mode." - ) - group.add_argument( - "-T", "--transparent", - action="store_true", dest="transparent_proxy", - help="Set transparent proxy mode." - ) - group.add_argument( - "-U", "--upstream", - action="store", - type=str, - dest="upstream_proxy", - help="Forward all requests to upstream proxy server: http://host[:port]" - ) - - def proxy_options(parser, opts): group = parser.add_argument_group("Proxy Options") opts.make_parser(group, "listen_host") @@ -315,8 +252,8 @@ def common_options(parser, opts): metavar="PATH", help="Configuration file" ) + opts.make_parser(parser, "mode") basic_options(parser, opts) - proxy_modes(parser, opts) proxy_options(parser, opts) proxy_ssl_options(parser, opts) onboarding_app(parser, opts) diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index a5611b28..3e524972 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -2,7 +2,6 @@ import os.path import urwid -import mitmproxy.net.http.url from mitmproxy.tools.console import common from mitmproxy.tools.console import pathedit from mitmproxy.tools.console import signals @@ -234,13 +233,8 @@ class StatusBar(urwid.WidgetWrap): if opts: r.append("[%s]" % (":".join(opts))) - if self.master.options.mode in ["reverse", "upstream"]: - dst = self.master.server.config.upstream_server - r.append("[dest:%s]" % mitmproxy.net.http.url.unparse( - dst.scheme, - dst.address[0], - dst.address[1], - )) + if self.master.options.mode != "regular": + r.append("[%s]" % self.master.options.mode) if self.master.options.scripts: r.append("[") r.append(("heading_key", "s")) diff --git a/test/mitmproxy/addons/test_core.py b/test/mitmproxy/addons/test_core.py index 95272716..533eb58e 100644 --- a/test/mitmproxy/addons/test_core.py +++ b/test/mitmproxy/addons/test_core.py @@ -2,6 +2,7 @@ from mitmproxy import exceptions from mitmproxy.addons import core from mitmproxy.test import taddons import pytest +from unittest import mock def test_simple(): @@ -11,3 +12,20 @@ def test_simple(): tctx.configure(sa, body_size_limit = "invalid") tctx.configure(sa, body_size_limit = "1m") assert tctx.options._processed["body_size_limit"] + + +@mock.patch("mitmproxy.platform.original_addr", None) +def test_no_transparent(): + sa = core.Core() + with taddons.context() as tctx: + with pytest.raises(Exception, match="Transparent mode not supported"): + tctx.configure(sa, mode = "transparent") + + +@mock.patch("mitmproxy.platform.original_addr") +def test_modes(m): + sa = core.Core() + with taddons.context() as tctx: + tctx.configure(sa, mode = "reverse:http://localhost") + with pytest.raises(Exception, match="Invalid mode"): + tctx.configure(sa, mode = "reverse:") diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index bcfecf6f..b90840ab 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -10,7 +10,7 @@ from mitmproxy import options from mitmproxy.addons import script from mitmproxy.addons import proxyauth from mitmproxy import http -from mitmproxy.proxy.config import HostMatcher, parse_server_spec +from mitmproxy.proxy.config import HostMatcher import mitmproxy.net.http from mitmproxy.net import tcp from mitmproxy.net import socks @@ -579,8 +579,6 @@ class TestHttps2Http(tservers.ReverseProxyTest): @classmethod def get_options(cls): opts = super().get_options() - s = parse_server_spec(opts.upstream_server) - opts.upstream_server = "http://{}:{}".format(s.address[0], s.address[1]) return opts def pathoc(self, ssl, sni=None): diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index 0ac3bfd6..f4d32cbb 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -63,8 +63,7 @@ class TestSerialize: r = self._treader() s = tservers.TestState() opts = options.Options( - mode="reverse", - upstream_server="https://use-this-domain" + mode="reverse:https://use-this-domain" ) conf = ProxyConfig(opts) fm = master.Master(opts, DummyServer(conf)) diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 5dd4a9e3..784a7d84 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -49,31 +49,6 @@ class TestProcessProxyOptions: with tutils.tmpdir() as cadir: self.assert_noerr("--cadir", cadir) - @mock.patch("mitmproxy.platform.original_addr", None) - def test_no_transparent(self): - with pytest.raises(Exception, match="Transparent mode not supported"): - self.p("-T") - - @mock.patch("mitmproxy.platform.original_addr") - def test_modes(self, _): - self.assert_noerr("-R", "http://localhost") - with pytest.raises(Exception, match="expected one argument"): - self.p("-R") - with pytest.raises(Exception, match="Invalid server specification"): - self.p("-R", "reverse") - - self.assert_noerr("-T") - - self.assert_noerr("-U", "http://localhost") - with pytest.raises(Exception, match="Invalid server specification"): - self.p("-U", "upstream") - - self.assert_noerr("--upstream-auth", "test:test") - with pytest.raises(Exception, match="expected one argument"): - self.p("--upstream-auth") - with pytest.raises(Exception, match="mutually exclusive"): - self.p("-R", "http://localhost", "-T") - def test_client_certs(self): with tutils.tmpdir() as cadir: self.assert_noerr("--client-certs", cadir) @@ -131,19 +106,19 @@ class TestDummyServer: class TestConnectionHandler: def test_fatal_error(self, capsys): - config = mock.Mock() - root_layer = mock.Mock() - root_layer.side_effect = RuntimeError - config.options.mode.return_value = root_layer + opts = options.Options() + pconf = config.ProxyConfig(opts) + channel = mock.Mock() def ask(_, x): - return x + raise RuntimeError + channel.ask = ask c = ConnectionHandler( mock.MagicMock(), ("127.0.0.1", 8080), - config, + pconf, channel ) c.handle() diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index 9a289ae5..a8aaa358 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -288,7 +288,7 @@ class ReverseProxyTest(ProxyTestBase): @classmethod def get_options(cls): opts = ProxyTestBase.get_options() - opts.upstream_server = "".join( + s = "".join( [ "https" if cls.ssl else "http", "://", @@ -296,7 +296,7 @@ class ReverseProxyTest(ProxyTestBase): str(cls.server.port) ] ) - opts.mode = "reverse" + opts.mode = "reverse:" + s return opts def pathoc(self, sni=None): @@ -373,9 +373,9 @@ class ChainProxyTest(ProxyTestBase): def get_options(cls): opts = super().get_options() if cls.chain: # First proxy is in normal mode. + s = "http://127.0.0.1:%s" % cls.chain[0].port opts.update( - mode="upstream", - upstream_server="http://127.0.0.1:%s" % cls.chain[0].port + mode="upstream:" + s, ) return opts -- cgit v1.2.3 From edfd62e42af921d0031ff95b7cf41ab1b6608a47 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 6 Mar 2017 20:58:51 +1300 Subject: Replacements and setheaders are always strings Instead of having two representations we have one canonical specification. Fixing the editor in console is left ot a further patch. --- mitmproxy/options.py | 20 ++++++++++++++++---- mitmproxy/tools/cmdline.py | 24 +++--------------------- test/mitmproxy/addons/test_replace.py | 24 ++++++++++++------------ test/mitmproxy/addons/test_setheaders.py | 14 +++++++------- 4 files changed, 38 insertions(+), 44 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 41e02031..788df7e9 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -1,4 +1,4 @@ -from typing import Tuple, Optional, Sequence, Union +from typing import Optional, Sequence from mitmproxy.net import tcp from mitmproxy import optmanager @@ -114,14 +114,26 @@ class Options(optmanager.OptManager): "showhost", False, bool, "Use the Host header to construct URLs for display." ) - self.add_option("replacements", [], Sequence[Union[Tuple[str, str, str], str]]) - self.add_option("replacement_files", [], Sequence[Union[Tuple[str, str, str], str]]) + self.add_option( + "replacements", [], Sequence[str], + "Replacement patterns." + ) + self.add_option( + "replacement_files", [], Sequence[str], + """ + Replacement pattern, where the replacement clause is a path to a + file. + """ + ) self.add_option( "server_replay_use_headers", [], Sequence[str], "Request headers to be considered during replay. " "Can be passed multiple times." ) - self.add_option("setheaders", [], Sequence[Union[Tuple[str, str, str], str]]) + self.add_option( + "setheaders", [], Sequence[str], + help="Header set pattern." + ) self.add_option( "server_replay", [], Sequence[str], "Replay server responses from a saved file." diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 1160e1e4..ef99ba37 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -198,21 +198,8 @@ def replacements(parser, opts): for more information. """.strip() ) - group.add_argument( - "--replace", - action="append", type=str, dest="replacements", - metavar="PATTERN", - help="Replacement pattern." - ) - group.add_argument( - "--replace-from-file", - action="append", type=str, dest="replacement_files", - metavar="PATH", - help=""" - Replacement pattern, where the replacement clause is a path to a - file. - """ - ) + opts.make_parser(group, "replacements", metavar="PATTERN") + opts.make_parser(group, "replacement_files", metavar="PATTERN") def set_headers(parser, opts): @@ -224,12 +211,7 @@ def set_headers(parser, opts): documentation for more information. """.strip() ) - group.add_argument( - "--setheader", - action="append", type=str, dest="setheaders", - metavar="PATTERN", - help="Header set pattern." - ) + opts.make_parser(group, "setheaders", metavar="PATTERN") def proxy_authentication(parser, opts): diff --git a/test/mitmproxy/addons/test_replace.py b/test/mitmproxy/addons/test_replace.py index 126c6e3d..8c280c51 100644 --- a/test/mitmproxy/addons/test_replace.py +++ b/test/mitmproxy/addons/test_replace.py @@ -22,11 +22,11 @@ class TestReplace: def test_configure(self): r = replace.Replace() with taddons.context() as tctx: - tctx.configure(r, replacements=[("one", "two", "three")]) + tctx.configure(r, replacements=["one/two/three"]) with pytest.raises(Exception, match="Invalid filter pattern"): - tctx.configure(r, replacements=[("~b", "two", "three")]) + tctx.configure(r, replacements=["/~b/two/three"]) with pytest.raises(Exception, match="Invalid regular expression"): - tctx.configure(r, replacements=[("foo", "+", "three")]) + tctx.configure(r, replacements=["/foo/+/three"]) tctx.configure(r, replacements=["/a/b/c/"]) def test_simple(self): @@ -35,8 +35,8 @@ class TestReplace: tctx.configure( r, replacements = [ - ("~q", "foo", "bar"), - ("~s", "foo", "bar"), + "/~q/foo/bar", + "/~s/foo/bar", ] ) f = tflow.tflow() @@ -58,10 +58,10 @@ class TestUpstreamProxy(tservers.HTTPUpstreamProxyTest): self.proxy.tmaster.addons.add(sa) self.proxy.tmaster.options.replacements = [ - ("~q", "foo", "bar"), - ("~q", "bar", "baz"), - ("~q", "foo", "oh noes!"), - ("~s", "baz", "ORLY") + "/~q/foo/bar", + "/~q/bar/baz", + "/~q/foo/oh noes!", + "/~s/baz/ORLY" ] p = self.pathoc() with p.connect(): @@ -81,9 +81,9 @@ class TestReplaceFile: tctx.configure( r, replacement_files = [ - ("~q", "foo", rp), - ("~s", "foo", rp), - ("~b nonexistent", "nonexistent", "nonexistent"), + "/~q/foo/" + rp, + "/~s/foo/" + rp, + "/~b nonexistent/nonexistent/nonexistent", ] ) f = tflow.tflow() diff --git a/test/mitmproxy/addons/test_setheaders.py b/test/mitmproxy/addons/test_setheaders.py index 6355f2be..3aaee7f4 100644 --- a/test/mitmproxy/addons/test_setheaders.py +++ b/test/mitmproxy/addons/test_setheaders.py @@ -21,7 +21,7 @@ class TestSetHeaders: sh = setheaders.SetHeaders() with taddons.context() as tctx: with pytest.raises(Exception, match="Invalid setheader filter pattern"): - tctx.configure(sh, setheaders = [("~b", "one", "two")]) + tctx.configure(sh, setheaders = ["/~b/one/two"]) tctx.configure(sh, setheaders = ["/foo/bar/voing"]) def test_setheaders(self): @@ -30,8 +30,8 @@ class TestSetHeaders: tctx.configure( sh, setheaders = [ - ("~q", "one", "two"), - ("~s", "one", "three") + "/~q/one/two", + "/~s/one/three" ] ) f = tflow.tflow() @@ -47,8 +47,8 @@ class TestSetHeaders: tctx.configure( sh, setheaders = [ - ("~s", "one", "two"), - ("~s", "one", "three") + "/~s/one/two", + "/~s/one/three" ] ) f = tflow.tflow(resp=True) @@ -60,8 +60,8 @@ class TestSetHeaders: tctx.configure( sh, setheaders = [ - ("~q", "one", "two"), - ("~q", "one", "three") + "/~q/one/two", + "/~q/one/three" ] ) f = tflow.tflow() -- cgit v1.2.3 From 2312cf6fb08e6c3f208a9bfe4af93c193dfd9ab1 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 7 Mar 2017 08:49:19 +1300 Subject: Automate slurping up command-line options Now that options are completely regular, we can automate reading them out of arguments, rather than listing them out by hand like savages. --- mitmproxy/addons/core.py | 7 ++++ mitmproxy/optmanager.py | 3 ++ mitmproxy/tools/cmdline.py | 79 +----------------------------------- mitmproxy/tools/main.py | 45 ++++---------------- test/mitmproxy/test_proxy.py | 3 +- test/mitmproxy/tools/test_cmdline.py | 10 +---- 6 files changed, 23 insertions(+), 124 deletions(-) diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index 8a2ea525..3de1638c 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -10,6 +10,13 @@ from mitmproxy.utils import human class Core: def configure(self, opts, updated): + if opts.add_upstream_certs_to_client_chain and not opts.upstream_cert: + raise exceptions.OptionsError( + "The no-upstream-cert and add-upstream-certs-to-client-chain " + "options are mutually exclusive. If no-upstream-cert is enabled " + "then the upstream certificate is not retrieved before generating " + "the client certificate chain." + ) if "body_size_limit" in updated and opts.body_size_limit: try: opts._processed["body_size_limit"] = human.parse_size( diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 21e366c3..319fe622 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -162,6 +162,9 @@ class OptManager: def keys(self): return set(self._options.keys()) + def __contains__(self, k): + return k in self._options + def reset(self): """ Restore defaults for all options. diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index ef99ba37..f3c3b6d0 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -1,7 +1,6 @@ import argparse import os -from mitmproxy import exceptions from mitmproxy import options from mitmproxy import version @@ -13,82 +12,6 @@ class ParseException(Exception): pass -def get_common_options(args): - if args.add_upstream_certs_to_client_chain and not args.upstream_cert: - raise exceptions.OptionsError( - "The no-upstream-cert and add-upstream-certs-to-client-chain " - "options are mutually exclusive. If no-upstream-cert is enabled " - "then the upstream certificate is not retrieved before generating " - "the client certificate chain." - ) - - if args.quiet: - args.verbose = 0 - - return dict( - onboarding=args.onboarding, - onboarding_host=args.onboarding_host, - onboarding_port=args.onboarding_port, - - anticache=args.anticache, - anticomp=args.anticomp, - client_replay=args.client_replay, - replay_kill_extra=args.replay_kill_extra, - no_server=args.no_server, - refresh_server_playback=args.refresh_server_playback, - server_replay_use_headers=args.server_replay_use_headers, - rfile=args.rfile, - replacements=args.replacements, - replacement_files=args.replacement_files, - setheaders=args.setheaders, - keep_host_header=args.keep_host_header, - server_replay=args.server_replay, - scripts=args.scripts, - stickycookie=args.stickycookie, - stickyauth=args.stickyauth, - stream_large_bodies=args.stream_large_bodies, - showhost=args.showhost, - streamfile=args.streamfile, - verbosity=args.verbose, - server_replay_nopop=args.server_replay_nopop, - server_replay_ignore_content=args.server_replay_ignore_content, - server_replay_ignore_params=args.server_replay_ignore_params, - server_replay_ignore_payload_params=args.server_replay_ignore_payload_params, - server_replay_ignore_host=args.server_replay_ignore_host, - - auth_nonanonymous = args.auth_nonanonymous, - auth_singleuser = args.auth_singleuser, - auth_htpasswd = args.auth_htpasswd, - add_upstream_certs_to_client_chain = args.add_upstream_certs_to_client_chain, - body_size_limit = args.body_size_limit, - cadir = args.cadir, - certs = args.certs, - ciphers_client = args.ciphers_client, - ciphers_server = args.ciphers_server, - client_certs = args.client_certs, - ignore_hosts = args.ignore_hosts, - listen_host = args.listen_host, - listen_port = args.listen_port, - upstream_bind_address = args.upstream_bind_address, - mode = args.mode, - upstream_cert = args.upstream_cert, - spoof_source_address = args.spoof_source_address, - - http2 = args.http2, - http2_priority = args.http2_priority, - websocket = args.websocket, - rawtcp = args.rawtcp, - - upstream_auth = args.upstream_auth, - ssl_version_client = args.ssl_version_client, - ssl_version_server = args.ssl_version_server, - ssl_insecure = args.ssl_insecure, - ssl_verify_upstream_trusted_cadir = args.ssl_verify_upstream_trusted_cadir, - ssl_verify_upstream_trusted_ca = args.ssl_verify_upstream_trusted_ca, - tcp_hosts = args.tcp_hosts, - ) - - def basic_options(parser, opts): parser.add_argument( '--version', @@ -272,7 +195,7 @@ def mitmdump(opts): opts.make_parser(parser, "keepserving") opts.make_parser(parser, "flow_detail", metavar = "LEVEL") parser.add_argument( - 'filter', + 'filter_args', nargs="...", help=""" Filter view expression, used to only show flows that match a certain filter. diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index c0293f28..a231c421 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -38,6 +38,14 @@ def process_options(parser, options, args): if args.version: print(debug.dump_system_info()) sys.exit(0) + if args.quiet: + args.flow_detail = 0 + + adict = {} + for n in dir(args): + if n in options: + adict[n] = getattr(args, n) + options.merge(adict) debug.register_info_dumpers() pconf = config.ProxyConfig(options) @@ -67,21 +75,6 @@ def mitmproxy(args=None): # pragma: no cover try: console_options.load_paths(args.conf) - console_options.merge(cmdline.get_common_options(args)) - console_options.merge( - dict( - console_palette = args.console_palette, - console_palette_transparent = args.console_palette_transparent, - console_eventlog = args.console_eventlog, - console_focus_follow = args.console_focus_follow, - console_mouse = args.console_mouse, - console_order = args.console_order, - - filter = args.filter, - intercept = args.intercept, - ) - ) - server = process_options(parser, console_options, args) m = console.master.ConsoleMaster(console_options, server) except exceptions.OptionsError as e: @@ -101,21 +94,9 @@ def mitmdump(args=None): # pragma: no cover dump_options = options.Options() parser = cmdline.mitmdump(dump_options) args = parser.parse_args(args) - if args.quiet: - args.flow_detail = 0 - master = None try: dump_options.load_paths(args.conf) - dump_options.merge(cmdline.get_common_options(args)) - dump_options.merge( - dict( - flow_detail = args.flow_detail, - keepserving = args.keepserving, - filtstr = " ".join(args.filter) if args.filter else None, - ) - ) - server = process_options(parser, dump_options, args) master = dump.DumpMaster(dump_options, server) @@ -145,16 +126,6 @@ def mitmweb(args=None): # pragma: no cover try: web_options.load_paths(args.conf) - web_options.merge(cmdline.get_common_options(args)) - web_options.merge( - dict( - intercept = args.intercept, - web_open_browser = args.web_open_browser, - web_debug = args.web_debug, - web_iface = args.web_iface, - web_port = args.web_port, - ) - ) server = process_options(parser, web_options, args) m = web.master.WebMaster(web_options, server) except exceptions.OptionsError as e: diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 784a7d84..6afda18c 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -6,6 +6,7 @@ import pytest from mitmproxy.tools import cmdline +from mitmproxy.tools import main from mitmproxy import options from mitmproxy.proxy import ProxyConfig from mitmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler @@ -33,7 +34,7 @@ class TestProcessProxyOptions: opts = options.Options() cmdline.common_options(parser, opts) args = parser.parse_args(args=args) - opts.merge(cmdline.get_common_options(args)) + main.process_options(parser, opts, args) pconf = config.ProxyConfig(opts) return parser, pconf diff --git a/test/mitmproxy/tools/test_cmdline.py b/test/mitmproxy/tools/test_cmdline.py index bae68f83..65cfeb07 100644 --- a/test/mitmproxy/tools/test_cmdline.py +++ b/test/mitmproxy/tools/test_cmdline.py @@ -1,5 +1,6 @@ import argparse from mitmproxy.tools import cmdline +from mitmproxy.tools import main from mitmproxy import options @@ -8,14 +9,7 @@ def test_common(): opts = options.Options() cmdline.common_options(parser, opts) args = parser.parse_args(args=[]) - - assert cmdline.get_common_options(args) - - args.stickycookie = "foo" - args.stickyauth = "foo" - v = cmdline.get_common_options(args) - assert v["stickycookie"] == "foo" - assert v["stickyauth"] == "foo" + assert main.process_options(parser, opts, args) def test_mitmproxy(): -- cgit v1.2.3 From b51df9a0b1cc9da1d7381421dfac481da6205a7a Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 7 Mar 2017 10:28:15 +1300 Subject: Cleanups and test coverage --- mitmproxy/addons/replace.py | 5 +---- mitmproxy/addons/setheaders.py | 5 +---- mitmproxy/optmanager.py | 6 ++++-- mitmproxy/test/taddons.py | 5 +++-- mitmproxy/tools/cmdline.py | 4 ---- mitmproxy/tools/dump.py | 6 +----- mitmproxy/tools/main.py | 2 +- test/mitmproxy/addons/test_core.py | 12 ++++++++++++ test/mitmproxy/test_optmanager.py | 21 +++++++++++++++++++-- test/mitmproxy/tools/test_dump.py | 5 +++-- 10 files changed, 45 insertions(+), 26 deletions(-) diff --git a/mitmproxy/addons/replace.py b/mitmproxy/addons/replace.py index 34bb40c2..0d0c3aa5 100644 --- a/mitmproxy/addons/replace.py +++ b/mitmproxy/addons/replace.py @@ -57,10 +57,7 @@ class _ReplaceBase: if self.optionName in updated: lst = [] for rep in getattr(options, self.optionName): - if isinstance(rep, str): - fpatt, rex, s = parse_hook(rep) - else: - fpatt, rex, s = rep + fpatt, rex, s = parse_hook(rep) flt = flowfilter.parse(fpatt) if not flt: diff --git a/mitmproxy/addons/setheaders.py b/mitmproxy/addons/setheaders.py index 95cf9a09..9e60eabd 100644 --- a/mitmproxy/addons/setheaders.py +++ b/mitmproxy/addons/setheaders.py @@ -54,10 +54,7 @@ class SetHeaders: if "setheaders" in updated: self.lst = [] for shead in options.setheaders: - if isinstance(shead, str): - fpatt, header, value = parse_setheader(shead) - else: - fpatt, header, value = shead + fpatt, header, value = parse_setheader(shead) flt = flowfilter.parse(fpatt) if not flt: diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 319fe622..3768b52c 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -110,7 +110,7 @@ class OptManager: self._options[name] = _Option(name, default, typespec, help, choices) @contextlib.contextmanager - def rollback(self, updated): + def rollback(self, updated, reraise=False): old = copy.deepcopy(self._options) try: yield @@ -120,6 +120,8 @@ class OptManager: # Rollback self.__dict__["_options"] = old self.changed.send(self, updated=updated) + if reraise: + raise e def subscribe(self, func, opts): """ @@ -337,7 +339,7 @@ class OptManager: dest=option, help=o.help ) - parser.set_defaults(**{option: o.default}) + parser.set_defaults(**{option: None}) elif o.typespec in (int, typing.Optional[int]): parser.add_argument( "--%s" % f, diff --git a/mitmproxy/test/taddons.py b/mitmproxy/test/taddons.py index c3e19cc7..8d6baa12 100644 --- a/mitmproxy/test/taddons.py +++ b/mitmproxy/test/taddons.py @@ -61,5 +61,6 @@ class context: Options object with the given keyword arguments, then calls the configure method on the addon with the updated value. """ - self.options.update(**kwargs) - addon.configure(self.options, kwargs.keys()) + with self.options.rollback(kwargs.keys(), reraise=True): + self.options.update(**kwargs) + addon.configure(self.options, kwargs.keys()) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index f3c3b6d0..13577997 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -8,10 +8,6 @@ from mitmproxy import version CONFIG_PATH = os.path.join(options.CA_DIR, "config.yaml") -class ParseException(Exception): - pass - - def basic_options(parser, opts): parser.add_argument( '--version', diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index 6b862475..f64f2241 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -6,10 +6,6 @@ from mitmproxy import master from mitmproxy.addons import dumper, termlog -class DumpError(Exception): - pass - - class DumpMaster(master.Master): def __init__( @@ -38,7 +34,7 @@ class DumpMaster(master.Master): self.load_flows_file(options.rfile) except exceptions.FlowReadException as v: self.add_log("Flow file corrupted.", "error") - raise DumpError(v) + raise exceptions.OptionsError(v) @controller.handler def log(self, e): diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index a231c421..05877e2c 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -105,7 +105,7 @@ def mitmdump(args=None): # pragma: no cover signal.signal(signal.SIGTERM, cleankill) master.run() - except (dump.DumpError, exceptions.OptionsError) as e: + except exceptions.OptionsError as e: print("mitmdump: %s" % e, file=sys.stderr) sys.exit(1) except (KeyboardInterrupt, RuntimeError): diff --git a/test/mitmproxy/addons/test_core.py b/test/mitmproxy/addons/test_core.py index 533eb58e..7b9e9614 100644 --- a/test/mitmproxy/addons/test_core.py +++ b/test/mitmproxy/addons/test_core.py @@ -13,6 +13,18 @@ def test_simple(): tctx.configure(sa, body_size_limit = "1m") assert tctx.options._processed["body_size_limit"] + with pytest.raises(exceptions.OptionsError, match="mutually exclusive"): + tctx.configure( + sa, + add_upstream_certs_to_client_chain = True, + upstream_cert = False + ) + with pytest.raises(exceptions.OptionsError, match="Invalid mode"): + tctx.configure( + sa, + mode = "Flibble" + ) + @mock.patch("mitmproxy.platform.original_addr", None) def test_no_transparent(): diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 1989cc0d..a38662d5 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -38,6 +38,12 @@ class TM(optmanager.OptManager): self.add_option("one", None, typing.Optional[str]) +def test_add_option(): + o = TO() + with pytest.raises(ValueError, match="already exists"): + o.add_option("one", None, typing.Optional[int]) + + def test_defaults(): o = TD2() defaults = { @@ -162,6 +168,8 @@ def test_rollback(): def err(opts, updated): if opts.one == 10: raise exceptions.OptionsError() + if opts.bool is True: + raise exceptions.OptionsError() o.changed.connect(sub) o.changed.connect(err) @@ -169,15 +177,24 @@ def test_rollback(): assert o.one is None o.one = 10 + o.bool = True assert isinstance(recerr[0]["exc"], exceptions.OptionsError) assert o.one is None - assert len(rec) == 2 + assert o.bool is False + assert len(rec) == 4 assert rec[0].one == 10 assert rec[1].one is None + assert rec[2].bool is True + assert rec[3].bool is False + + with pytest.raises(exceptions.OptionsError): + with o.rollback(set(["one"]), reraise=True): + raise exceptions.OptionsError() -def test_repr(): +def test_simple(): assert repr(TO()) + assert "one" in TO() def test_serialize(): diff --git a/test/mitmproxy/tools/test_dump.py b/test/mitmproxy/tools/test_dump.py index 3210b0bb..2542ec4b 100644 --- a/test/mitmproxy/tools/test_dump.py +++ b/test/mitmproxy/tools/test_dump.py @@ -3,6 +3,7 @@ import pytest from unittest import mock from mitmproxy import proxy +from mitmproxy import exceptions from mitmproxy import log from mitmproxy import controller from mitmproxy import options @@ -26,9 +27,9 @@ class TestDumpMaster(tservers.MasterTest): self.mkmaster(None, rfile=p), 1, b"", ) - with pytest.raises(dump.DumpError): + with pytest.raises(exceptions.OptionsError): self.mkmaster(None, rfile="/nonexistent") - with pytest.raises(dump.DumpError): + with pytest.raises(exceptions.OptionsError): self.mkmaster(None, rfile="test_dump.py") def test_has_error(self): -- cgit v1.2.3 From 3d9c2233be328ecd1c243ddb6a180a0f1a3ef6c5 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 7 Mar 2017 10:51:24 +1300 Subject: Flatten commandline arg structure, extract common run func for tools --- mitmproxy/tools/cmdline.py | 69 ++++++++++++++------------------------ mitmproxy/tools/main.py | 82 +++++++++++++++------------------------------- 2 files changed, 51 insertions(+), 100 deletions(-) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 13577997..e789aeab 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -8,7 +8,7 @@ from mitmproxy import version CONFIG_PATH = os.path.join(options.CA_DIR, "config.yaml") -def basic_options(parser, opts): +def common_options(parser, opts): parser.add_argument( '--version', action='store_true', @@ -20,30 +20,38 @@ def basic_options(parser, opts): help="show program's short version number and exit", version=version.VERSION ) - opts.make_parser(parser, "anticache") - opts.make_parser(parser, "cadir") - opts.make_parser(parser, "showhost") parser.add_argument( "-q", "--quiet", action="store_true", dest="quiet", help="Quiet." ) - opts.make_parser(parser, "rfile") - opts.make_parser(parser, "scripts", metavar="SCRIPT") - opts.make_parser(parser, "stickycookie", metavar="FILTER") - opts.make_parser(parser, "stickyauth", metavar="FILTER") parser.add_argument( "-v", "--verbose", action="store_const", dest="verbose", const=3, help="Increase log verbosity." ) + parser.add_argument( + "--conf", + type=str, dest="conf", default=CONFIG_PATH, + metavar="PATH", + help="Configuration file" + ) + + # Basic options + opts.make_parser(parser, "mode") + opts.make_parser(parser, "anticache") + opts.make_parser(parser, "cadir") + opts.make_parser(parser, "showhost") + opts.make_parser(parser, "rfile") + opts.make_parser(parser, "scripts", metavar="SCRIPT") + opts.make_parser(parser, "stickycookie", metavar="FILTER") + opts.make_parser(parser, "stickyauth", metavar="FILTER") opts.make_parser(parser, "streamfile") opts.make_parser(parser, "anticomp") opts.make_parser(parser, "body_size_limit", metavar="SIZE") opts.make_parser(parser, "stream_large_bodies") - -def proxy_options(parser, opts): + # Proxy options group = parser.add_argument_group("Proxy Options") opts.make_parser(group, "listen_host") opts.make_parser(group, "ignore_hosts", metavar="HOST") @@ -64,9 +72,7 @@ def proxy_options(parser, opts): opts.make_parser(group, "upstream_bind_address", metavar="ADDR") opts.make_parser(group, "keep_host_header") - -def proxy_ssl_options(parser, opts): - # TODO: Agree to consistently either use "upstream" or "server". + # Proxy SSL options group = parser.add_argument_group("SSL") opts.make_parser(group, "certs", metavar="SPEC") opts.make_parser(group, "ciphers_server", metavar="CIPHERS") @@ -80,20 +86,17 @@ def proxy_ssl_options(parser, opts): opts.make_parser(group, "ssl_version_client", metavar="VERSION") opts.make_parser(group, "ssl_version_server", metavar="VERSION") - -def onboarding_app(parser, opts): + # Onboarding app group = parser.add_argument_group("Onboarding App") opts.make_parser(group, "onboarding") opts.make_parser(group, "onboarding_host", metavar="HOST") opts.make_parser(group, "onboarding_port", metavar="PORT") - -def client_replay(parser, opts): + # Client replay group = parser.add_argument_group("Client Replay") opts.make_parser(group, "client_replay", metavar="PATH") - -def server_replay(parser, opts): + # Server replay group = parser.add_argument_group("Server Replay") opts.make_parser(group, "server_replay", metavar="PATH") opts.make_parser(group, "replay_kill_extra") @@ -107,8 +110,7 @@ def server_replay(parser, opts): opts.make_parser(payload, "server_replay_ignore_params") opts.make_parser(payload, "server_replay_ignore_host") - -def replacements(parser, opts): + # Replacements group = parser.add_argument_group( "Replacements", """ @@ -120,8 +122,7 @@ def replacements(parser, opts): opts.make_parser(group, "replacements", metavar="PATTERN") opts.make_parser(group, "replacement_files", metavar="PATTERN") - -def set_headers(parser, opts): + # Set headers group = parser.add_argument_group( "Set Headers", """ @@ -132,8 +133,7 @@ def set_headers(parser, opts): ) opts.make_parser(group, "setheaders", metavar="PATTERN") - -def proxy_authentication(parser, opts): + # Proxy authentication group = parser.add_argument_group( "Proxy Authentication", """ @@ -146,25 +146,6 @@ def proxy_authentication(parser, opts): opts.make_parser(group, "auth_htpasswd", metavar="PATH") -def common_options(parser, opts): - parser.add_argument( - "--conf", - type=str, dest="conf", default=CONFIG_PATH, - metavar="PATH", - help="Configuration file" - ) - opts.make_parser(parser, "mode") - basic_options(parser, opts) - proxy_options(parser, opts) - proxy_ssl_options(parser, opts) - onboarding_app(parser, opts) - client_replay(parser, opts) - server_replay(parser, opts) - replacements(parser, opts) - set_headers(parser, opts) - proxy_authentication(parser, opts) - - def mitmproxy(opts): parser = argparse.ArgumentParser(usage="%(prog)s [options]") common_options(parser, opts) diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 05877e2c..0478d3c4 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -47,7 +47,6 @@ def process_options(parser, options, args): adict[n] = getattr(args, n) options.merge(adict) - debug.register_info_dumpers() pconf = config.ProxyConfig(options) if options.no_server: return server.DummyServer(pconf) @@ -59,46 +58,18 @@ def process_options(parser, options, args): sys.exit(1) -def mitmproxy(args=None): # pragma: no cover - if os.name == "nt": - print("Error: mitmproxy's console interface is not supported on Windows. " - "You can run mitmdump or mitmweb instead.", file=sys.stderr) - sys.exit(1) - from mitmproxy.tools import console - - version_check.check_pyopenssl_version() - assert_utf8_env() - - console_options = options.Options() - parser = cmdline.mitmproxy(console_options) - args = parser.parse_args(args) - - try: - console_options.load_paths(args.conf) - server = process_options(parser, console_options, args) - m = console.master.ConsoleMaster(console_options, server) - except exceptions.OptionsError as e: - print("mitmproxy: %s" % e, file=sys.stderr) - sys.exit(1) - try: - m.run() - except (KeyboardInterrupt, RuntimeError): - pass - - -def mitmdump(args=None): # pragma: no cover - from mitmproxy.tools import dump - +def run(MasterKlass, args): # pragma: no cover version_check.check_pyopenssl_version() + debug.register_info_dumpers() - dump_options = options.Options() - parser = cmdline.mitmdump(dump_options) + opts = options.Options() + parser = cmdline.mitmdump(opts) args = parser.parse_args(args) master = None try: - dump_options.load_paths(args.conf) - server = process_options(parser, dump_options, args) - master = dump.DumpMaster(dump_options, server) + opts.load_paths(args.conf) + server = process_options(parser, opts, args) + master = MasterKlass(opts, server) def cleankill(*args, **kwargs): master.shutdown() @@ -106,32 +77,31 @@ def mitmdump(args=None): # pragma: no cover signal.signal(signal.SIGTERM, cleankill) master.run() except exceptions.OptionsError as e: - print("mitmdump: %s" % e, file=sys.stderr) + print("%s: %s" % (sys.argv[0], e), file=sys.stderr) sys.exit(1) except (KeyboardInterrupt, RuntimeError): pass - if master is None or master.has_errored: - print("mitmdump: errors occurred during run", file=sys.stderr) + if master is None or getattr(master, "has_errored", None): + print("%s: errors occurred during run" % sys.argv[0], file=sys.stderr) sys.exit(1) -def mitmweb(args=None): # pragma: no cover - from mitmproxy.tools import web +def mitmproxy(args=None): # pragma: no cover + if os.name == "nt": + print("Error: mitmproxy's console interface is not supported on Windows. " + "You can run mitmdump or mitmweb instead.", file=sys.stderr) + sys.exit(1) + assert_utf8_env() - version_check.check_pyopenssl_version() + from mitmproxy.tools import console + run(console.master.ConsoleMaster, args) - web_options = options.Options() - parser = cmdline.mitmweb(web_options) - args = parser.parse_args(args) - try: - web_options.load_paths(args.conf) - server = process_options(parser, web_options, args) - m = web.master.WebMaster(web_options, server) - except exceptions.OptionsError as e: - print("mitmweb: %s" % e, file=sys.stderr) - sys.exit(1) - try: - m.run() - except (KeyboardInterrupt, RuntimeError): - pass +def mitmdump(args=None): # pragma: no cover + from mitmproxy.tools import dump + run(dump.DumpMaster, args) + + +def mitmweb(args=None): # pragma: no cover + from mitmproxy.tools import web + run(web.master.WebMaster, args) \ No newline at end of file -- cgit v1.2.3 From d13df40753a3e8259ef4d2f25d9cb0c1e8141223 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 7 Mar 2017 10:52:28 +1300 Subject: Fix lint --- mitmproxy/tools/main.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 0478d3c4..36133555 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -104,4 +104,4 @@ def mitmdump(args=None): # pragma: no cover def mitmweb(args=None): # pragma: no cover from mitmproxy.tools import web - run(web.master.WebMaster, args) \ No newline at end of file + run(web.master.WebMaster, args) -- cgit v1.2.3 From 99a6b0dbc1cc68bbcdbae1060a6f75ff4f0e9bf8 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 7 Mar 2017 13:16:28 +1300 Subject: Add --options that dumps annotated option defaults --- mitmproxy/optmanager.py | 20 ++++++++++++++++++++ mitmproxy/tools/cmdline.py | 5 +++++ mitmproxy/tools/main.py | 14 +++++++++----- test/mitmproxy/test_optmanager.py | 5 +++++ 4 files changed, 39 insertions(+), 5 deletions(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 3768b52c..5b156841 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -6,6 +6,7 @@ import functools import weakref import os import typing +import textwrap import ruamel.yaml @@ -371,3 +372,22 @@ class OptManager: ) else: raise ValueError("Unsupported option type: %s", o.typespec) + + +def dump(opts): + """ + Dumps an annotated file with all options. + """ + # Sort data + s = ruamel.yaml.comments.CommentedMap() + for k in sorted(opts.keys()): + o = opts._options[k] + s[k] = o.default + if o.help: + s.yaml_set_comment_before_after_key( + k, + before = "\n" + "\n".join(textwrap.wrap( + textwrap.dedent(o.help.strip()) + )), + ) + return ruamel.yaml.round_trip_dump(s) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index e789aeab..7b0da34f 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -20,6 +20,11 @@ def common_options(parser, opts): help="show program's short version number and exit", version=version.VERSION ) + parser.add_argument( + '--options', + action='store_true', + help="Dump all options", + ) parser.add_argument( "-q", "--quiet", action="store_true", dest="quiet", diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 36133555..5c58d995 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -12,6 +12,7 @@ import signal # noqa from mitmproxy.tools import cmdline # noqa from mitmproxy import exceptions # noqa from mitmproxy import options # noqa +from mitmproxy import optmanager # noqa from mitmproxy.proxy import config # noqa from mitmproxy.proxy import server # noqa from mitmproxy.utils import version_check # noqa @@ -34,21 +35,24 @@ def assert_utf8_env(): sys.exit(1) -def process_options(parser, options, args): +def process_options(parser, opts, args): if args.version: print(debug.dump_system_info()) sys.exit(0) + if args.options: + print(optmanager.dump(opts)) + sys.exit(0) if args.quiet: args.flow_detail = 0 adict = {} for n in dir(args): - if n in options: + if n in opts: adict[n] = getattr(args, n) - options.merge(adict) + opts.merge(adict) - pconf = config.ProxyConfig(options) - if options.no_server: + pconf = config.ProxyConfig(opts) + if opts.no_server: return server.DummyServer(pconf) else: try: diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index a38662d5..6729b155 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -280,3 +280,8 @@ def test_option(): assert o2 == o o2.set(5) assert o2 != o + + +def test_dump(): + o = options.Options() + assert optmanager.dump(o) \ No newline at end of file -- cgit v1.2.3 From 79f5883c2fbe475269a02920aaaad053e797abec Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 7 Mar 2017 13:47:39 +1300 Subject: Option spacing and coverage --- mitmproxy/options.py | 10 +++++----- mitmproxy/optmanager.py | 18 +++++++++--------- test/mitmproxy/test_optmanager.py | 24 +++++++++++++++++++++++- 3 files changed, 37 insertions(+), 15 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 788df7e9..1b5b700a 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -170,16 +170,16 @@ class Options(optmanager.OptManager): self.add_option( "server_replay_ignore_params", [], Sequence[str], """ - Request's parameters to be ignored while searching for a saved flow - to replay. Can be passed multiple times. + Request's parameters to be ignored while searching for a saved + flow to replay. Can be passed multiple times. """ ) self.add_option( "server_replay_ignore_payload_params", [], Sequence[str], """ - Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to - be ignored while searching for a saved flow to replay. - Can be passed multiple times. + Request's payload parameters (application/x-www-form-urlencoded + or multipart/form-data) to be ignored while searching for a + saved flow to replay. Can be passed multiple times. """ ) self.add_option( diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 5b156841..beb9084c 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -323,30 +323,30 @@ class OptManager: options=options ) - def make_parser(self, parser, option, metavar=None): - o = self._options[option] - f = option.replace("_", "-") + def make_parser(self, parser, optname, metavar=None): + o = self._options[optname] + f = optname.replace("_", "-") if o.typespec == bool: g = parser.add_mutually_exclusive_group(required=False) g.add_argument( "--%s" % f, action="store_true", - dest=option, + dest=optname, help=o.help ) g.add_argument( "--no-%s" % f, action="store_false", - dest=option, + dest=optname, help=o.help ) - parser.set_defaults(**{option: None}) + parser.set_defaults(**{optname: None}) elif o.typespec in (int, typing.Optional[int]): parser.add_argument( "--%s" % f, action="store", type=int, - dest=option, + dest=optname, help=o.help, metavar=metavar, ) @@ -355,7 +355,7 @@ class OptManager: "--%s" % f, action="store", type=str, - dest=option, + dest=optname, help=o.help, metavar=metavar, choices=o.choices @@ -365,7 +365,7 @@ class OptManager: "--%s" % f, action="append", type=str, - dest=option, + dest=optname, help=o.help, metavar=metavar, choices=o.choices, diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 6729b155..010fc339 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -2,6 +2,7 @@ import copy import os import pytest import typing +import argparse from mitmproxy import options from mitmproxy import optmanager @@ -284,4 +285,25 @@ def test_option(): def test_dump(): o = options.Options() - assert optmanager.dump(o) \ No newline at end of file + assert optmanager.dump(o) + + +class TTypes(optmanager.OptManager): + def __init__(self): + super().__init__() + self.add_option("str", "str", str) + self.add_option("bool", False, bool) + self.add_option("int", 0, int) + self.add_option("seqstr", [], typing.Sequence[str]) + self.add_option("unknown", 0.0, float) + + +def test_make_parser(): + parser = argparse.ArgumentParser() + opts = TTypes() + opts.make_parser(parser, "str") + opts.make_parser(parser, "bool") + opts.make_parser(parser, "int") + opts.make_parser(parser, "seqstr") + with pytest.raises(ValueError): + opts.make_parser(parser, "unknown") -- cgit v1.2.3 From ac3b0d69cc29b5469f6b4cc55af528f6266d42cf Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 7 Mar 2017 14:27:50 +1300 Subject: Add the --set option to set options directly The --set option is a universal flag for setting options. Some examples: Turn on a boolean: mitmdump --set onboarding=false Add a value to a sequence: mitumdupm --set setheaders=/foo/bar/voing Zero a sequence: mitumdupm --set setheaders --- mitmproxy/optmanager.py | 39 ++++++++++++++++++++++++++++++++++++++ mitmproxy/tools/cmdline.py | 24 +++++++++++++++++------ mitmproxy/tools/main.py | 3 +++ test/mitmproxy/test_optmanager.py | 40 +++++++++++++++++++++++++++++++++++++++ 4 files changed, 100 insertions(+), 6 deletions(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index beb9084c..eb56ef2d 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -323,6 +323,45 @@ class OptManager: options=options ) + def set(self, spec): + parts = spec.split("=", maxsplit=1) + if len(parts) == 1: + optname, optval = parts[0], None + else: + optname, optval = parts[0], parts[1] + o = self._options[optname] + + if o.typespec in (str, typing.Optional[str]): + setattr(self, optname, optval) + elif o.typespec in (int, typing.Optional[int]): + if optval: + try: + optval = int(optval) + except ValueError: + raise exceptions.OptionsError("Not an integer: %s" % optval) + setattr(self, optname, optval) + elif o.typespec == bool: + if not optval or optval == "true": + v = True + elif optval == "false": + v = False + else: + raise exceptions.OptionsError( + "Boolean must be \"true\", \"false\", or have the value " "omitted (a synonym for \"true\")." + ) + setattr(self, optname, v) + elif o.typespec == typing.Sequence[str]: + if not optval: + setattr(self, optname, []) + else: + setattr( + self, + optname, + getattr(self, optname) + [optval] + ) + else: + raise ValueError("Unsupported option type: %s", o.typespec) + def make_parser(self, parser, optname, metavar=None): o = self._options[optname] f = optname.replace("_", "-") diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 7b0da34f..ea77acfd 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -25,6 +25,24 @@ def common_options(parser, opts): action='store_true', help="Dump all options", ) + parser.add_argument( + "--conf", + type=str, dest="conf", default=CONFIG_PATH, + metavar="PATH", + help="Read options from a configuration file" + ) + parser.add_argument( + "--set", + type=str, dest="setoptions", default=[], + action="append", + metavar="option[=value]", + help=""" + Set an option. When the value is omitted, booleans are set to true, + strings and integers are set to None (if permitted), and sequences + are emptied. + """ + ) + parser.add_argument( "-q", "--quiet", action="store_true", dest="quiet", @@ -35,12 +53,6 @@ def common_options(parser, opts): action="store_const", dest="verbose", const=3, help="Increase log verbosity." ) - parser.add_argument( - "--conf", - type=str, dest="conf", default=CONFIG_PATH, - metavar="PATH", - help="Configuration file" - ) # Basic options opts.make_parser(parser, "mode") diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 5c58d995..7c1e7b32 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -45,6 +45,9 @@ def process_options(parser, opts, args): if args.quiet: args.flow_detail = 0 + for i in args.setoptions: + opts.set(i) + adict = {} for n in dir(args): if n in opts: diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 010fc339..9311e82d 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -292,8 +292,10 @@ class TTypes(optmanager.OptManager): def __init__(self): super().__init__() self.add_option("str", "str", str) + self.add_option("optstr", "optstr", typing.Optional[str]) self.add_option("bool", False, bool) self.add_option("int", 0, int) + self.add_option("optint", 0, typing.Optional[int]) self.add_option("seqstr", [], typing.Sequence[str]) self.add_option("unknown", 0.0, float) @@ -307,3 +309,41 @@ def test_make_parser(): opts.make_parser(parser, "seqstr") with pytest.raises(ValueError): opts.make_parser(parser, "unknown") + + +def test_set(): + opts = TTypes() + + opts.set("str=foo") + assert opts.str == "foo" + with pytest.raises(TypeError): + opts.set("str") + + opts.set("optstr=foo") + assert opts.optstr == "foo" + opts.set("optstr") + assert opts.optstr is None + + opts.set("bool=false") + assert opts.bool is False + opts.set("bool") + assert opts.bool is True + opts.set("bool=true") + assert opts.bool is True + with pytest.raises(exceptions.OptionsError): + opts.set("bool=wobble") + + opts.set("int=1") + assert opts.int == 1 + with pytest.raises(exceptions.OptionsError): + opts.set("int=wobble") + opts.set("optint") + assert opts.optint is None + + assert opts.seqstr == [] + opts.set("seqstr=foo") + assert opts.seqstr == ["foo"] + opts.set("seqstr=bar") + assert opts.seqstr == ["foo", "bar"] + opts.set("seqstr") + assert opts.seqstr == [] -- cgit v1.2.3 From 10db254791480a35028554bc2821b9c8002e26ca Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 7 Mar 2017 14:46:28 +1300 Subject: Remove less commonly used command-line options We now have --set, so only options that really deserve it get a dedicated flag. I'm inclined to strip this back even more. Feel free to argue that YOUR favourite option deserves special treatment here. --- mitmproxy/optmanager.py | 9 ++++----- mitmproxy/tools/cmdline.py | 46 +------------------------------------------- test/mitmproxy/test_proxy.py | 24 ----------------------- 3 files changed, 5 insertions(+), 74 deletions(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index eb56ef2d..e5277371 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -368,14 +368,13 @@ class OptManager: if o.typespec == bool: g = parser.add_mutually_exclusive_group(required=False) g.add_argument( - "--%s" % f, - action="store_true", + "--no-%s" % f, + action="store_false", dest=optname, - help=o.help ) g.add_argument( - "--no-%s" % f, - action="store_false", + "--%s" % f, + action="store_true", dest=optname, help=o.help ) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index ea77acfd..d4f3aa69 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -57,7 +57,6 @@ def common_options(parser, opts): # Basic options opts.make_parser(parser, "mode") opts.make_parser(parser, "anticache") - opts.make_parser(parser, "cadir") opts.make_parser(parser, "showhost") opts.make_parser(parser, "rfile") opts.make_parser(parser, "scripts", metavar="SCRIPT") @@ -65,49 +64,21 @@ def common_options(parser, opts): opts.make_parser(parser, "stickyauth", metavar="FILTER") opts.make_parser(parser, "streamfile") opts.make_parser(parser, "anticomp") - opts.make_parser(parser, "body_size_limit", metavar="SIZE") - opts.make_parser(parser, "stream_large_bodies") # Proxy options group = parser.add_argument_group("Proxy Options") - opts.make_parser(group, "listen_host") + opts.make_parser(group, "listen_host", metavar="HOST") opts.make_parser(group, "ignore_hosts", metavar="HOST") opts.make_parser(group, "tcp_hosts", metavar="HOST") opts.make_parser(group, "no_server") opts.make_parser(group, "listen_port", metavar="PORT") - - http2 = group.add_mutually_exclusive_group() - opts.make_parser(http2, "http2") - opts.make_parser(http2, "http2_priority") - - websocket = group.add_mutually_exclusive_group() - opts.make_parser(websocket, "websocket") - opts.make_parser(group, "upstream_auth", metavar="USER:PASS") opts.make_parser(group, "rawtcp") - opts.make_parser(group, "spoof_source_address") - opts.make_parser(group, "upstream_bind_address", metavar="ADDR") - opts.make_parser(group, "keep_host_header") # Proxy SSL options group = parser.add_argument_group("SSL") opts.make_parser(group, "certs", metavar="SPEC") - opts.make_parser(group, "ciphers_server", metavar="CIPHERS") - opts.make_parser(group, "ciphers_client", metavar="CIPHERS") - opts.make_parser(group, "client_certs") - opts.make_parser(group, "upstream_cert") - opts.make_parser(group, "add_upstream_certs_to_client_chain") opts.make_parser(group, "ssl_insecure") - opts.make_parser(group, "ssl_verify_upstream_trusted_cadir", metavar="PATH") - opts.make_parser(group, "ssl_verify_upstream_trusted_ca", metavar="PATH") - opts.make_parser(group, "ssl_version_client", metavar="VERSION") - opts.make_parser(group, "ssl_version_server", metavar="VERSION") - - # Onboarding app - group = parser.add_argument_group("Onboarding App") - opts.make_parser(group, "onboarding") - opts.make_parser(group, "onboarding_host", metavar="HOST") - opts.make_parser(group, "onboarding_port", metavar="PORT") # Client replay group = parser.add_argument_group("Client Replay") @@ -117,16 +88,8 @@ def common_options(parser, opts): group = parser.add_argument_group("Server Replay") opts.make_parser(group, "server_replay", metavar="PATH") opts.make_parser(group, "replay_kill_extra") - opts.make_parser(group, "server_replay_use_headers", metavar="HEADER") - opts.make_parser(group, "refresh_server_playback") opts.make_parser(group, "server_replay_nopop") - payload = group.add_mutually_exclusive_group() - opts.make_parser(payload, "server_replay_ignore_content") - opts.make_parser(payload, "server_replay_ignore_payload_params") - opts.make_parser(payload, "server_replay_ignore_params") - opts.make_parser(payload, "server_replay_ignore_host") - # Replacements group = parser.add_argument_group( "Replacements", @@ -167,12 +130,7 @@ def mitmproxy(opts): parser = argparse.ArgumentParser(usage="%(prog)s [options]") common_options(parser, opts) - opts.make_parser(parser, "console_palette") - opts.make_parser(parser, "console_palette_transparent") opts.make_parser(parser, "console_eventlog") - opts.make_parser(parser, "console_focus_follow") - opts.make_parser(parser, "console_order") - opts.make_parser(parser, "console_mouse") group = parser.add_argument_group( "Filters", "See help in mitmproxy for filter expression syntax." @@ -186,7 +144,6 @@ def mitmdump(opts): parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]") common_options(parser, opts) - opts.make_parser(parser, "keepserving") opts.make_parser(parser, "flow_detail", metavar = "LEVEL") parser.add_argument( 'filter_args', @@ -206,7 +163,6 @@ def mitmweb(opts): opts.make_parser(group, "web_open_browser") opts.make_parser(group, "web_port", metavar="PORT") opts.make_parser(group, "web_iface", metavar="INTERFACE") - opts.make_parser(group, "web_debug") common_options(parser, opts) group = parser.add_argument_group( diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 6afda18c..7a49c530 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -1,4 +1,3 @@ -import os import argparse from unittest import mock from OpenSSL import SSL @@ -46,19 +45,6 @@ class TestProcessProxyOptions: def test_simple(self): assert self.p() - def test_cadir(self): - with tutils.tmpdir() as cadir: - self.assert_noerr("--cadir", cadir) - - def test_client_certs(self): - with tutils.tmpdir() as cadir: - self.assert_noerr("--client-certs", cadir) - self.assert_noerr( - "--client-certs", - os.path.join(tutils.test_data.path("mitmproxy/data/clientcert"), "client.pem")) - with pytest.raises(Exception, match="path does not exist"): - self.p("--client-certs", "nonexistent") - def test_certs(self): self.assert_noerr( "--cert", @@ -70,16 +56,6 @@ class TestProcessProxyOptions: p = self.assert_noerr("--ssl-insecure") assert p.openssl_verification_mode_server == SSL.VERIFY_NONE - def test_upstream_trusted_cadir(self): - expected_dir = "/path/to/a/ca/dir" - p = self.assert_noerr("--ssl-verify-upstream-trusted-cadir", expected_dir) - assert p.options.ssl_verify_upstream_trusted_cadir == expected_dir - - def test_upstream_trusted_ca(self): - expected_file = "/path/to/a/cert/file" - p = self.assert_noerr("--ssl-verify-upstream-trusted-ca", expected_file) - assert p.options.ssl_verify_upstream_trusted_ca == expected_file - class TestProxyServer: -- cgit v1.2.3 From 320d8848abea644965256da651bad8a4a8e92678 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 7 Mar 2017 15:23:46 +1300 Subject: Options tweaks - Regularise presentation and wording - Help is mandatory for all options - Auto-generate wording to say that sequence options can be passed multiple times on the command-line --- mitmproxy/options.py | 167 +++++++++++++++++++++++--------------- mitmproxy/optmanager.py | 19 ++--- mitmproxy/tools/cmdline.py | 22 +---- test/mitmproxy/test_optmanager.py | 34 ++++---- 4 files changed, 131 insertions(+), 111 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 1b5b700a..21ee257d 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -55,20 +55,20 @@ class Options(optmanager.OptManager): self.add_option( "onboarding_host", APP_HOST, str, """ - Domain to serve the onboarding app from. For transparent mode, use - an IP when a DNS entry for the app domain is not present. Default: - %s + Domain to serve the onboarding app from. For transparent mode, use + an IP when a DNS entry for the app domain is not present. Default: + %s """ % APP_HOST ) self.add_option( "onboarding_port", APP_PORT, int, - help="Port to serve the onboarding app from." + "Port to serve the onboarding app from." ) self.add_option( "anticache", False, bool, """ - Strip out request headers that might cause the server to return - 304-not-modified. + Strip out request headers that might cause the server to return + 304-not-modified. """ ) self.add_option( @@ -93,11 +93,17 @@ class Options(optmanager.OptManager): ) self.add_option( "server_replay_nopop", False, bool, - "Disable response pop from response flow. " - "This makes it possible to replay same response multiple times." + """ + Disable response pop from response flow. This makes it possible to + replay same response multiple times. + """ ) self.add_option( "refresh_server_playback", True, bool, + """ + Refresh server replay responses by adjusting date, expires and + last-modified headers, as well as adjusting cookie expiration. + """ ) self.add_option( "rfile", None, Optional[str], @@ -106,8 +112,7 @@ class Options(optmanager.OptManager): self.add_option( "scripts", [], Sequence[str], """ - Run a script. Surround with quotes to pass script arguments. Can - be passed multiple times. + Execute a script. """ ) self.add_option( @@ -116,23 +121,28 @@ class Options(optmanager.OptManager): ) self.add_option( "replacements", [], Sequence[str], - "Replacement patterns." + """ + Replacement patterns of the form "/pattern/regex/replacement", where + the separator can be any character. + """ ) self.add_option( "replacement_files", [], Sequence[str], """ - Replacement pattern, where the replacement clause is a path to a - file. + Replacement pattern, where the replacement clause is a path to a + file. """ ) self.add_option( "server_replay_use_headers", [], Sequence[str], - "Request headers to be considered during replay. " - "Can be passed multiple times." + "Request headers to be considered during replay." ) self.add_option( "setheaders", [], Sequence[str], - help="Header set pattern." + """ + Header set pattern of the form "/pattern/header/value", where the + separator can be any character. + """ ) self.add_option( "server_replay", [], Sequence[str], @@ -149,19 +159,22 @@ class Options(optmanager.OptManager): self.add_option( "stream_large_bodies", None, Optional[str], """ - Stream data to the client if response body exceeds the given - threshold. If streamed, the body will not be stored in any way. - Understands k/m/g suffixes, i.e. 3m for 3 megabytes. + Stream data to the client if response body exceeds the given + threshold. If streamed, the body will not be stored in any way. + Understands k/m/g suffixes, i.e. 3m for 3 megabytes. """ ) self.add_option( "verbosity", 2, int, "Log verbosity." ) - self.add_option("default_contentview", "auto", str) + self.add_option( + "default_contentview", "auto", str, + "The default content view mode." + ) self.add_option( "streamfile", None, Optional[str], - help="Write flows to file. Prefix path with + to append." + "Write flows to file. Prefix path with + to append." ) self.add_option( "server_replay_ignore_content", False, bool, @@ -170,22 +183,24 @@ class Options(optmanager.OptManager): self.add_option( "server_replay_ignore_params", [], Sequence[str], """ - Request's parameters to be ignored while searching for a saved - flow to replay. Can be passed multiple times. + Request's parameters to be ignored while searching for a saved flow + to replay. Can be passed multiple times. """ ) self.add_option( "server_replay_ignore_payload_params", [], Sequence[str], """ - Request's payload parameters (application/x-www-form-urlencoded - or multipart/form-data) to be ignored while searching for a - saved flow to replay. Can be passed multiple times. + Request's payload parameters (application/x-www-form-urlencoded or + multipart/form-data) to be ignored while searching for a saved flow + to replay. """ ) self.add_option( "server_replay_ignore_host", False, bool, - "Ignore request's destination host while searching for a saved" - " flow to replay" + """ + Ignore request's destination host while searching for a saved flow + to replay. + """ ) # Proxy options @@ -196,8 +211,8 @@ class Options(optmanager.OptManager): self.add_option( "auth_singleuser", None, Optional[str], """ - Allows access to a a single user, specified in the form - username:password. + Allows access to a a single user, specified in the form + username:password. """ ) self.add_option( @@ -206,13 +221,17 @@ class Options(optmanager.OptManager): ) self.add_option( "add_upstream_certs_to_client_chain", False, bool, - "Add all certificates of the upstream server to the certificate chain " - "that will be served to the proxy client, as extras." + """ + Add all certificates of the upstream server to the certificate chain + that will be served to the proxy client, as extras. + """ ) self.add_option( "body_size_limit", None, Optional[str], - "Byte size limit of HTTP request and response bodies." - " Understands k/m/g suffixes, i.e. 3m for 3 megabytes." + """ + Byte size limit of HTTP request and response bodies. Understands + k/m/g suffixes, i.e. 3m for 3 megabytes. + """ ) self.add_option( "cadir", CA_DIR, str, @@ -221,7 +240,7 @@ class Options(optmanager.OptManager): self.add_option( "certs", [], Sequence[str], """ - Add an SSL certificate. SPEC is of the form "[domain=]path". The + SSL certificates. SPEC is of the form "[domain=]path". The domain may include a wildcard, and is equal to "*" if not specified. The file at path is a certificate in PEM format. If a private key is included in the PEM, it is used, else the default key in the conf @@ -245,11 +264,11 @@ class Options(optmanager.OptManager): self.add_option( "ignore_hosts", [], Sequence[str], """ - Ignore host and forward all traffic without processing it. In - transparent mode, it is recommended to use an IP address (range), - not the hostname. In regular mode, only SSL traffic is ignored and - the hostname should be used. The supplied value is interpreted as a - regular expression and matched on the ip or the hostname. + Ignore host and forward all traffic without processing it. In + transparent mode, it is recommended to use an IP address (range), + not the hostname. In regular mode, only SSL traffic is ignored and + the hostname should be used. The supplied value is interpreted as a + regular expression and matched on the ip or the hostname. """ ) self.add_option( @@ -267,9 +286,9 @@ class Options(optmanager.OptManager): self.add_option( "mode", "regular", str, """ - Mode can be "regular", "transparent", "socks5", "reverse:SPEC", - or "upstream:SPEC". For reverse and upstream proxy modes, SPEC - is proxy specification in the form of "http[s]://host[:port]". + Mode can be "regular", "transparent", "socks5", "reverse:SPEC", + or "upstream:SPEC". For reverse and upstream proxy modes, SPEC + is proxy specification in the form of "http[s]://host[:port]". """ ) self.add_option( @@ -278,8 +297,10 @@ class Options(optmanager.OptManager): ) self.add_option( "keep_host_header", False, bool, - "Reverse Proxy: Keep the original host header instead of rewriting it" - " to the reverse proxy target." + """ + Reverse Proxy: Keep the original host header instead of rewriting it + to the reverse proxy target. + """ ) self.add_option( @@ -289,9 +310,11 @@ class Options(optmanager.OptManager): ) self.add_option( "http2_priority", False, bool, - "Enable/disable PRIORITY forwarding for HTTP/2 connections. " - "PRIORITY forwarding is disabled by default, " - "because some webservers fail to implement the RFC properly.", + """ + PRIORITY forwarding for HTTP/2 connections. PRIORITY forwarding is + disabled by default, because some webservers fail to implement the + RFC properly. + """ ) self.add_option( "websocket", True, bool, @@ -306,28 +329,32 @@ class Options(optmanager.OptManager): self.add_option( "spoof_source_address", False, bool, - "Use the client's IP for server-side connections. " - "Combine with --upstream-bind-address to spoof a fixed source address." + """ + Use the client's IP for server-side connections. Combine with + --upstream-bind-address to spoof a fixed source address. + """ ) self.add_option( "upstream_auth", None, Optional[str], """ - Add HTTP Basic authentcation to upstream proxy and reverse proxy - requests. Format: username:password + Add HTTP Basic authentcation to upstream proxy and reverse proxy + requests. Format: username:password """ ) self.add_option( "ssl_version_client", "secure", str, - "Set supported SSL/TLS versions for client connections. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which " - "is TLS1.0+.", + """ + Set supported SSL/TLS versions for client connections. SSLv2, SSLv3 + and 'all' are INSECURE. Defaults to secure, which is TLS1.0+. + """, choices=tcp.sslversion_choices.keys(), ) self.add_option( "ssl_version_server", "secure", str, - "Set supported SSL/TLS versions for server connections. " - "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, " - "which is TLS1.0+.", + """ + Set supported SSL/TLS versions for server connections. SSLv2, SSLv3 + and 'all' are INSECURE. Defaults to secure, which is TLS1.0+. + """, choices=tcp.sslversion_choices.keys(), ) self.add_option( @@ -336,8 +363,10 @@ class Options(optmanager.OptManager): ) self.add_option( "ssl_verify_upstream_trusted_cadir", None, Optional[str], - "Path to a directory of trusted CA certificates for upstream " - "server verification prepared using the c_rehash tool." + """ + Path to a directory of trusted CA certificates for upstream server + verification prepared using the c_rehash tool. + """ ) self.add_option( "ssl_verify_upstream_trusted_ca", None, Optional[str], @@ -346,9 +375,9 @@ class Options(optmanager.OptManager): self.add_option( "tcp_hosts", [], Sequence[str], """ - Generic TCP SSL proxy mode for all hosts that match the pattern. - Similar to --ignore, but SSL connections are intercepted. The - communication contents are printed to the log in verbose mode. + Generic TCP SSL proxy mode for all hosts that match the pattern. + Similar to --ignore, but SSL connections are intercepted. The + communication contents are printed to the log in verbose mode. """ ) @@ -368,7 +397,7 @@ class Options(optmanager.OptManager): ) self.add_option( "console_palette", "dark", Optional[str], - help="Select color palette: " + ", ".join(console_palettes), + "Color palette.", choices=sorted(console_palettes), ) self.add_option( @@ -384,7 +413,10 @@ class Options(optmanager.OptManager): "Flow sort order.", choices=view_orders, ) - self.add_option("console_order_reversed", False, bool) + self.add_option( + "console_order_reversed", False, bool, + "Reverse the sorting order." + ) self.add_option( "filter", None, Optional[str], @@ -410,7 +442,10 @@ class Options(optmanager.OptManager): ) # Dump options - self.add_option("filtstr", None, Optional[str]) + self.add_option( + "filtstr", None, Optional[str], + "The filter string for mitmdump" + ) self.add_option( "flow_detail", 1, int, "Flow detail display level" diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index e5277371..f03055eb 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -28,7 +28,7 @@ class _Option: name: str, default: typing.Any, typespec: typing.Type, - help: typing.Optional[str], + help: str, choices: typing.Optional[typing.Sequence[str]] ) -> None: typecheck.check_type(name, default, typespec) @@ -103,7 +103,7 @@ class OptManager: name: str, default: typing.Any, typespec: typing.Type, - help: typing.Optional[str] = None, + help: str, choices: typing.Optional[typing.Sequence[str]] = None ) -> None: if name in self._options: @@ -404,7 +404,7 @@ class OptManager: action="append", type=str, dest=optname, - help=o.help, + help=o.help + " May be passed multiple times.", metavar=metavar, choices=o.choices, ) @@ -421,11 +421,10 @@ def dump(opts): for k in sorted(opts.keys()): o = opts._options[k] s[k] = o.default - if o.help: - s.yaml_set_comment_before_after_key( - k, - before = "\n" + "\n".join(textwrap.wrap( - textwrap.dedent(o.help.strip()) - )), - ) + s.yaml_set_comment_before_after_key( + k, + before = "\n" + "\n".join(textwrap.wrap( + textwrap.dedent(o.help.strip()) + )), + ) return ruamel.yaml.round_trip_dump(s) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index d4f3aa69..bb33c1e4 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -91,26 +91,12 @@ def common_options(parser, opts): opts.make_parser(group, "server_replay_nopop") # Replacements - group = parser.add_argument_group( - "Replacements", - """ - Replacements are of the form "/pattern/regex/replacement", where - the separator can be any character. Please see the documentation - for more information. - """.strip() - ) + group = parser.add_argument_group("Replacements") opts.make_parser(group, "replacements", metavar="PATTERN") opts.make_parser(group, "replacement_files", metavar="PATTERN") # Set headers - group = parser.add_argument_group( - "Set Headers", - """ - Header specifications are of the form "/pattern/header/value", - where the separator can be any character. Please see the - documentation for more information. - """.strip() - ) + group = parser.add_argument_group("Set Headers") opts.make_parser(group, "setheaders", metavar="PATTERN") # Proxy authentication @@ -149,8 +135,8 @@ def mitmdump(opts): 'filter_args', nargs="...", help=""" - Filter view expression, used to only show flows that match a certain filter. - See help in mitmproxy for filter expression syntax. + Filter view expression, used to only show flows that match a certain + filter. See help in mitmproxy for filter expression syntax. """ ) return parser diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 9311e82d..d6ce87e6 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -13,36 +13,36 @@ from mitmproxy.test import tutils class TO(optmanager.OptManager): def __init__(self): super().__init__() - self.add_option("one", None, typing.Optional[int]) - self.add_option("two", 2, typing.Optional[int]) - self.add_option("bool", False, bool) + self.add_option("one", None, typing.Optional[int], "help") + self.add_option("two", 2, typing.Optional[int], "help") + self.add_option("bool", False, bool, "help") class TD(optmanager.OptManager): def __init__(self): super().__init__() - self.add_option("one", "done", str) - self.add_option("two", "dtwo", str) + self.add_option("one", "done", str, "help") + self.add_option("two", "dtwo", str, "help") class TD2(TD): def __init__(self): super().__init__() - self.add_option("three", "dthree", str) - self.add_option("four", "dfour", str) + self.add_option("three", "dthree", str, "help") + self.add_option("four", "dfour", str, "help") class TM(optmanager.OptManager): def __init__(self): super().__init__() - self.add_option("two", ["foo"], typing.Sequence[str]) - self.add_option("one", None, typing.Optional[str]) + self.add_option("two", ["foo"], typing.Sequence[str], "help") + self.add_option("one", None, typing.Optional[str], "help") def test_add_option(): o = TO() with pytest.raises(ValueError, match="already exists"): - o.add_option("one", None, typing.Optional[int]) + o.add_option("one", None, typing.Optional[int], "help") def test_defaults(): @@ -291,13 +291,13 @@ def test_dump(): class TTypes(optmanager.OptManager): def __init__(self): super().__init__() - self.add_option("str", "str", str) - self.add_option("optstr", "optstr", typing.Optional[str]) - self.add_option("bool", False, bool) - self.add_option("int", 0, int) - self.add_option("optint", 0, typing.Optional[int]) - self.add_option("seqstr", [], typing.Sequence[str]) - self.add_option("unknown", 0.0, float) + self.add_option("str", "str", str, "help") + self.add_option("optstr", "optstr", typing.Optional[str], "help", "help") + self.add_option("bool", False, bool, "help") + self.add_option("int", 0, int, "help") + self.add_option("optint", 0, typing.Optional[int], "help") + self.add_option("seqstr", [], typing.Sequence[str], "help") + self.add_option("unknown", 0.0, float, "help") def test_make_parser(): -- cgit v1.2.3 From b0ba76559869053e49b9c11b826a542b4885b49d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 7 Mar 2017 19:29:08 +1300 Subject: Add type and choices to options dump commets. --- mitmproxy/options.py | 26 ++++++++++++-------------- mitmproxy/optmanager.py | 29 ++++++++++++++++++++++------- 2 files changed, 34 insertions(+), 21 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 21ee257d..cee80158 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -56,9 +56,7 @@ class Options(optmanager.OptManager): "onboarding_host", APP_HOST, str, """ Domain to serve the onboarding app from. For transparent mode, use - an IP when a DNS entry for the app domain is not present. Default: - %s - """ % APP_HOST + an IP when a DNS entry for the app domain is not present. """ ) self.add_option( "onboarding_port", APP_PORT, int, @@ -235,7 +233,7 @@ class Options(optmanager.OptManager): ) self.add_option( "cadir", CA_DIR, str, - "Location of the default mitmproxy CA files. (%s)" % CA_DIR + "Location of the default mitmproxy CA files." ) self.add_option( "certs", [], Sequence[str], @@ -251,11 +249,11 @@ class Options(optmanager.OptManager): ) self.add_option( "ciphers_client", DEFAULT_CLIENT_CIPHERS, str, - "Set supported ciphers for client connections. (OpenSSL Syntax)" + "Set supported ciphers for client connections using OpenSSL syntax." ) self.add_option( "ciphers_server", None, Optional[str], - "Set supported ciphers for server connections. (OpenSSL Syntax)" + "Set supported ciphers for server connections using OpenSSL syntax." ) self.add_option( "client_certs", None, Optional[str], @@ -273,7 +271,7 @@ class Options(optmanager.OptManager): ) self.add_option( "listen_host", "", str, - "Address to bind proxy to (defaults to all interfaces)" + "Address to bind proxy to." ) self.add_option( "listen_port", LISTEN_PORT, int, @@ -281,7 +279,7 @@ class Options(optmanager.OptManager): ) self.add_option( "upstream_bind_address", "", str, - "Address to bind upstream requests to (defaults to none)" + "Address to bind upstream requests to." ) self.add_option( "mode", "regular", str, @@ -338,7 +336,7 @@ class Options(optmanager.OptManager): "upstream_auth", None, Optional[str], """ Add HTTP Basic authentcation to upstream proxy and reverse proxy - requests. Format: username:password + requests. Format: username:password. """ ) self.add_option( @@ -396,7 +394,7 @@ class Options(optmanager.OptManager): "Focus follows new flows." ) self.add_option( - "console_palette", "dark", Optional[str], + "console_palette", "dark", str, "Color palette.", choices=sorted(console_palettes), ) @@ -426,11 +424,11 @@ class Options(optmanager.OptManager): # Web options self.add_option( "web_open_browser", True, bool, - "Start a browser" + "Start a browser." ) self.add_option( "web_debug", False, bool, - "Mitmweb debugging" + "Mitmweb debugging." ) self.add_option( "web_port", 8081, int, @@ -444,11 +442,11 @@ class Options(optmanager.OptManager): # Dump options self.add_option( "filtstr", None, Optional[str], - "The filter string for mitmdump" + "The filter string for mitmdump." ) self.add_option( "flow_detail", 1, int, - "Flow detail display level" + "Flow detail display level." ) self.update(**kwargs) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index f03055eb..8dddd11a 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -359,8 +359,8 @@ class OptManager: optname, getattr(self, optname) + [optval] ) - else: - raise ValueError("Unsupported option type: %s", o.typespec) + else: # pragma: no cover + raise NotImplementedError("Unsupported option type: %s", o.typespec) def make_parser(self, parser, optname, metavar=None): o = self._options[optname] @@ -421,10 +421,25 @@ def dump(opts): for k in sorted(opts.keys()): o = opts._options[k] s[k] = o.default - s.yaml_set_comment_before_after_key( - k, - before = "\n" + "\n".join(textwrap.wrap( - textwrap.dedent(o.help.strip()) - )), + txt = o.help.strip() + + if o.choices: + txt += " Valid values are %s." % ", ".join(repr(c) for c in o.choices) + else: + if o.typespec in (str, int, bool): + t = o.typespec.__name__ + elif o.typespec == typing.Optional[str]: + t = "optional str" + elif o.typespec == typing.Sequence[str]: + t = "sequence of str" + else: # pragma: no cover + raise NotImplementedError + txt += " Type %s." % t + + txt = "\n".join( + textwrap.wrap( + textwrap.dedent(txt) + ) ) + s.yaml_set_comment_before_after_key(k, before = "\n" + txt) return ruamel.yaml.round_trip_dump(s) -- cgit v1.2.3 From aed780bf48f6231e63a2b193318bc234152e2743 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 7 Mar 2017 20:18:30 +1300 Subject: Change the way proxy authetication is specified We now have one option "proxyauth". If this is "any", we accept any credentials, if it starts with an @ it's treated as a path to an htpasswd file, if it is of the form username:password it's a single-user credential. --- mitmproxy/addons/proxyauth.py | 46 ++++++++++++++++----------------- mitmproxy/options.py | 17 +++++------- mitmproxy/tools/cmdline.py | 4 +-- test/mitmproxy/addons/test_proxyauth.py | 38 ++++++++++++++------------- test/mitmproxy/proxy/test_server.py | 4 +-- 5 files changed, 51 insertions(+), 58 deletions(-) diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py index 18a85866..61477658 100644 --- a/mitmproxy/addons/proxyauth.py +++ b/mitmproxy/addons/proxyauth.py @@ -114,30 +114,28 @@ class ProxyAuth: # Handlers def configure(self, options, updated): - if "auth_nonanonymous" in updated: - self.nonanonymous = options.auth_nonanonymous - if "auth_singleuser" in updated: - if options.auth_singleuser: - parts = options.auth_singleuser.split(':') - if len(parts) != 2: - raise exceptions.OptionsError( - "Invalid single-user auth specification." - ) - self.singleuser = parts - else: - self.singleuser = None - if "auth_htpasswd" in updated: - if options.auth_htpasswd: - try: - self.htpasswd = passlib.apache.HtpasswdFile( - options.auth_htpasswd - ) - except (ValueError, OSError) as v: - raise exceptions.OptionsError( - "Could not open htpasswd file: %s" % v - ) - else: - self.htpasswd = None + if "proxyauth" in updated: + self.nonanonymous = False + self.singleuser = None + self.htpasswd = None + if options.proxyauth: + if options.proxyauth == "any": + self.nonanonymous = True + elif options.proxyauth.startswith("@"): + p = options.proxyauth[1:] + try: + self.htpasswd = passlib.apache.HtpasswdFile(p) + except (ValueError, OSError) as v: + raise exceptions.OptionsError( + "Could not open htpasswd file: %s" % p + ) + else: + parts = options.proxyauth.split(':') + if len(parts) != 2: + raise exceptions.OptionsError( + "Invalid single-user auth specification." + ) + self.singleuser = parts if "mode" in updated: self.mode = options.mode if self.enabled(): diff --git a/mitmproxy/options.py b/mitmproxy/options.py index cee80158..a754325a 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -203,20 +203,15 @@ class Options(optmanager.OptManager): # Proxy options self.add_option( - "auth_nonanonymous", False, bool, - "Allow access to any user long as a credentials are specified." - ) - self.add_option( - "auth_singleuser", None, Optional[str], + "proxyauth", None, Optional[str], """ - Allows access to a a single user, specified in the form - username:password. + Require authentication before proxying requests. If the value is + "any", we prompt for authentication, but permit any values. If it + starts with an "@", it is treated as a path to an Apache htpasswd + file. If its is of the form "username:password", it is treated as a + single-user credential. """ ) - self.add_option( - "auth_htpasswd", None, Optional[str], - "Allow access to users specified in an Apache htpasswd file." - ) self.add_option( "add_upstream_certs_to_client_chain", False, bool, """ diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index bb33c1e4..fcf9ecaf 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -107,9 +107,7 @@ def common_options(parser, opts): used for authenticating them. """ ).add_mutually_exclusive_group() - opts.make_parser(group, "auth_nonanonymous") - opts.make_parser(group, "auth_singleuser", metavar="USER:PASS") - opts.make_parser(group, "auth_htpasswd", metavar="PATH") + opts.make_parser(group, "proxyauth", metavar="SPEC") def mitmproxy(opts): diff --git a/test/mitmproxy/addons/test_proxyauth.py b/test/mitmproxy/addons/test_proxyauth.py index dd5829ab..14782755 100644 --- a/test/mitmproxy/addons/test_proxyauth.py +++ b/test/mitmproxy/addons/test_proxyauth.py @@ -28,40 +28,43 @@ def test_configure(): up = proxyauth.ProxyAuth() with taddons.context() as ctx: with pytest.raises(exceptions.OptionsError): - ctx.configure(up, auth_singleuser="foo") + ctx.configure(up, proxyauth="foo") - ctx.configure(up, auth_singleuser="foo:bar") + ctx.configure(up, proxyauth="foo:bar") assert up.singleuser == ["foo", "bar"] - ctx.configure(up, auth_singleuser=None) + ctx.configure(up, proxyauth=None) assert up.singleuser is None - ctx.configure(up, auth_nonanonymous=True) + ctx.configure(up, proxyauth="any") assert up.nonanonymous - ctx.configure(up, auth_nonanonymous=False) + ctx.configure(up, proxyauth=None) assert not up.nonanonymous with pytest.raises(exceptions.OptionsError): - ctx.configure(up, auth_htpasswd=tutils.test_data.path("mitmproxy/net/data/server.crt")) + ctx.configure( + up, + proxyauth= "@" + tutils.test_data.path("mitmproxy/net/data/server.crt") + ) with pytest.raises(exceptions.OptionsError): - ctx.configure(up, auth_htpasswd="nonexistent") + ctx.configure(up, proxyauth="@nonexistent") ctx.configure( up, - auth_htpasswd=tutils.test_data.path( + proxyauth= "@" + tutils.test_data.path( "mitmproxy/net/data/htpasswd" ) ) assert up.htpasswd assert up.htpasswd.check_password("test", "test") assert not up.htpasswd.check_password("test", "foo") - ctx.configure(up, auth_htpasswd=None) + ctx.configure(up, proxyauth=None) assert not up.htpasswd with pytest.raises(exceptions.OptionsError): - ctx.configure(up, auth_nonanonymous=True, mode="transparent") + ctx.configure(up, proxyauth="any", mode="transparent") with pytest.raises(exceptions.OptionsError): - ctx.configure(up, auth_nonanonymous=True, mode="socks5") + ctx.configure(up, proxyauth="any", mode="socks5") ctx.configure(up, mode="regular") assert up.mode == "regular" @@ -70,7 +73,7 @@ def test_configure(): def test_check(): up = proxyauth.ProxyAuth() with taddons.context() as ctx: - ctx.configure(up, auth_nonanonymous=True, mode="regular") + ctx.configure(up, proxyauth="any", mode="regular") f = tflow.tflow() assert not up.check(f) f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( @@ -86,18 +89,17 @@ def test_check(): ) assert not up.check(f) - ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:test") + ctx.configure(up, proxyauth="test:test") f.request.headers["Proxy-Authorization"] = proxyauth.mkauth( "test", "test" ) assert up.check(f) - ctx.configure(up, auth_nonanonymous=False, auth_singleuser="test:foo") + ctx.configure(up, proxyauth="test:foo") assert not up.check(f) ctx.configure( up, - auth_singleuser=None, - auth_htpasswd=tutils.test_data.path( + proxyauth="@" + tutils.test_data.path( "mitmproxy/net/data/htpasswd" ) ) @@ -114,7 +116,7 @@ def test_check(): def test_authenticate(): up = proxyauth.ProxyAuth() with taddons.context() as ctx: - ctx.configure(up, auth_nonanonymous=True, mode="regular") + ctx.configure(up, proxyauth="any", mode="regular") f = tflow.tflow() assert not f.response @@ -147,7 +149,7 @@ def test_authenticate(): def test_handlers(): up = proxyauth.ProxyAuth() with taddons.context() as ctx: - ctx.configure(up, auth_nonanonymous=True, mode="regular") + ctx.configure(up, proxyauth="any", mode="regular") f = tflow.tflow() assert not f.response diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index b90840ab..aa45761a 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -302,7 +302,7 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin): class TestHTTPAuth(tservers.HTTPProxyTest): def test_auth(self): self.master.addons.add(proxyauth.ProxyAuth()) - self.master.options.auth_singleuser = "test:test" + self.master.options.proxyauth = "test:test" assert self.pathod("202").status_code == 407 p = self.pathoc() with p.connect(): @@ -321,7 +321,7 @@ class TestHTTPAuth(tservers.HTTPProxyTest): class TestHTTPReverseAuth(tservers.ReverseProxyTest): def test_auth(self): self.master.addons.add(proxyauth.ProxyAuth()) - self.master.options.auth_singleuser = "test:test" + self.master.options.proxyauth = "test:test" assert self.pathod("202").status_code == 401 p = self.pathoc() with p.connect(): -- cgit v1.2.3 From 98ec3b77fe8d3c0bcb0761980568ddd067380bd1 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Tue, 7 Mar 2017 20:57:54 +1300 Subject: Carefully re-add a small number of short flags The mechanism for booleans attaches the short flag to whatever the opposite of the default is. --- mitmproxy/options.py | 4 ++-- mitmproxy/optmanager.py | 31 ++++++++++++++++++++++++------- mitmproxy/tools/cmdline.py | 34 ++++++++++++---------------------- mitmproxy/tools/dump.py | 2 +- mitmproxy/tools/main.py | 6 +++--- test/mitmproxy/test_optmanager.py | 10 ++++++---- 6 files changed, 48 insertions(+), 39 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index a754325a..08d97cad 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -86,8 +86,8 @@ class Options(optmanager.OptManager): "Continue serving after client playback or file read." ) self.add_option( - "no_server", False, bool, - "Don't start a proxy server." + "server", True, bool, + "Start a proxy server." ) self.add_option( "server_replay_nopop", False, bool, diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 8dddd11a..b05e78eb 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -362,18 +362,35 @@ class OptManager: else: # pragma: no cover raise NotImplementedError("Unsupported option type: %s", o.typespec) - def make_parser(self, parser, optname, metavar=None): + def make_parser(self, parser, optname, metavar=None, short=None): o = self._options[optname] - f = optname.replace("_", "-") + + def mkf(l, s): + l = l.replace("_", "-") + f = ["--%s" % l] + if s: + f.append("-" + s) + return f + + flags = mkf(optname, short) + if o.typespec == bool: g = parser.add_mutually_exclusive_group(required=False) + onf = mkf(optname, None) + offf = mkf("no-" + optname, None) + # The short option for a bool goes to whatever is NOT the default + if short: + if o.default: + offf = mkf("no-" + optname, short) + else: + onf = mkf(optname, short) g.add_argument( - "--no-%s" % f, + *offf, action="store_false", dest=optname, ) g.add_argument( - "--%s" % f, + *onf, action="store_true", dest=optname, help=o.help @@ -381,7 +398,7 @@ class OptManager: parser.set_defaults(**{optname: None}) elif o.typespec in (int, typing.Optional[int]): parser.add_argument( - "--%s" % f, + *flags, action="store", type=int, dest=optname, @@ -390,7 +407,7 @@ class OptManager: ) elif o.typespec in (str, typing.Optional[str]): parser.add_argument( - "--%s" % f, + *flags, action="store", type=str, dest=optname, @@ -400,7 +417,7 @@ class OptManager: ) elif o.typespec == typing.Sequence[str]: parser.add_argument( - "--%s" % f, + *flags, action="append", type=str, dest=optname, diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index fcf9ecaf..aaefd10a 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -42,7 +42,6 @@ def common_options(parser, opts): are emptied. """ ) - parser.add_argument( "-q", "--quiet", action="store_true", dest="quiet", @@ -55,59 +54,50 @@ def common_options(parser, opts): ) # Basic options - opts.make_parser(parser, "mode") + opts.make_parser(parser, "mode", short="m") opts.make_parser(parser, "anticache") opts.make_parser(parser, "showhost") - opts.make_parser(parser, "rfile") - opts.make_parser(parser, "scripts", metavar="SCRIPT") + opts.make_parser(parser, "rfile", metavar="PATH", short="r") + opts.make_parser(parser, "scripts", metavar="SCRIPT", short="s") opts.make_parser(parser, "stickycookie", metavar="FILTER") opts.make_parser(parser, "stickyauth", metavar="FILTER") - opts.make_parser(parser, "streamfile") + opts.make_parser(parser, "streamfile", metavar="PATH", short="w") opts.make_parser(parser, "anticomp") # Proxy options group = parser.add_argument_group("Proxy Options") opts.make_parser(group, "listen_host", metavar="HOST") + opts.make_parser(group, "listen_port", metavar="PORT", short="p") + opts.make_parser(group, "server", short="n") opts.make_parser(group, "ignore_hosts", metavar="HOST") opts.make_parser(group, "tcp_hosts", metavar="HOST") - opts.make_parser(group, "no_server") - opts.make_parser(group, "listen_port", metavar="PORT") opts.make_parser(group, "upstream_auth", metavar="USER:PASS") + opts.make_parser(group, "proxyauth", metavar="SPEC") opts.make_parser(group, "rawtcp") # Proxy SSL options group = parser.add_argument_group("SSL") opts.make_parser(group, "certs", metavar="SPEC") - opts.make_parser(group, "ssl_insecure") + opts.make_parser(group, "ssl_insecure", short="k") # Client replay group = parser.add_argument_group("Client Replay") - opts.make_parser(group, "client_replay", metavar="PATH") + opts.make_parser(group, "client_replay", metavar="PATH", short="C") # Server replay group = parser.add_argument_group("Server Replay") - opts.make_parser(group, "server_replay", metavar="PATH") + opts.make_parser(group, "server_replay", metavar="PATH", short="S") opts.make_parser(group, "replay_kill_extra") opts.make_parser(group, "server_replay_nopop") # Replacements group = parser.add_argument_group("Replacements") - opts.make_parser(group, "replacements", metavar="PATTERN") + opts.make_parser(group, "replacements", metavar="PATTERN", short="R") opts.make_parser(group, "replacement_files", metavar="PATTERN") # Set headers group = parser.add_argument_group("Set Headers") - opts.make_parser(group, "setheaders", metavar="PATTERN") - - # Proxy authentication - group = parser.add_argument_group( - "Proxy Authentication", - """ - Specify which users are allowed to access the proxy and the method - used for authenticating them. - """ - ).add_mutually_exclusive_group() - opts.make_parser(group, "proxyauth", metavar="SPEC") + opts.make_parser(group, "setheaders", metavar="PATTERN", short="H") def mitmproxy(opts): diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index f64f2241..e70ce2f9 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -23,7 +23,7 @@ class DumpMaster(master.Master): if with_dumper: self.addons.add(dumper.Dumper()) - if not self.options.no_server: + if self.options.server: self.add_log( "Proxy server listening at http://{}:{}".format(server.address[0], server.address[1]), "info" diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 7c1e7b32..17c1abbb 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -55,14 +55,14 @@ def process_options(parser, opts, args): opts.merge(adict) pconf = config.ProxyConfig(opts) - if opts.no_server: - return server.DummyServer(pconf) - else: + if opts.server: try: return server.ProxyServer(pconf) except exceptions.ServerException as v: print(str(v), file=sys.stderr) sys.exit(1) + else: + return server.DummyServer(pconf) def run(MasterKlass, args): # pragma: no cover diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index d6ce87e6..ba31c750 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -294,6 +294,7 @@ class TTypes(optmanager.OptManager): self.add_option("str", "str", str, "help") self.add_option("optstr", "optstr", typing.Optional[str], "help", "help") self.add_option("bool", False, bool, "help") + self.add_option("bool_on", True, bool, "help") self.add_option("int", 0, int, "help") self.add_option("optint", 0, typing.Optional[int], "help") self.add_option("seqstr", [], typing.Sequence[str], "help") @@ -303,10 +304,11 @@ class TTypes(optmanager.OptManager): def test_make_parser(): parser = argparse.ArgumentParser() opts = TTypes() - opts.make_parser(parser, "str") - opts.make_parser(parser, "bool") - opts.make_parser(parser, "int") - opts.make_parser(parser, "seqstr") + opts.make_parser(parser, "str", short="a") + opts.make_parser(parser, "bool", short="b") + opts.make_parser(parser, "int", short="c") + opts.make_parser(parser, "seqstr", short="d") + opts.make_parser(parser, "bool_on", short="e") with pytest.raises(ValueError): opts.make_parser(parser, "unknown") -- cgit v1.2.3 From 8707928b16c3904249309b6c81244359860cf897 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 8 Mar 2017 00:16:49 +0100 Subject: unify server spec parsing --- mitmproxy/addons/core.py | 11 +++-- mitmproxy/net/check.py | 2 +- mitmproxy/net/server_spec.py | 76 ++++++++++++++++++++++++++++++++++ mitmproxy/options.py | 5 --- mitmproxy/proxy/config.py | 25 ++--------- test/mitmproxy/net/test_server_spec.py | 32 ++++++++++++++ test/mitmproxy/proxy/test_config.py | 21 +--------- 7 files changed, 118 insertions(+), 54 deletions(-) create mode 100644 mitmproxy/net/server_spec.py create mode 100644 test/mitmproxy/net/test_server_spec.py diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index 3de1638c..4c6a5516 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -3,8 +3,8 @@ checked by other addons. """ from mitmproxy import exceptions -from mitmproxy import options from mitmproxy import platform +from mitmproxy.net import server_spec from mitmproxy.utils import human @@ -30,11 +30,10 @@ class Core: if "mode" in updated: mode = opts.mode if mode.startswith("reverse:") or mode.startswith("upstream:"): - spec = options.get_mode_spec(mode) - if not spec: - raise exceptions.OptionsError( - "Invalid mode specification: %s" % mode - ) + try: + server_spec.parse_with_mode(mode) + except ValueError as e: + raise exceptions.OptionsError(str(e)) from e elif mode == "transparent": if not platform.original_addr: raise exceptions.OptionsError( diff --git a/mitmproxy/net/check.py b/mitmproxy/net/check.py index d30c1df6..aaea851f 100644 --- a/mitmproxy/net/check.py +++ b/mitmproxy/net/check.py @@ -29,5 +29,5 @@ def is_valid_host(host: bytes) -> bool: return False -def is_valid_port(port): +def is_valid_port(port: int) -> bool: return 0 <= port <= 65535 diff --git a/mitmproxy/net/server_spec.py b/mitmproxy/net/server_spec.py new file mode 100644 index 00000000..efbf1012 --- /dev/null +++ b/mitmproxy/net/server_spec.py @@ -0,0 +1,76 @@ +""" +Parse scheme, host and port from a string. +""" +import collections +import re +from typing import Tuple + +from mitmproxy.net import check + +ServerSpec = collections.namedtuple("ServerSpec", ["scheme", "address"]) + +server_spec_re = re.compile( + r""" + ^ + (?:(?P\w+)://)? # scheme is optional + (?P[^:/]+|\[.+\]) # hostname can be DNS name, IPv4, or IPv6 address. + (?::(?P\d+))? # port is optional + /? # we allow a trailing backslash, but no path + $ + """, + re.VERBOSE +) + + +def parse(server_spec: str) -> ServerSpec: + """ + Parses a server mode specification, e.g.: + + - http://example.com/ + - example.org + - example.com:443 + + Raises: + ValueError, if the server specification is invalid. + """ + m = server_spec_re.match(server_spec) + if not m: + raise ValueError("Invalid server specification: {}".format(server_spec)) + + # defaulting to https/port 443 may annoy some folks, but it's secure-by-default. + scheme = m.group("scheme") or "https" + if scheme not in ("http", "https"): + raise ValueError("Invalid server scheme: {}".format(scheme)) + + host = m.group("host") + # IPv6 brackets + if host.startswith("[") and host.endswith("]"): + host = host[1:-1] + if not check.is_valid_host(host.encode("idna")): + raise ValueError("Invalid hostname: {}".format(host)) + + if m.group("port"): + port = int(m.group("port")) + else: + port = { + "http": 80, + "https": 443 + }[scheme] + if not check.is_valid_port(port): + raise ValueError("Invalid port: {}".format(port)) + + return ServerSpec(scheme, (host, port)) + + +def parse_with_mode(mode: str) -> Tuple[str, ServerSpec]: + """ + Parse a proxy mode specification, which is usually just (reverse|upstream):server-spec + + Returns: + A (mode, server_spec) tuple. + + Raises: + ValueError, if the specification is invalid. + """ + mode, server_spec = mode.split(":", maxsplit=1) + return mode, parse(server_spec) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 08d97cad..1063bab9 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -22,11 +22,6 @@ view_orders = [ "size", ] - -def get_mode_spec(m): - return m.split(":", maxsplit=1)[1] - - APP_HOST = "mitm.it" APP_PORT = 80 CA_DIR = "~/.mitmproxy" diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 9cf2b00f..8417ebad 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -1,4 +1,3 @@ -import collections import os import re from typing import Any @@ -9,7 +8,7 @@ from mitmproxy import exceptions from mitmproxy import options as moptions from mitmproxy import certs from mitmproxy.net import tcp -from mitmproxy.net.http import url +from mitmproxy.net import server_spec CONF_BASENAME = "mitmproxy" @@ -33,24 +32,6 @@ class HostMatcher: return bool(self.patterns) -ServerSpec = collections.namedtuple("ServerSpec", "scheme address") - - -def parse_server_spec(spec): - try: - p = url.parse(spec) - if p[0] not in (b"http", b"https"): - raise ValueError() - except ValueError: - raise exceptions.OptionsError( - "Invalid server specification: %s" % spec - ) - host, port = p[1:3] - address = (host.decode("ascii"), port) - scheme = p[0].decode("ascii").lower() - return ServerSpec(scheme, address) - - class ProxyConfig: def __init__(self, options: moptions.Options) -> None: @@ -123,5 +104,5 @@ class ProxyConfig: ) m = options.mode if m.startswith("upstream:") or m.startswith("reverse:"): - spec = moptions.get_mode_spec(options.mode) - self.upstream_server = parse_server_spec(spec) + _, spec = server_spec.parse_with_mode(options.mode) + self.upstream_server = spec diff --git a/test/mitmproxy/net/test_server_spec.py b/test/mitmproxy/net/test_server_spec.py new file mode 100644 index 00000000..095ad519 --- /dev/null +++ b/test/mitmproxy/net/test_server_spec.py @@ -0,0 +1,32 @@ +import pytest + +from mitmproxy.net import server_spec + + +def test_parse(): + assert server_spec.parse("example.com") == ("https", ("example.com", 443)) + assert server_spec.parse("example.com") == ("https", ("example.com", 443)) + assert server_spec.parse("http://example.com") == ("http", ("example.com", 80)) + assert server_spec.parse("http://127.0.0.1") == ("http", ("127.0.0.1", 80)) + assert server_spec.parse("http://[::1]") == ("http", ("::1", 80)) + assert server_spec.parse("http://[::1]/") == ("http", ("::1", 80)) + assert server_spec.parse("https://[::1]/") == ("https", ("::1", 443)) + assert server_spec.parse("http://[::1]:8080") == ("http", ("::1", 8080)) + + with pytest.raises(ValueError, match="Invalid server specification"): + server_spec.parse(":") + + with pytest.raises(ValueError, match="Invalid server scheme"): + server_spec.parse("ftp://example.com") + + with pytest.raises(ValueError, match="Invalid hostname"): + server_spec.parse("$$$") + + with pytest.raises(ValueError, match="Invalid port"): + server_spec.parse("example.com:999999") + + +def test_parse_with_mode(): + assert server_spec.parse_with_mode("m:example.com") == ("m", ("https", ("example.com", 443))) + with pytest.raises(ValueError): + server_spec.parse_with_mode("moo") diff --git a/test/mitmproxy/proxy/test_config.py b/test/mitmproxy/proxy/test_config.py index 4272d952..777ab4dd 100644 --- a/test/mitmproxy/proxy/test_config.py +++ b/test/mitmproxy/proxy/test_config.py @@ -1,20 +1 @@ -import pytest -from mitmproxy.proxy import config - - -def test_parse_server_spec(): - with pytest.raises(Exception, match="Invalid server specification"): - config.parse_server_spec("") - assert config.parse_server_spec("http://foo.com:88") == ( - "http", ("foo.com", 88) - ) - assert config.parse_server_spec("http://foo.com") == ( - "http", ("foo.com", 80) - ) - assert config.parse_server_spec("https://foo.com") == ( - "https", ("foo.com", 443) - ) - with pytest.raises(Exception, match="Invalid server specification"): - config.parse_server_spec("foo.com") - with pytest.raises(Exception, match="Invalid server specification"): - config.parse_server_spec("http://") +# TODO: write tests -- cgit v1.2.3 From f276c7a80d63c0a464feca5f76096a23afdc69b2 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 8 Mar 2017 00:52:37 +0100 Subject: change add_option parameter order name, type, value matches natural language ($x is a bool with a default value of $y) and also matches the python type annotation order of name: type = value --- mitmproxy/options.py | 163 +++++++++++++++++++------------------ mitmproxy/optmanager.py | 10 +-- test/mitmproxy/addons/test_core.py | 2 +- test/mitmproxy/test_optmanager.py | 46 +++++------ 4 files changed, 111 insertions(+), 110 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 1063bab9..6dd8616b 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -1,8 +1,7 @@ from typing import Optional, Sequence -from mitmproxy.net import tcp from mitmproxy import optmanager - +from mitmproxy.net import tcp # We redefine these here for now to avoid importing Urwid-related guff on # platforms that don't support it, and circular imports. We can do better using @@ -29,128 +28,130 @@ LISTEN_PORT = 8080 # We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. # https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old -DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:" \ - "ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:" \ - "ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:" \ - "ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:" \ - "DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:" \ - "DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:" \ - "AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:" \ - "HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:" \ +DEFAULT_CLIENT_CIPHERS = ( + "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:" + "ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:" + "ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:" + "ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:" + "DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:" + "DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:" + "AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:" + "HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:" "!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA" +) class Options(optmanager.OptManager): def __init__(self, **kwargs) -> None: super().__init__() self.add_option( - "onboarding", True, bool, + "onboarding", bool, True, "Toggle the mitmproxy onboarding app." ) self.add_option( - "onboarding_host", APP_HOST, str, + "onboarding_host", str, APP_HOST, """ Domain to serve the onboarding app from. For transparent mode, use an IP when a DNS entry for the app domain is not present. """ ) self.add_option( - "onboarding_port", APP_PORT, int, + "onboarding_port", int, APP_PORT, "Port to serve the onboarding app from." ) self.add_option( - "anticache", False, bool, + "anticache", bool, False, """ Strip out request headers that might cause the server to return 304-not-modified. """ ) self.add_option( - "anticomp", False, bool, + "anticomp", bool, False, "Try to convince servers to send us un-compressed data." ) self.add_option( - "client_replay", [], Sequence[str], + "client_replay", Sequence[str], [], "Replay client requests from a saved file." ) self.add_option( - "replay_kill_extra", False, bool, + "replay_kill_extra", bool, False, "Kill extra requests during replay." ) self.add_option( - "keepserving", True, bool, + "keepserving", bool, True, "Continue serving after client playback or file read." ) self.add_option( - "server", True, bool, + "server", bool, True, "Start a proxy server." ) self.add_option( - "server_replay_nopop", False, bool, + "server_replay_nopop", bool, False, """ Disable response pop from response flow. This makes it possible to replay same response multiple times. """ ) self.add_option( - "refresh_server_playback", True, bool, + "refresh_server_playback", bool, True, """ Refresh server replay responses by adjusting date, expires and last-modified headers, as well as adjusting cookie expiration. """ ) self.add_option( - "rfile", None, Optional[str], + "rfile", Optional[str], None, "Read flows from file." ) self.add_option( - "scripts", [], Sequence[str], + "scripts", Sequence[str], [], """ Execute a script. """ ) self.add_option( - "showhost", False, bool, + "showhost", bool, False, "Use the Host header to construct URLs for display." ) self.add_option( - "replacements", [], Sequence[str], + "replacements", Sequence[str], [], """ Replacement patterns of the form "/pattern/regex/replacement", where the separator can be any character. """ ) self.add_option( - "replacement_files", [], Sequence[str], + "replacement_files", Sequence[str], [], """ Replacement pattern, where the replacement clause is a path to a file. """ ) self.add_option( - "server_replay_use_headers", [], Sequence[str], + "server_replay_use_headers", Sequence[str], [], "Request headers to be considered during replay." ) self.add_option( - "setheaders", [], Sequence[str], + "setheaders", Sequence[str], [], """ Header set pattern of the form "/pattern/header/value", where the separator can be any character. """ ) self.add_option( - "server_replay", [], Sequence[str], + "server_replay", Sequence[str], [], "Replay server responses from a saved file." ) self.add_option( - "stickycookie", None, Optional[str], + "stickycookie", Optional[str], None, "Set sticky cookie filter. Matched against requests." ) self.add_option( - "stickyauth", None, Optional[str], + "stickyauth", Optional[str], None, "Set sticky auth filter. Matched against requests." ) self.add_option( - "stream_large_bodies", None, Optional[str], + "stream_large_bodies", Optional[str], None, """ Stream data to the client if response body exceeds the given threshold. If streamed, the body will not be stored in any way. @@ -158,30 +159,30 @@ class Options(optmanager.OptManager): """ ) self.add_option( - "verbosity", 2, int, + "verbosity", int, 2, "Log verbosity." ) self.add_option( - "default_contentview", "auto", str, + "default_contentview", str, "auto", "The default content view mode." ) self.add_option( - "streamfile", None, Optional[str], + "streamfile", Optional[str], None, "Write flows to file. Prefix path with + to append." ) self.add_option( - "server_replay_ignore_content", False, bool, + "server_replay_ignore_content", bool, False, "Ignore request's content while searching for a saved flow to replay." ) self.add_option( - "server_replay_ignore_params", [], Sequence[str], + "server_replay_ignore_params", Sequence[str], [], """ Request's parameters to be ignored while searching for a saved flow to replay. Can be passed multiple times. """ ) self.add_option( - "server_replay_ignore_payload_params", [], Sequence[str], + "server_replay_ignore_payload_params", Sequence[str], [], """ Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to be ignored while searching for a saved flow @@ -189,7 +190,7 @@ class Options(optmanager.OptManager): """ ) self.add_option( - "server_replay_ignore_host", False, bool, + "server_replay_ignore_host", bool, False, """ Ignore request's destination host while searching for a saved flow to replay. @@ -198,7 +199,7 @@ class Options(optmanager.OptManager): # Proxy options self.add_option( - "proxyauth", None, Optional[str], + "proxyauth", Optional[str], None, """ Require authentication before proxying requests. If the value is "any", we prompt for authentication, but permit any values. If it @@ -208,25 +209,25 @@ class Options(optmanager.OptManager): """ ) self.add_option( - "add_upstream_certs_to_client_chain", False, bool, + "add_upstream_certs_to_client_chain", bool, False, """ Add all certificates of the upstream server to the certificate chain that will be served to the proxy client, as extras. """ ) self.add_option( - "body_size_limit", None, Optional[str], + "body_size_limit", Optional[str], None, """ Byte size limit of HTTP request and response bodies. Understands k/m/g suffixes, i.e. 3m for 3 megabytes. """ ) self.add_option( - "cadir", CA_DIR, str, + "cadir", str, CA_DIR, "Location of the default mitmproxy CA files." ) self.add_option( - "certs", [], Sequence[str], + "certs", Sequence[str], [], """ SSL certificates. SPEC is of the form "[domain=]path". The domain may include a wildcard, and is equal to "*" if not specified. @@ -238,19 +239,19 @@ class Options(optmanager.OptManager): """ ) self.add_option( - "ciphers_client", DEFAULT_CLIENT_CIPHERS, str, + "ciphers_client", str, DEFAULT_CLIENT_CIPHERS, "Set supported ciphers for client connections using OpenSSL syntax." ) self.add_option( - "ciphers_server", None, Optional[str], + "ciphers_server", Optional[str], None, "Set supported ciphers for server connections using OpenSSL syntax." ) self.add_option( - "client_certs", None, Optional[str], + "client_certs", Optional[str], None, "Client certificate file or directory." ) self.add_option( - "ignore_hosts", [], Sequence[str], + "ignore_hosts", Sequence[str], [], """ Ignore host and forward all traffic without processing it. In transparent mode, it is recommended to use an IP address (range), @@ -260,19 +261,19 @@ class Options(optmanager.OptManager): """ ) self.add_option( - "listen_host", "", str, + "listen_host", str, "", "Address to bind proxy to." ) self.add_option( - "listen_port", LISTEN_PORT, int, + "listen_port", int, LISTEN_PORT, "Proxy service port." ) self.add_option( - "upstream_bind_address", "", str, + "upstream_bind_address", str, "", "Address to bind upstream requests to." ) self.add_option( - "mode", "regular", str, + "mode", str, "regular", """ Mode can be "regular", "transparent", "socks5", "reverse:SPEC", or "upstream:SPEC". For reverse and upstream proxy modes, SPEC @@ -280,11 +281,11 @@ class Options(optmanager.OptManager): """ ) self.add_option( - "upstream_cert", True, bool, + "upstream_cert", bool, True, "Connect to upstream server to look up certificate details." ) self.add_option( - "keep_host_header", False, bool, + "keep_host_header", bool, False, """ Reverse Proxy: Keep the original host header instead of rewriting it to the reverse proxy target. @@ -292,12 +293,12 @@ class Options(optmanager.OptManager): ) self.add_option( - "http2", True, bool, + "http2", bool, True, "Enable/disable HTTP/2 support. " "HTTP/2 support is enabled by default.", ) self.add_option( - "http2_priority", False, bool, + "http2_priority", bool, False, """ PRIORITY forwarding for HTTP/2 connections. PRIORITY forwarding is disabled by default, because some webservers fail to implement the @@ -305,32 +306,32 @@ class Options(optmanager.OptManager): """ ) self.add_option( - "websocket", True, bool, + "websocket", bool, True, "Enable/disable WebSocket support. " "WebSocket support is enabled by default.", ) self.add_option( - "rawtcp", False, bool, + "rawtcp", bool, False, "Enable/disable experimental raw TCP support. " "Disabled by default. " ) self.add_option( - "spoof_source_address", False, bool, + "spoof_source_address", bool, False, """ Use the client's IP for server-side connections. Combine with --upstream-bind-address to spoof a fixed source address. """ ) self.add_option( - "upstream_auth", None, Optional[str], + "upstream_auth", Optional[str], None, """ Add HTTP Basic authentcation to upstream proxy and reverse proxy requests. Format: username:password. """ ) self.add_option( - "ssl_version_client", "secure", str, + "ssl_version_client", str, "secure", """ Set supported SSL/TLS versions for client connections. SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+. @@ -338,7 +339,7 @@ class Options(optmanager.OptManager): choices=tcp.sslversion_choices.keys(), ) self.add_option( - "ssl_version_server", "secure", str, + "ssl_version_server", str, "secure", """ Set supported SSL/TLS versions for server connections. SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+. @@ -346,22 +347,22 @@ class Options(optmanager.OptManager): choices=tcp.sslversion_choices.keys(), ) self.add_option( - "ssl_insecure", False, bool, + "ssl_insecure", bool, False, "Do not verify upstream server SSL/TLS certificates." ) self.add_option( - "ssl_verify_upstream_trusted_cadir", None, Optional[str], + "ssl_verify_upstream_trusted_cadir", Optional[str], None, """ Path to a directory of trusted CA certificates for upstream server verification prepared using the c_rehash tool. """ ) self.add_option( - "ssl_verify_upstream_trusted_ca", None, Optional[str], + "ssl_verify_upstream_trusted_ca", Optional[str], None, "Path to a PEM formatted trusted CA certificate." ) self.add_option( - "tcp_hosts", [], Sequence[str], + "tcp_hosts", Sequence[str], [], """ Generic TCP SSL proxy mode for all hosts that match the pattern. Similar to --ignore, but SSL connections are intercepted. The @@ -370,72 +371,72 @@ class Options(optmanager.OptManager): ) self.add_option( - "intercept", None, Optional[str], + "intercept", Optional[str], None, "Intercept filter expression." ) # Console options self.add_option( - "console_eventlog", False, bool, + "console_eventlog", bool, False, "Show event log." ) self.add_option( - "console_focus_follow", False, bool, + "console_focus_follow", bool, False, "Focus follows new flows." ) self.add_option( - "console_palette", "dark", str, + "console_palette", str, "dark", "Color palette.", choices=sorted(console_palettes), ) self.add_option( - "console_palette_transparent", False, bool, + "console_palette_transparent", bool, False, "Set transparent background for palette." ) self.add_option( - "console_mouse", True, bool, + "console_mouse", bool, True, "Console mouse interaction." ) self.add_option( - "console_order", None, Optional[str], + "console_order", Optional[str], None, "Flow sort order.", choices=view_orders, ) self.add_option( - "console_order_reversed", False, bool, + "console_order_reversed", bool, False, "Reverse the sorting order." ) self.add_option( - "filter", None, Optional[str], + "filter", Optional[str], None, "Filter view expression." ) # Web options self.add_option( - "web_open_browser", True, bool, + "web_open_browser", bool, True, "Start a browser." ) self.add_option( - "web_debug", False, bool, + "web_debug", bool, False, "Mitmweb debugging." ) self.add_option( - "web_port", 8081, int, + "web_port", int, 8081, "Mitmweb port." ) self.add_option( - "web_iface", "127.0.0.1", str, + "web_iface", str, "127.0.0.1", "Mitmweb interface." ) # Dump options self.add_option( - "filtstr", None, Optional[str], + "filtstr", Optional[str], None, "The filter string for mitmdump." ) self.add_option( - "flow_detail", 1, int, + "flow_detail", int, 1, "Flow detail display level." ) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index b05e78eb..a928b953 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -26,15 +26,15 @@ class _Option: def __init__( self, name: str, - default: typing.Any, typespec: typing.Type, + default: typing.Any, help: str, choices: typing.Optional[typing.Sequence[str]] ) -> None: typecheck.check_type(name, default, typespec) self.name = name - self._default = default self.typespec = typespec + self._default = default self.value = unset self.help = help self.choices = choices @@ -71,7 +71,7 @@ class _Option: def __deepcopy__(self, _): o = _Option( - self.name, self.default, self.typespec, self.help, self.choices + self.name, self.typespec, self.default, self.help, self.choices ) if self.has_changed(): o.value = self.current() @@ -101,14 +101,14 @@ class OptManager: def add_option( self, name: str, - default: typing.Any, typespec: typing.Type, + default: typing.Any, help: str, choices: typing.Optional[typing.Sequence[str]] = None ) -> None: if name in self._options: raise ValueError("Option %s already exists" % name) - self._options[name] = _Option(name, default, typespec, help, choices) + self._options[name] = _Option(name, typespec, default, help, choices) @contextlib.contextmanager def rollback(self, updated, reraise=False): diff --git a/test/mitmproxy/addons/test_core.py b/test/mitmproxy/addons/test_core.py index 7b9e9614..db739b5d 100644 --- a/test/mitmproxy/addons/test_core.py +++ b/test/mitmproxy/addons/test_core.py @@ -39,5 +39,5 @@ def test_modes(m): sa = core.Core() with taddons.context() as tctx: tctx.configure(sa, mode = "reverse:http://localhost") - with pytest.raises(Exception, match="Invalid mode"): + with pytest.raises(Exception, match="Invalid server specification"): tctx.configure(sa, mode = "reverse:") diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index ba31c750..db33cddd 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -13,36 +13,36 @@ from mitmproxy.test import tutils class TO(optmanager.OptManager): def __init__(self): super().__init__() - self.add_option("one", None, typing.Optional[int], "help") - self.add_option("two", 2, typing.Optional[int], "help") - self.add_option("bool", False, bool, "help") + self.add_option("one", typing.Optional[int], None, "help") + self.add_option("two", typing.Optional[int], 2, "help") + self.add_option("bool", bool, False, "help") class TD(optmanager.OptManager): def __init__(self): super().__init__() - self.add_option("one", "done", str, "help") - self.add_option("two", "dtwo", str, "help") + self.add_option("one", str, "done", "help") + self.add_option("two", str, "dtwo", "help") class TD2(TD): def __init__(self): super().__init__() - self.add_option("three", "dthree", str, "help") - self.add_option("four", "dfour", str, "help") + self.add_option("three", str, "dthree", "help") + self.add_option("four", str, "dfour", "help") class TM(optmanager.OptManager): def __init__(self): super().__init__() - self.add_option("two", ["foo"], typing.Sequence[str], "help") - self.add_option("one", None, typing.Optional[str], "help") + self.add_option("two", typing.Sequence[str], ["foo"], "help") + self.add_option("one", typing.Optional[str], None, "help") def test_add_option(): o = TO() with pytest.raises(ValueError, match="already exists"): - o.add_option("one", None, typing.Optional[int], "help") + o.add_option("one", typing.Optional[int], None, "help") def test_defaults(): @@ -76,7 +76,7 @@ def test_defaults(): def test_options(): o = TO() - assert o.keys() == set(["bool", "one", "two"]) + assert o.keys() == {"bool", "one", "two"} assert o.one is None assert o.two == 2 @@ -189,7 +189,7 @@ def test_rollback(): assert rec[3].bool is False with pytest.raises(exceptions.OptionsError): - with o.rollback(set(["one"]), reraise=True): + with o.rollback({"one"}, reraise=True): raise exceptions.OptionsError() @@ -270,14 +270,14 @@ def test_merge(): def test_option(): - o = optmanager._Option("test", 1, int, None, None) + o = optmanager._Option("test", int, 1, None, None) assert o.current() == 1 with pytest.raises(TypeError): o.set("foo") with pytest.raises(TypeError): - optmanager._Option("test", 1, str, None, None) + optmanager._Option("test", str, 1, None, None) - o2 = optmanager._Option("test", 1, int, None, None) + o2 = optmanager._Option("test", int, 1, None, None) assert o2 == o o2.set(5) assert o2 != o @@ -291,14 +291,14 @@ def test_dump(): class TTypes(optmanager.OptManager): def __init__(self): super().__init__() - self.add_option("str", "str", str, "help") - self.add_option("optstr", "optstr", typing.Optional[str], "help", "help") - self.add_option("bool", False, bool, "help") - self.add_option("bool_on", True, bool, "help") - self.add_option("int", 0, int, "help") - self.add_option("optint", 0, typing.Optional[int], "help") - self.add_option("seqstr", [], typing.Sequence[str], "help") - self.add_option("unknown", 0.0, float, "help") + self.add_option("str", str, "str", "help") + self.add_option("optstr", typing.Optional[str], "optstr", "help", "help") + self.add_option("bool", bool, False, "help") + self.add_option("bool_on", bool, True, "help") + self.add_option("int", int, 0, "help") + self.add_option("optint", typing.Optional[int], 0, "help") + self.add_option("seqstr", typing.Sequence[str], [], "help") + self.add_option("unknown", float, 0.0, "help") def test_make_parser(): -- cgit v1.2.3 From 63179d97511519ce0ba4384b2c6849ba96748d88 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 8 Mar 2017 15:17:07 +0100 Subject: core -> core_option_validation longer, but much clearer for devs who are unfamiliar with the codebase. --- mitmproxy/addons/__init__.py | 4 +- mitmproxy/addons/core.py | 45 ---------------------- mitmproxy/addons/core_option_validation.py | 45 ++++++++++++++++++++++ test/mitmproxy/addons/test_core.py | 43 --------------------- .../addons/test_core_option_validation.py | 43 +++++++++++++++++++++ 5 files changed, 90 insertions(+), 90 deletions(-) delete mode 100644 mitmproxy/addons/core.py create mode 100644 mitmproxy/addons/core_option_validation.py delete mode 100644 test/mitmproxy/addons/test_core.py create mode 100644 test/mitmproxy/addons/test_core_option_validation.py diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py index 1bf89bbb..16510640 100644 --- a/mitmproxy/addons/__init__.py +++ b/mitmproxy/addons/__init__.py @@ -3,7 +3,7 @@ from mitmproxy.addons import anticomp from mitmproxy.addons import check_alpn from mitmproxy.addons import check_ca from mitmproxy.addons import clientplayback -from mitmproxy.addons import core +from mitmproxy.addons import core_option_validation from mitmproxy.addons import disable_h2c_upgrade from mitmproxy.addons import onboarding from mitmproxy.addons import proxyauth @@ -20,7 +20,7 @@ from mitmproxy.addons import upstream_auth def default_addons(): return [ - core.Core(), + core_option_validation.CoreOptionValidation(), anticache.AntiCache(), anticomp.AntiComp(), check_alpn.CheckALPN(), diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py deleted file mode 100644 index 4c6a5516..00000000 --- a/mitmproxy/addons/core.py +++ /dev/null @@ -1,45 +0,0 @@ -""" - The core addon is responsible for verifying core settings that are not - checked by other addons. -""" -from mitmproxy import exceptions -from mitmproxy import platform -from mitmproxy.net import server_spec -from mitmproxy.utils import human - - -class Core: - def configure(self, opts, updated): - if opts.add_upstream_certs_to_client_chain and not opts.upstream_cert: - raise exceptions.OptionsError( - "The no-upstream-cert and add-upstream-certs-to-client-chain " - "options are mutually exclusive. If no-upstream-cert is enabled " - "then the upstream certificate is not retrieved before generating " - "the client certificate chain." - ) - if "body_size_limit" in updated and opts.body_size_limit: - try: - opts._processed["body_size_limit"] = human.parse_size( - opts.body_size_limit - ) - except ValueError as e: - raise exceptions.OptionsError( - "Invalid body size limit specification: %s" % - opts.body_size_limit - ) - if "mode" in updated: - mode = opts.mode - if mode.startswith("reverse:") or mode.startswith("upstream:"): - try: - server_spec.parse_with_mode(mode) - except ValueError as e: - raise exceptions.OptionsError(str(e)) from e - elif mode == "transparent": - if not platform.original_addr: - raise exceptions.OptionsError( - "Transparent mode not supported on this platform." - ) - elif mode not in ["regular", "socks5"]: - raise exceptions.OptionsError( - "Invalid mode specification: %s" % mode - ) diff --git a/mitmproxy/addons/core_option_validation.py b/mitmproxy/addons/core_option_validation.py new file mode 100644 index 00000000..fd5f2788 --- /dev/null +++ b/mitmproxy/addons/core_option_validation.py @@ -0,0 +1,45 @@ +""" + The core addon is responsible for verifying core settings that are not + checked by other addons. +""" +from mitmproxy import exceptions +from mitmproxy import platform +from mitmproxy.net import server_spec +from mitmproxy.utils import human + + +class CoreOptionValidation: + def configure(self, opts, updated): + if opts.add_upstream_certs_to_client_chain and not opts.upstream_cert: + raise exceptions.OptionsError( + "The no-upstream-cert and add-upstream-certs-to-client-chain " + "options are mutually exclusive. If no-upstream-cert is enabled " + "then the upstream certificate is not retrieved before generating " + "the client certificate chain." + ) + if "body_size_limit" in updated and opts.body_size_limit: + try: + opts._processed["body_size_limit"] = human.parse_size( + opts.body_size_limit + ) + except ValueError as e: + raise exceptions.OptionsError( + "Invalid body size limit specification: %s" % + opts.body_size_limit + ) + if "mode" in updated: + mode = opts.mode + if mode.startswith("reverse:") or mode.startswith("upstream:"): + try: + server_spec.parse_with_mode(mode) + except ValueError as e: + raise exceptions.OptionsError(str(e)) from e + elif mode == "transparent": + if not platform.original_addr: + raise exceptions.OptionsError( + "Transparent mode not supported on this platform." + ) + elif mode not in ["regular", "socks5"]: + raise exceptions.OptionsError( + "Invalid mode specification: %s" % mode + ) diff --git a/test/mitmproxy/addons/test_core.py b/test/mitmproxy/addons/test_core.py deleted file mode 100644 index db739b5d..00000000 --- a/test/mitmproxy/addons/test_core.py +++ /dev/null @@ -1,43 +0,0 @@ -from mitmproxy import exceptions -from mitmproxy.addons import core -from mitmproxy.test import taddons -import pytest -from unittest import mock - - -def test_simple(): - sa = core.Core() - with taddons.context() as tctx: - with pytest.raises(exceptions.OptionsError): - tctx.configure(sa, body_size_limit = "invalid") - tctx.configure(sa, body_size_limit = "1m") - assert tctx.options._processed["body_size_limit"] - - with pytest.raises(exceptions.OptionsError, match="mutually exclusive"): - tctx.configure( - sa, - add_upstream_certs_to_client_chain = True, - upstream_cert = False - ) - with pytest.raises(exceptions.OptionsError, match="Invalid mode"): - tctx.configure( - sa, - mode = "Flibble" - ) - - -@mock.patch("mitmproxy.platform.original_addr", None) -def test_no_transparent(): - sa = core.Core() - with taddons.context() as tctx: - with pytest.raises(Exception, match="Transparent mode not supported"): - tctx.configure(sa, mode = "transparent") - - -@mock.patch("mitmproxy.platform.original_addr") -def test_modes(m): - sa = core.Core() - with taddons.context() as tctx: - tctx.configure(sa, mode = "reverse:http://localhost") - with pytest.raises(Exception, match="Invalid server specification"): - tctx.configure(sa, mode = "reverse:") diff --git a/test/mitmproxy/addons/test_core_option_validation.py b/test/mitmproxy/addons/test_core_option_validation.py new file mode 100644 index 00000000..0bb2bb0d --- /dev/null +++ b/test/mitmproxy/addons/test_core_option_validation.py @@ -0,0 +1,43 @@ +from mitmproxy import exceptions +from mitmproxy.addons import core_option_validation +from mitmproxy.test import taddons +import pytest +from unittest import mock + + +def test_simple(): + sa = core_option_validation.CoreOptionValidation() + with taddons.context() as tctx: + with pytest.raises(exceptions.OptionsError): + tctx.configure(sa, body_size_limit = "invalid") + tctx.configure(sa, body_size_limit = "1m") + assert tctx.options._processed["body_size_limit"] + + with pytest.raises(exceptions.OptionsError, match="mutually exclusive"): + tctx.configure( + sa, + add_upstream_certs_to_client_chain = True, + upstream_cert = False + ) + with pytest.raises(exceptions.OptionsError, match="Invalid mode"): + tctx.configure( + sa, + mode = "Flibble" + ) + + +@mock.patch("mitmproxy.platform.original_addr", None) +def test_no_transparent(): + sa = core_option_validation.CoreOptionValidation() + with taddons.context() as tctx: + with pytest.raises(Exception, match="Transparent mode not supported"): + tctx.configure(sa, mode = "transparent") + + +@mock.patch("mitmproxy.platform.original_addr") +def test_modes(m): + sa = core_option_validation.CoreOptionValidation() + with taddons.context() as tctx: + tctx.configure(sa, mode = "reverse:http://localhost") + with pytest.raises(Exception, match="Invalid server specification"): + tctx.configure(sa, mode = "reverse:") -- cgit v1.2.3 From b345f5d432f3b8c100b2e709f96bfd1701c87196 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 8 Mar 2017 01:00:55 +0100 Subject: typing.Type -> type `typing.Type` should have been `typing.Type[Any]`, which in turn is equivalent to `type` --- mitmproxy/optmanager.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index a928b953..8661aece 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -26,7 +26,7 @@ class _Option: def __init__( self, name: str, - typespec: typing.Type, + typespec: type, default: typing.Any, help: str, choices: typing.Optional[typing.Sequence[str]] @@ -101,7 +101,7 @@ class OptManager: def add_option( self, name: str, - typespec: typing.Type, + typespec: type, default: typing.Any, help: str, choices: typing.Optional[typing.Sequence[str]] = None -- cgit v1.2.3 From 53178f35be4ed93c7660d92a88577506bafed5cf Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 9 Mar 2017 08:53:36 +1300 Subject: Send change event when options are reset --- mitmproxy/optmanager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 8661aece..9553bd32 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -174,6 +174,7 @@ class OptManager: """ for o in self._options.values(): o.reset() + self.changed.send(self._options.keys()) def update(self, **kwargs): updated = set(kwargs.keys()) -- cgit v1.2.3