From b4d33aaebf5fa78736b5d5f6864f672ccd08e716 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 2 Dec 2016 16:15:14 +1300 Subject: options: save defaults, add .reset() to restore defaults Use .reset() in console app to clear options. --- mitmproxy/optmanager.py | 20 ++++++++++++++++++++ mitmproxy/tools/console/options.py | 17 +---------------- test/mitmproxy/test_optmanager.py | 37 +++++++++++++++++++++++++++++++++++++ 3 files changed, 58 insertions(+), 16 deletions(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 20492f82..0421d4be 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -1,6 +1,7 @@ import contextlib import blinker import pprint +import inspect from mitmproxy import exceptions from mitmproxy.utils import typecheck @@ -12,6 +13,10 @@ from mitmproxy.utils import typecheck class OptManager: """ + OptManager is the base class from which Options objects are derived. + Note that the __init__ method of all child classes must force all + arguments to be positional only, by including a "*" argument. + .changed is a blinker Signal that triggers whenever options are updated. If any handler in the chain raises an exceptions.OptionsError exception, all changes are rolled back, the exception is suppressed, @@ -26,6 +31,15 @@ class OptManager: # ._initialized = True as the final operation. instance = super().__new__(cls) instance.__dict__["_opts"] = {} + + defaults = {} + for klass in reversed(inspect.getmro(cls)): + for p in inspect.signature(klass.__init__).parameters.values(): + if p.kind in (p.KEYWORD_ONLY, p.POSITIONAL_OR_KEYWORD): + if not p.default == p.empty: + defaults[p.name] = p.default + instance.__dict__["_defaults"] = defaults + return instance def __init__(self): @@ -78,6 +92,12 @@ class OptManager: def get(self, k, d=None): return self._opts.get(k, d) + def reset(self): + """ + Restore defaults for all options. + """ + self.update(**self._defaults) + def update(self, **kwargs): updated = set(kwargs.keys()) for k, v in kwargs.items(): diff --git a/mitmproxy/tools/console/options.py b/mitmproxy/tools/console/options.py index 824041dc..9d698161 100644 --- a/mitmproxy/tools/console/options.py +++ b/mitmproxy/tools/console/options.py @@ -160,22 +160,7 @@ class Options(urwid.WidgetWrap): return super().keypress(size, key) def clearall(self): - self.master.options.update( - anticache = False, - anticomp = False, - ignore_hosts = (), - tcp_hosts = (), - replay_kill_extra = False, - no_upstream_cert = False, - refresh_server_playback = True, - replacements = [], - scripts = [], - setheaders = [], - showhost = False, - stickyauth = None, - stickycookie = None, - default_contentview = "auto", - ) + self.master.options.reset() signals.update_settings.send(self) signals.status_message.send( message = "All select.Options cleared", diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 3c845707..47e40d98 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -12,6 +12,43 @@ class TO(optmanager.OptManager): super().__init__() +class TD(optmanager.OptManager): + def __init__(self, one="done", two="dtwo", three="error"): + self.one = one + self.two = two + self.three = three + super().__init__() + + +class TD2(TD): + def __init__(self, *, three="dthree", four="dfour", **kwargs): + self.three = three + self.four = four + super().__init__(**kwargs) + + +def test_defaults(): + o = TD2() + assert o._defaults == { + "one": "done", + "two": "dtwo", + "three": "dthree", + "four": "dfour", + } + newvals = dict( + one="xone", + two="xtwo", + three="xthree", + four="xfour", + ) + o.update(**newvals) + for k, v in newvals.items(): + assert v == o.get(k) + o.reset() + for k, v in o._defaults.items(): + assert v == o.get(k) + + def test_options(): o = TO(two="three") assert o.keys() == set(["one", "two"]) -- cgit v1.2.3 From 297493801ded90ee5b9717115fb2fa25bac43d22 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 3 Dec 2016 09:00:55 +1300 Subject: Clean up dump tests - Remove tests that redundantly test addon funtionality that we've already tested more comprehensively elsewhere. - Extend to 100% coverage for tools/dump.py --- mitmproxy/tools/dump.py | 11 --- test/mitmproxy/test_dump.py | 163 -------------------------------------- test/mitmproxy/test_tools_dump.py | 38 +++++++++ 3 files changed, 38 insertions(+), 174 deletions(-) delete mode 100644 test/mitmproxy/test_dump.py create mode 100644 test/mitmproxy/test_tools_dump.py diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index 3cd94c30..69258ae2 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -3,7 +3,6 @@ from typing import Optional, IO from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import addons -from mitmproxy import io from mitmproxy import options from mitmproxy import master from mitmproxy.addons import dumper, termlog @@ -62,16 +61,6 @@ class DumpMaster(master.Master): self.add_log("Flow file corrupted.", "error") raise DumpError(v) - def _readflow(self, paths): - """ - Utitility function that reads a list of flows - or raises a DumpError if that fails. - """ - try: - return io.read_flows_from_paths(paths) - except exceptions.FlowReadException as e: - raise DumpError(str(e)) - @controller.handler def log(self, e): if e.level == "error": diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py deleted file mode 100644 index c6b15c84..00000000 --- a/test/mitmproxy/test_dump.py +++ /dev/null @@ -1,163 +0,0 @@ -from mitmproxy.test import tflow -import os -import io - -from mitmproxy.tools import dump -from mitmproxy import exceptions -from mitmproxy import proxy -from mitmproxy.test import tutils -from . import mastertest - - -class TestDumpMaster(mastertest.MasterTest): - def dummy_cycle(self, master, n, content): - mastertest.MasterTest.dummy_cycle(self, master, n, content) - return master.options.tfile.getvalue() - - def mkmaster(self, flt, **options): - if "verbosity" not in options: - options["verbosity"] = 0 - if "flow_detail" not in options: - options["flow_detail"] = 0 - o = dump.Options(filtstr=flt, tfile=io.StringIO(), **options) - return dump.DumpMaster(o, proxy.DummyServer()) - - def test_basic(self): - for i in (1, 2, 3): - assert "GET" in self.dummy_cycle( - self.mkmaster("~s", flow_detail=i), - 1, - b"" - ) - assert "GET" in self.dummy_cycle( - self.mkmaster("~s", flow_detail=i), - 1, - b"\x00\x00\x00" - ) - assert "GET" in self.dummy_cycle( - self.mkmaster("~s", flow_detail=i), - 1, - b"ascii" - ) - - def test_error(self): - o = dump.Options( - tfile=io.StringIO(), - flow_detail=1 - ) - m = dump.DumpMaster(o, proxy.DummyServer()) - f = tflow.tflow(err=True) - m.error(f) - assert "error" in o.tfile.getvalue() - - def test_replay(self): - o = dump.Options(http2=False, server_replay=["nonexistent"], replay_kill_extra=True) - tutils.raises(exceptions.OptionsError, dump.DumpMaster, o, proxy.DummyServer()) - - with tutils.tmpdir() as t: - p = os.path.join(t, "rep") - self.flowfile(p) - - o = dump.Options(http2=False, server_replay=[p], replay_kill_extra=True) - o.verbosity = 0 - o.flow_detail = 0 - m = dump.DumpMaster(o, proxy.DummyServer()) - - self.cycle(m, b"content") - self.cycle(m, b"content") - - o = dump.Options(http2=False, server_replay=[p], replay_kill_extra=False) - o.verbosity = 0 - o.flow_detail = 0 - m = dump.DumpMaster(o, proxy.DummyServer()) - self.cycle(m, b"nonexistent") - - o = dump.Options(http2=False, client_replay=[p], replay_kill_extra=False) - o.verbosity = 0 - o.flow_detail = 0 - m = dump.DumpMaster(o, proxy.DummyServer()) - - def test_read(self): - with tutils.tmpdir() as t: - p = os.path.join(t, "read") - self.flowfile(p) - assert "GET" in self.dummy_cycle( - self.mkmaster(None, flow_detail=1, rfile=p), - 1, b"", - ) - tutils.raises( - dump.DumpError, - self.mkmaster, None, verbosity=1, rfile="/nonexistent" - ) - tutils.raises( - dump.DumpError, - self.mkmaster, None, verbosity=1, rfile="test_dump.py" - ) - - def test_options(self): - o = dump.Options(verbosity = 2) - assert o.verbosity == 2 - - def test_filter(self): - assert "GET" not in self.dummy_cycle( - self.mkmaster("~u foo", verbosity=1), 1, b"" - ) - - def test_replacements(self): - o = dump.Options( - replacements=[(".*", "content", "foo")], - tfile = io.StringIO(), - ) - o.verbosity = 0 - o.flow_detail = 0 - m = dump.DumpMaster(o, proxy.DummyServer()) - f = self.cycle(m, b"content") - assert f.request.content == b"foo" - - def test_setheader(self): - o = dump.Options( - setheaders=[(".*", "one", "two")], - tfile=io.StringIO() - ) - o.verbosity = 0 - o.flow_detail = 0 - m = dump.DumpMaster(o, proxy.DummyServer()) - f = self.cycle(m, b"content") - assert f.request.headers["one"] == "two" - - def test_script(self): - ret = self.dummy_cycle( - self.mkmaster( - None, - scripts=[tutils.test_data.path("mitmproxy/data/scripts/all.py")], - verbosity=2 - ), - 1, b"", - ) - assert "XCLIENTCONNECT" in ret - assert "XSERVERCONNECT" in ret - assert "XREQUEST" in ret - assert "XRESPONSE" in ret - assert "XCLIENTDISCONNECT" in ret - tutils.raises( - exceptions.AddonError, - self.mkmaster, - None, scripts=["nonexistent"] - ) - tutils.raises( - exceptions.AddonError, - self.mkmaster, - None, scripts=["starterr.py"] - ) - - def test_stickycookie(self): - self.dummy_cycle( - self.mkmaster(None, stickycookie = ".*"), - 1, b"" - ) - - def test_stickyauth(self): - self.dummy_cycle( - self.mkmaster(None, stickyauth = ".*"), - 1, b"" - ) diff --git a/test/mitmproxy/test_tools_dump.py b/test/mitmproxy/test_tools_dump.py new file mode 100644 index 00000000..1488f33b --- /dev/null +++ b/test/mitmproxy/test_tools_dump.py @@ -0,0 +1,38 @@ +import os + +from mitmproxy.tools import dump +from mitmproxy import proxy +from mitmproxy.test import tutils +from mitmproxy import log +from mitmproxy import controller +from . import mastertest + + +class TestDumpMaster(mastertest.MasterTest): + def mkmaster(self, flt, **options): + o = dump.Options(filtstr=flt, verbosity=-1, flow_detail=0, **options) + return dump.DumpMaster(o, proxy.DummyServer()) + + def test_read(self): + with tutils.tmpdir() as t: + p = os.path.join(t, "read") + self.flowfile(p) + self.dummy_cycle( + self.mkmaster(None, rfile=p), + 1, b"", + ) + tutils.raises( + dump.DumpError, + self.mkmaster, None, rfile="/nonexistent" + ) + tutils.raises( + dump.DumpError, + self.mkmaster, None, rfile="test_dump.py" + ) + + def test_has_error(self): + m = self.mkmaster(None) + ent = log.LogEntry("foo", "error") + ent.reply = controller.DummyReply() + m.log(ent) + assert m.has_errored -- cgit v1.2.3 From b231836c70a68daa6016537e5ec18ed7a7cc3b1a Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 3 Dec 2016 09:20:27 +1300 Subject: Get rid of tfile testing option It's weird, it's ugly, it's getting in the way of my options refactoring, and it must therefore die. --- mitmproxy/addons/dumper.py | 6 ++--- mitmproxy/addons/termlog.py | 6 +++-- mitmproxy/tools/dump.py | 4 +--- mitmproxy/tools/web/master.py | 4 +--- test/mitmproxy/addons/test_dumper.py | 45 ++++++++++++++++------------------- test/mitmproxy/addons/test_termlog.py | 4 ++-- 6 files changed, 31 insertions(+), 38 deletions(-) diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 29f60cfe..e94d6a79 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -1,4 +1,5 @@ import itertools +import sys import click @@ -25,10 +26,10 @@ def colorful(line, styles): class Dumper: - def __init__(self): + def __init__(self, outfile=sys.stdout): self.filter = None # type: flowfilter.TFilter self.flow_detail = None # type: int - self.outfp = None # type: typing.io.TextIO + self.outfp = outfile # type: typing.io.TextIO self.showhost = None # type: bool self.default_contentview = "auto" # type: str @@ -43,7 +44,6 @@ class Dumper: else: self.filter = None self.flow_detail = options.flow_detail - self.outfp = options.tfile self.showhost = options.showhost self.default_contentview = options.default_contentview diff --git a/mitmproxy/addons/termlog.py b/mitmproxy/addons/termlog.py index 05be32d0..b75f5f5a 100644 --- a/mitmproxy/addons/termlog.py +++ b/mitmproxy/addons/termlog.py @@ -1,11 +1,13 @@ +import sys import click from mitmproxy import log class TermLog: - def __init__(self): + def __init__(self, outfile=sys.stdout): self.options = None + self.outfile = outfile def configure(self, options, updated): self.options = options @@ -14,7 +16,7 @@ class TermLog: if self.options.verbosity >= log.log_tier(e.level): click.secho( e.msg, - file=self.options.tfile, + file=self.outfile, fg=dict(error="red", warn="yellow").get(e.level), dim=(e.level == "debug"), err=(e.level == "error") diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index 69258ae2..90332627 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -1,4 +1,4 @@ -from typing import Optional, IO +from typing import Optional from mitmproxy import controller from mitmproxy import exceptions @@ -20,13 +20,11 @@ class Options(options.Options): keepserving: bool = False, filtstr: Optional[str] = None, flow_detail: int = 1, - tfile: Optional[IO[str]] = None, **kwargs ) -> None: self.filtstr = filtstr self.flow_detail = flow_detail self.keepserving = keepserving - self.tfile = tfile super().__init__(**kwargs) diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 89ad698d..36bdcb07 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -1,5 +1,5 @@ import webbrowser -from typing import Optional, IO +from typing import Optional import tornado.httpserver import tornado.ioloop @@ -20,7 +20,6 @@ class Options(options.Options): self, *, # all args are keyword-only. intercept: Optional[str] = None, - tfile: Optional[IO[str]] = None, open_browser: bool = True, wdebug: bool = False, wport: int = 8081, @@ -28,7 +27,6 @@ class Options(options.Options): **kwargs ) -> None: self.intercept = intercept - self.tfile = tfile self.open_browser = open_browser self.wdebug = wdebug self.wport = wport diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index 0d61f800..f87df329 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -28,43 +28,40 @@ def test_configure(): def test_simple(): - d = dumper.Dumper() + sio = io.StringIO() + d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - sio = io.StringIO() - ctx.configure(d, tfile = sio, flow_detail = 0) + ctx.configure(d, flow_detail = 0) d.response(tflow.tflow(resp=True)) assert not sio.getvalue() sio.truncate(0) - ctx.configure(d, tfile = sio, flow_detail = 1) + ctx.configure(d, flow_detail = 1) d.response(tflow.tflow(resp=True)) assert sio.getvalue() sio.truncate(0) - ctx.configure(d, tfile = sio, flow_detail = 1) + ctx.configure(d, flow_detail = 1) d.error(tflow.tflow(err=True)) assert sio.getvalue() sio.truncate(0) - ctx.configure(d, tfile = sio, flow_detail = 4) + ctx.configure(d, flow_detail = 4) d.response(tflow.tflow(resp=True)) assert sio.getvalue() sio.truncate(0) - sio = io.StringIO() - ctx.configure(d, tfile = sio, flow_detail = 4) + ctx.configure(d, flow_detail = 4) d.response(tflow.tflow(resp=True)) assert "<<" in sio.getvalue() sio.truncate(0) - sio = io.StringIO() - ctx.configure(d, tfile = sio, flow_detail = 4) + ctx.configure(d, flow_detail = 4) d.response(tflow.tflow(err=True)) assert "<<" in sio.getvalue() sio.truncate(0) - sio = io.StringIO() - ctx.configure(d, tfile = sio, flow_detail = 4) + ctx.configure(d, flow_detail = 4) flow = tflow.tflow() flow.request = tutils.treq() flow.request.stickycookie = True @@ -77,8 +74,7 @@ def test_simple(): assert sio.getvalue() sio.truncate(0) - sio = io.StringIO() - ctx.configure(d, tfile = sio, flow_detail = 4) + ctx.configure(d, flow_detail = 4) flow = tflow.tflow(resp=tutils.tresp(content=b"{")) flow.response.headers["content-type"] = "application/json" flow.response.status_code = 400 @@ -86,8 +82,7 @@ def test_simple(): assert sio.getvalue() sio.truncate(0) - sio = io.StringIO() - ctx.configure(d, tfile = sio, flow_detail = 4) + ctx.configure(d, flow_detail = 4) flow = tflow.tflow() flow.request.content = None flow.response = http.HTTPResponse.wrap(tutils.tresp()) @@ -102,20 +97,20 @@ def test_echo_body(): f.response.headers["content-type"] = "text/html" f.response.content = b"foo bar voing\n" * 100 - d = dumper.Dumper() sio = io.StringIO() + d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - ctx.configure(d, tfile=sio, flow_detail = 3) + ctx.configure(d, flow_detail = 3) d._echo_message(f.response) t = sio.getvalue() assert "cut off" in t def test_echo_request_line(): - d = dumper.Dumper() sio = io.StringIO() + d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - ctx.configure(d, tfile=sio, flow_detail = 3, showhost = True) + ctx.configure(d, flow_detail = 3, showhost = True) f = tflow.tflow(client_conn=None, server_conn=True, resp=True) f.request.is_replay = True d._echo_request_line(f) @@ -139,19 +134,19 @@ class TestContentView: @mock.patch("mitmproxy.contentviews.ViewAuto.__call__") def test_contentview(self, view_auto): view_auto.side_effect = exceptions.ContentViewException("") - d = dumper.Dumper() + sio = io.StringIO() + d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - sio = io.StringIO() - ctx.configure(d, flow_detail=4, verbosity=3, tfile=sio) + ctx.configure(d, flow_detail=4, verbosity=3) d.response(tflow.tflow()) assert "Content viewer failed" in ctx.master.event_log[0][1] def test_tcp(): - d = dumper.Dumper() sio = io.StringIO() + d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - ctx.configure(d, tfile=sio, flow_detail = 3, showhost = True) + ctx.configure(d, flow_detail = 3, showhost = True) f = tflow.ttcpflow(client_conn=True, server_conn=True) d.tcp_message(f) assert "it's me" in sio.getvalue() diff --git a/test/mitmproxy/addons/test_termlog.py b/test/mitmproxy/addons/test_termlog.py index 880fcb51..d9e18134 100644 --- a/test/mitmproxy/addons/test_termlog.py +++ b/test/mitmproxy/addons/test_termlog.py @@ -7,9 +7,9 @@ from mitmproxy.tools import dump class TestTermLog: def test_simple(self): - t = termlog.TermLog() sio = io.StringIO() - t.configure(dump.Options(tfile = sio, verbosity = 2), set([])) + t = termlog.TermLog(outfile=sio) + t.configure(dump.Options(verbosity = 2), set([])) t.log(log.LogEntry("one", "info")) assert "one" in sio.getvalue() t.log(log.LogEntry("two", "debug")) -- cgit v1.2.3 From 0c091bd92b9f0f59c65cf392f334719294ef397e Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 3 Dec 2016 09:42:14 +1300 Subject: Options - avoid mutation, API cleanup, has_changed - Always return a deepcopy of options to avoid accidental mutation of options state. - Remove .get(opt, default). This is an inappropriate API for Options - trying to retrieve an option that doesn't exist should always be an error. - Add the has_changed method that checks if an option differs from the default, use it in mitmproxy console. --- mitmproxy/addons/streamfile.py | 2 +- mitmproxy/options.py | 22 ++++++++++---------- mitmproxy/optmanager.py | 16 +++++++++++---- mitmproxy/tools/console/master.py | 2 +- mitmproxy/tools/console/options.py | 41 +++++++++++++++++++++----------------- test/mitmproxy/test_optmanager.py | 7 +++++-- 6 files changed, 53 insertions(+), 37 deletions(-) diff --git a/mitmproxy/addons/streamfile.py b/mitmproxy/addons/streamfile.py index 377f277d..2fc61015 100644 --- a/mitmproxy/addons/streamfile.py +++ b/mitmproxy/addons/streamfile.py @@ -23,7 +23,7 @@ class StreamFile: def configure(self, options, updated): # We're already streaming - stop the previous stream and restart if "filtstr" in updated: - if options.get("filtstr"): + if options.filtstr: self.filt = flowfilter.parse(options.filtstr) if not self.filt: raise exceptions.OptionsError( diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 8a9385da..bf35cd59 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -30,19 +30,19 @@ class Options(optmanager.OptManager): app_port: int = APP_PORT, anticache: bool = False, anticomp: bool = False, - client_replay: Sequence[str] = (), + client_replay: Sequence[str] = [], replay_kill_extra: bool = False, keepserving: bool = True, no_server: bool = False, server_replay_nopop: bool = False, refresh_server_playback: bool = False, rfile: Optional[str] = None, - scripts: Sequence[str] = (), + scripts: Sequence[str] = [], showhost: bool = False, - replacements: Sequence[Tuple[str, str, str]] = (), - server_replay_use_headers: Sequence[str] = (), - setheaders: Sequence[Tuple[str, str, str]] = (), - server_replay: Sequence[str] = (), + replacements: Sequence[Tuple[str, str, str]] = [], + server_replay_use_headers: Sequence[str] = [], + setheaders: Sequence[Tuple[str, str, str]] = [], + server_replay: Sequence[str] = [], stickycookie: Optional[str] = None, stickyauth: Optional[str] = None, stream_large_bodies: Optional[int] = None, @@ -51,8 +51,8 @@ class Options(optmanager.OptManager): streamfile: Optional[str] = None, streamfile_append: bool = False, server_replay_ignore_content: bool = False, - server_replay_ignore_params: Sequence[str] = (), - server_replay_ignore_payload_params: Sequence[str] = (), + server_replay_ignore_params: Sequence[str] = [], + server_replay_ignore_payload_params: Sequence[str] = [], server_replay_ignore_host: bool = False, # Proxy options auth_nonanonymous: bool = False, @@ -61,12 +61,12 @@ class Options(optmanager.OptManager): add_upstream_certs_to_client_chain: bool = False, body_size_limit: Optional[int] = None, cadir: str = CA_DIR, - certs: Sequence[Tuple[str, str]] = (), + certs: Sequence[Tuple[str, str]] = [], ciphers_client: str=DEFAULT_CLIENT_CIPHERS, ciphers_server: Optional[str]=None, clientcerts: Optional[str] = None, http2: bool = True, - ignore_hosts: Sequence[str] = (), + ignore_hosts: Sequence[str] = [], listen_host: str = "", listen_port: int = LISTEN_PORT, upstream_bind_address: str = "", @@ -82,7 +82,7 @@ class Options(optmanager.OptManager): ssl_insecure: bool = False, ssl_verify_upstream_trusted_cadir: Optional[str] = None, ssl_verify_upstream_trusted_ca: Optional[str] = None, - tcp_hosts: Sequence[str] = () + tcp_hosts: Sequence[str] = [] ) -> None: # We could replace all assignments with clever metaprogramming, # but type hints are a much more valueable asset. diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 0421d4be..56122baa 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -2,6 +2,7 @@ import contextlib import blinker import pprint import inspect +import copy from mitmproxy import exceptions from mitmproxy.utils import typecheck @@ -21,6 +22,9 @@ class OptManager: updated. If any handler in the chain raises an exceptions.OptionsError exception, all changes are rolled back, the exception is suppressed, and the .errored signal is notified. + + Optmanager always returns a deep copy of options to ensure that + mutation doesn't change the option state inadvertently. """ _initialized = False attributes = [] @@ -67,7 +71,7 @@ class OptManager: def __getattr__(self, attr): if attr in self._opts: - return self._opts[attr] + return copy.deepcopy(self._opts[attr]) else: raise AttributeError("No such option: %s" % attr) @@ -89,9 +93,6 @@ class OptManager: def keys(self): return set(self._opts.keys()) - def get(self, k, d=None): - return self._opts.get(k, d) - def reset(self): """ Restore defaults for all options. @@ -132,6 +133,13 @@ class OptManager: setattr(self, attr, not getattr(self, attr)) return toggle + def has_changed(self, option): + """ + Has the option changed from the default? + """ + if getattr(self, option) != self._defaults[option]: + return True + def __repr__(self): options = pprint.pformat(self._opts, indent=4).strip(" {}") if "\n" in options: diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 5d0e0ef4..455824d3 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -47,7 +47,7 @@ class Options(mitmproxy.options.Options): focus_follow: bool = False, intercept: Optional[str] = None, filter: Optional[str] = None, - palette: Optional[str] = None, + palette: Optional[str] = palettes.DEFAULT, palette_transparent: bool = False, no_mouse: bool = False, order: Optional[str] = None, diff --git a/mitmproxy/tools/console/options.py b/mitmproxy/tools/console/options.py index 9d698161..94483b3d 100644 --- a/mitmproxy/tools/console/options.py +++ b/mitmproxy/tools/console/options.py @@ -3,7 +3,6 @@ import urwid from mitmproxy import contentviews from mitmproxy.tools.console import common from mitmproxy.tools.console import grideditor -from mitmproxy.tools.console import palettes from mitmproxy.tools.console import select from mitmproxy.tools.console import signals @@ -26,6 +25,12 @@ def _mkhelp(): help_context = _mkhelp() +def checker(opt, options): + def _check(): + return options.has_changed(opt) + return _check + + class Options(urwid.WidgetWrap): def __init__(self, master): @@ -36,25 +41,25 @@ class Options(urwid.WidgetWrap): select.Option( "Header Set Patterns", "H", - lambda: len(master.options.setheaders), + checker("setheaders", master.options), self.setheaders ), select.Option( "Ignore Patterns", "I", - lambda: master.options.ignore_hosts, + checker("ignore_hosts", master.options), self.ignore_hosts ), select.Option( "Replacement Patterns", "R", - lambda: len(master.options.replacements), + checker("replacements", master.options), self.replacepatterns ), select.Option( "Scripts", "S", - lambda: master.options.scripts, + checker("scripts", master.options), self.scripts ), @@ -62,19 +67,19 @@ class Options(urwid.WidgetWrap): select.Option( "Default Display Mode", "M", - lambda: self.master.options.default_contentview != "auto", + checker("default_contentview", master.options), self.default_displaymode ), select.Option( "Palette", "P", - lambda: self.master.palette != palettes.DEFAULT, + checker("palette", master.options), self.palette ), select.Option( "Show Host", "w", - lambda: master.options.showhost, + checker("showhost", master.options), master.options.toggler("showhost") ), @@ -82,19 +87,19 @@ class Options(urwid.WidgetWrap): select.Option( "No Upstream Certs", "U", - lambda: master.options.no_upstream_cert, + checker("no_upstream_cert", master.options), master.options.toggler("no_upstream_cert") ), select.Option( "TCP Proxying", "T", - lambda: master.options.tcp_hosts, + checker("tcp_hosts", master.options), self.tcp_hosts ), select.Option( "Don't Verify SSL/TLS Certificates", "V", - lambda: master.options.ssl_insecure, + checker("ssl_insecure", master.options), master.options.toggler("ssl_insecure") ), @@ -102,37 +107,37 @@ class Options(urwid.WidgetWrap): select.Option( "Anti-Cache", "a", - lambda: master.options.anticache, + checker("anticache", master.options), master.options.toggler("anticache") ), select.Option( "Anti-Compression", "o", - lambda: master.options.anticomp, + checker("anticomp", master.options), master.options.toggler("anticomp") ), select.Option( "Kill Extra", "x", - lambda: master.options.replay_kill_extra, + checker("replay_kill_extra", master.options), master.options.toggler("replay_kill_extra") ), select.Option( "No Refresh", "f", - lambda: not master.options.refresh_server_playback, + checker("refresh_server_playback", master.options), master.options.toggler("refresh_server_playback") ), select.Option( "Sticky Auth", "A", - lambda: master.options.stickyauth, + checker("stickyauth", master.options), self.sticky_auth ), select.Option( "Sticky Cookies", "t", - lambda: master.options.stickycookie, + checker("stickycookie", master.options), self.sticky_cookie ), ] @@ -163,7 +168,7 @@ class Options(urwid.WidgetWrap): self.master.options.reset() signals.update_settings.send(self) signals.status_message.send( - message = "All select.Options cleared", + message = "Options cleared", expire = 1 ) diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 47e40d98..345512fd 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -35,6 +35,7 @@ def test_defaults(): "three": "dthree", "four": "dfour", } + assert not o.has_changed("one") newvals = dict( one="xone", two="xtwo", @@ -42,11 +43,13 @@ def test_defaults(): four="xfour", ) o.update(**newvals) + assert o.has_changed("one") for k, v in newvals.items(): - assert v == o.get(k) + assert v == getattr(o, k) o.reset() + assert not o.has_changed("one") for k, v in o._defaults.items(): - assert v == o.get(k) + assert v == getattr(o, k) def test_options(): -- cgit v1.2.3 From c062e302e9281e4d8e216595b14fb089ed784b4e Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 3 Dec 2016 11:54:04 +1300 Subject: Add OptManager.subscribe, use it to clean up palettes in console .subscribe lets you subscribe a function to a specified set of options. --- mitmproxy/optmanager.py | 23 +++++++++++++++++++++++ mitmproxy/tools/console/master.py | 16 +++++++++------- mitmproxy/tools/console/palettepicker.py | 21 ++++++--------------- test/mitmproxy/test_optmanager.py | 23 +++++++++++++++++++++++ 4 files changed, 61 insertions(+), 22 deletions(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 56122baa..3ce76a45 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -3,6 +3,8 @@ import blinker import pprint import inspect import copy +import functools +import weakref from mitmproxy import exceptions from mitmproxy.utils import typecheck @@ -63,6 +65,27 @@ class OptManager: self.__dict__["_opts"] = old self.changed.send(self, updated=updated) + def subscribe(self, func, opts): + """ + Subscribe a callable to the .changed signal, but only for a + specified list of options. The callable should accept arguments + (options, updated), and may raise an OptionsError. + """ + func = weakref.proxy(func) + + @functools.wraps(func) + def _call(options, updated): + if updated.intersection(set(opts)): + try: + func(options, updated) + except ReferenceError: + self.changed.disconnect(_call) + + # Our wrapper function goes out of scope immediately, so we have to set + # weakrefs to false. This means we need to keep our own weakref, and + # clean up the hook when it's gone. + self.changed.connect(_call, weak=False) + def __eq__(self, other): return self._opts == other._opts diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 455824d3..27f5cb58 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -77,9 +77,6 @@ class ConsoleMaster(master.Master): self.options = self.options # type: Options self.options.errored.connect(self.options_error) - self.palette = options.palette - self.palette_transparent = options.palette_transparent - self.logbuffer = urwid.SimpleListWalker([]) self.view_stack = [] @@ -253,10 +250,11 @@ class ConsoleMaster(master.Master): self.ui.start() os.unlink(name) - def set_palette(self, name): - self.palette = name + def set_palette(self, options, updated): self.ui.register_palette( - palettes.palettes[name].palette(self.palette_transparent) + palettes.palettes[options.palette].palette( + options.palette_transparent + ) ) self.ui.clear() @@ -269,7 +267,11 @@ class ConsoleMaster(master.Master): def run(self): self.ui = urwid.raw_display.Screen() self.ui.set_terminal_properties(256) - self.set_palette(self.palette) + self.set_palette(self.options, None) + self.options.subscribe( + self.set_palette, + ["palette", "palette_transparent"] + ) self.loop = urwid.MainLoop( urwid.SolidFill("x"), screen = self.ui, diff --git a/mitmproxy/tools/console/palettepicker.py b/mitmproxy/tools/console/palettepicker.py index a3eb9b90..0d943baf 100644 --- a/mitmproxy/tools/console/palettepicker.py +++ b/mitmproxy/tools/console/palettepicker.py @@ -3,7 +3,6 @@ import urwid from mitmproxy.tools.console import common from mitmproxy.tools.console import palettes from mitmproxy.tools.console import select -from mitmproxy.tools.console import signals footer = [ ('heading_key', "enter/space"), ":select", @@ -43,8 +42,8 @@ class PalettePicker(urwid.WidgetWrap): return select.Option( i, None, - lambda: self.master.palette == name, - lambda: self.select(name) + lambda: self.master.options.palette == name, + lambda: setattr(self.master.options, "palette", name) ) for i in high: @@ -59,8 +58,8 @@ class PalettePicker(urwid.WidgetWrap): select.Option( "Transparent", "T", - lambda: master.palette_transparent, - self.toggle_palette_transparent + lambda: master.options.palette_transparent, + master.options.toggler("palette_transparent") ) ] ) @@ -73,15 +72,7 @@ class PalettePicker(urwid.WidgetWrap): self.lb, header = title ) - signals.update_settings.connect(self.sig_update_settings) + master.options.changed.connect(self.sig_options_changed) - def sig_update_settings(self, sender): + def sig_options_changed(self, options, updated): self.lb.walker._modified() - - def select(self, name): - self.master.set_palette(name) - - def toggle_palette_transparent(self): - self.master.palette_transparent = not self.master.palette_transparent - self.master.set_palette(self.master.palette) - signals.update_settings.send(self) diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 345512fd..c7808391 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -104,6 +104,29 @@ def test_toggler(): o.toggler("nonexistent") +class Rec(): + def __init__(self): + self.called = None + + def __call__(self, *args, **kwargs): + self.called = (args, kwargs) + + +def test_subscribe(): + o = TO() + r = Rec() + o.subscribe(r, ["two"]) + o.one = "foo" + assert not r.called + o.two = "foo" + assert r.called + + assert len(o.changed.receivers) == 1 + del r + o.two = "bar" + assert len(o.changed.receivers) == 0 + + def test_rollback(): o = TO(one="two") -- cgit v1.2.3 From 3594faf5c42bfe4d422b16e4ceedb482802c95ce Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 3 Dec 2016 11:56:50 +1300 Subject: Correct refresh_server_playback option default --- mitmproxy/options.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index bf35cd59..06ea30ee 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -35,7 +35,7 @@ class Options(optmanager.OptManager): keepserving: bool = True, no_server: bool = False, server_replay_nopop: bool = False, - refresh_server_playback: bool = False, + refresh_server_playback: bool = True, rfile: Optional[str] = None, scripts: Sequence[str] = [], showhost: bool = False, -- cgit v1.2.3 From c94cd512d108fd9a12c87015791ce977b895fa2d Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 3 Dec 2016 12:16:50 +1300 Subject: options: defaults are a class attribute --- mitmproxy/optmanager.py | 28 ++++++++++++++++++---------- test/mitmproxy/test_optmanager.py | 5 +++++ 2 files changed, 23 insertions(+), 10 deletions(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 3ce76a45..304f5129 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -14,7 +14,20 @@ from mitmproxy.utils import typecheck """ -class OptManager: +class _DefaultsMeta(type): + def __new__(cls, name, bases, namespace, **kwds): + ret = type.__new__(cls, name, bases, dict(namespace)) + defaults = {} + for klass in reversed(inspect.getmro(ret)): + for p in inspect.signature(klass.__init__).parameters.values(): + if p.kind in (p.KEYWORD_ONLY, p.POSITIONAL_OR_KEYWORD): + if not p.default == p.empty: + defaults[p.name] = p.default + ret._defaults = defaults + return ret + + +class OptManager(metaclass=_DefaultsMeta): """ OptManager is the base class from which Options objects are derived. Note that the __init__ method of all child classes must force all @@ -37,15 +50,6 @@ class OptManager: # ._initialized = True as the final operation. instance = super().__new__(cls) instance.__dict__["_opts"] = {} - - defaults = {} - for klass in reversed(inspect.getmro(cls)): - for p in inspect.signature(klass.__init__).parameters.values(): - if p.kind in (p.KEYWORD_ONLY, p.POSITIONAL_OR_KEYWORD): - if not p.default == p.empty: - defaults[p.name] = p.default - instance.__dict__["_defaults"] = defaults - return instance def __init__(self): @@ -122,6 +126,10 @@ class OptManager: """ self.update(**self._defaults) + @classmethod + def default(klass, opt): + return copy.deepcopy(klass._defaults[opt]) + def update(self, **kwargs): updated = set(kwargs.keys()) for k, v in kwargs.items(): diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index c7808391..385cf621 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -28,6 +28,11 @@ class TD2(TD): def test_defaults(): + assert TD2.default("one") == "done" + assert TD2.default("two") == "dtwo" + assert TD2.default("three") == "dthree" + assert TD2.default("four") == "dfour" + o = TD2() assert o._defaults == { "one": "done", -- cgit v1.2.3 From d74cac265a9d1d8ce176a7ef96be2d91c4f40819 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 5 Dec 2016 07:18:53 +1300 Subject: Add YAML serialization of options This uses ruamel.yaml. The library seems well-supported, and can do in-place modification of config files that retains user comments and file structure. --- mitmproxy/optmanager.py | 70 +++++++++++++++++++++++++++++++++++++++ setup.py | 1 + test/mitmproxy/test_optmanager.py | 55 +++++++++++++++++++++++++++++- 3 files changed, 125 insertions(+), 1 deletion(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 304f5129..cbf656f5 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -5,6 +5,9 @@ import inspect import copy import functools import weakref +import os + +import ruamel.yaml from mitmproxy import exceptions from mitmproxy.utils import typecheck @@ -171,6 +174,73 @@ class OptManager(metaclass=_DefaultsMeta): if getattr(self, option) != self._defaults[option]: return True + def save(self, path, defaults=False): + """ + Save to path. If the destination file exists, modify it in-place. + """ + if os.path.exists(path) and os.path.isfile(path): + data = open(path, "r").read() + else: + data = "" + data = self.serialize(data, defaults) + fp = open(path, "w") + fp.write(data) + + def serialize(self, text, defaults=False): + """ + Performs a round-trip serialization. If text is not None, it is + treated as a previous serialization that should be modified + in-place. + + - If "defaults" is False, only options with non-default values are + serialized. Default values in text are preserved. + - Unknown options in text are removed. + - Raises OptionsError if text is invalid. + """ + data = self._load(text) + for k in self.keys(): + if defaults or self.has_changed(k): + data[k] = getattr(self, k) + for k in list(data.keys()): + if k not in self._opts: + del data[k] + return ruamel.yaml.round_trip_dump(data) + + def _load(self, text): + if not text: + return {} + try: + data = ruamel.yaml.load(text, ruamel.yaml.Loader) + except ruamel.yaml.error.YAMLError as v: + snip = v.problem_mark.get_snippet() + raise exceptions.OptionsError( + "Config error at line %s:\n%s\n%s" % + (v.problem_mark.line+1, snip, v.problem) + ) + if isinstance(data, str): + raise exceptions.OptionsError("Config error - no keys found.") + return data + + def load(self, text): + """ + Load configuration from text, over-writing options already set in + this object. May raise OptionsError if the config file is invalid. + """ + data = self._load(text) + for k, v in data.items(): + setattr(self, k, v) + + def load_paths(self, *paths): + """ + Load paths in order. Each path takes precedence over the previous + path. Paths that don't exist are ignored, errors raise an + OptionsError. + """ + for p in paths: + if os.path.exists(p) and os.path.isfile(p): + txt = open(p, "r").read() + self.load(txt) + def __repr__(self): options = pprint.pformat(self._opts, indent=4).strip(" {}") if "\n" in options: diff --git a/setup.py b/setup.py index 56ba46fc..35f7edb3 100644 --- a/setup.py +++ b/setup.py @@ -79,6 +79,7 @@ setup( "pyparsing>=2.1.3, <2.2", "pyperclip>=1.5.22, <1.6", "requests>=2.9.1, <3", + "ruamel.yaml>=0.13.2, <0.14", "tornado>=4.3, <4.5", "urwid>=1.3.1, <1.4", "watchdog>=0.8.3, <0.9", diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 385cf621..0c98daea 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -1,5 +1,7 @@ import copy +import os +from mitmproxy import options from mitmproxy import optmanager from mitmproxy import exceptions from mitmproxy.test import tutils @@ -24,7 +26,7 @@ class TD2(TD): def __init__(self, *, three="dthree", four="dfour", **kwargs): self.three = three self.four = four - super().__init__(**kwargs) + super().__init__(three=three, **kwargs) def test_defaults(): @@ -167,3 +169,54 @@ def test_repr(): 'one': 'xxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxxx', 'two': None })""" + + +def test_serialize(): + o = TD2() + o.three = "set" + assert "dfour" in o.serialize(None, defaults=True) + + data = o.serialize(None) + assert "dfour" not in data + + o2 = TD2() + o2.load(data) + assert o2 == o + + t = """ + unknown: foo + """ + data = o.serialize(t) + o2 = TD2() + o2.load(data) + assert o2 == o + + t = "invalid: foo\ninvalid" + tutils.raises("config error", o2.load, t) + + t = "invalid" + tutils.raises("config error", o2.load, t) + + t = "" + o2.load(t) + + +def test_serialize_defaults(): + o = options.Options() + assert o.serialize(None, defaults=True) + + +def test_saving(): + o = TD2() + o.three = "set" + with tutils.tmpdir() as tdir: + dst = os.path.join(tdir, "conf") + o.save(dst, defaults=True) + + o2 = TD2() + o2.load_paths(dst) + o2.three = "foo" + o2.save(dst, defaults=True) + + o.load_paths(dst) + assert o.three == "foo" -- cgit v1.2.3 From 00c897a185b4bba7116fa22d14214dff68ce6e69 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 2 Dec 2016 16:15:14 +1300 Subject: options: save defaults, add .reset() to restore defaults Use .reset() in console app to clear options. --- mitmproxy/optmanager.py | 19 ++++++++++++++----- test/mitmproxy/test_optmanager.py | 5 ----- 2 files changed, 14 insertions(+), 10 deletions(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index cbf656f5..713a0b62 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -53,6 +53,15 @@ class OptManager(metaclass=_DefaultsMeta): # ._initialized = True as the final operation. instance = super().__new__(cls) instance.__dict__["_opts"] = {} + + defaults = {} + for klass in reversed(inspect.getmro(cls)): + for p in inspect.signature(klass.__init__).parameters.values(): + if p.kind in (p.KEYWORD_ONLY, p.POSITIONAL_OR_KEYWORD): + if not p.default == p.empty: + defaults[p.name] = p.default + instance.__dict__["_defaults"] = defaults + return instance def __init__(self): @@ -123,16 +132,16 @@ class OptManager(metaclass=_DefaultsMeta): def keys(self): return set(self._opts.keys()) + @classmethod + def default(klass, opt): + return copy.deepcopy(klass._defaults[opt]) + def reset(self): """ Restore defaults for all options. """ self.update(**self._defaults) - @classmethod - def default(klass, opt): - return copy.deepcopy(klass._defaults[opt]) - def update(self, **kwargs): updated = set(kwargs.keys()) for k, v in kwargs.items(): @@ -215,7 +224,7 @@ class OptManager(metaclass=_DefaultsMeta): snip = v.problem_mark.get_snippet() raise exceptions.OptionsError( "Config error at line %s:\n%s\n%s" % - (v.problem_mark.line+1, snip, v.problem) + (v.problem_mark.line + 1, snip, v.problem) ) if isinstance(data, str): raise exceptions.OptionsError("Config error - no keys found.") diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 0c98daea..97124368 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -30,11 +30,6 @@ class TD2(TD): def test_defaults(): - assert TD2.default("one") == "done" - assert TD2.default("two") == "dtwo" - assert TD2.default("three") == "dthree" - assert TD2.default("four") == "dfour" - o = TD2() assert o._defaults == { "one": "done", -- cgit v1.2.3 From 2cb1f7038196d31ecdd4190c009dc63a1f6d0538 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 3 Dec 2016 09:42:14 +1300 Subject: Options - avoid mutation, API cleanup, has_changed - Always return a deepcopy of options to avoid accidental mutation of options state. - Remove .get(opt, default). This is an inappropriate API for Options - trying to retrieve an option that doesn't exist should always be an error. - Add the has_changed method that checks if an option differs from the default, use it in mitmproxy console. --- mitmproxy/optmanager.py | 1 + 1 file changed, 1 insertion(+) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 713a0b62..fd777d74 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -12,6 +12,7 @@ import ruamel.yaml from mitmproxy import exceptions from mitmproxy.utils import typecheck + """ The base implementation for Options. """ -- cgit v1.2.3 From 150372e29728459b93f0fb46028ced9c2afe1d15 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 3 Dec 2016 12:16:50 +1300 Subject: options: defaults are a class attribute --- mitmproxy/optmanager.py | 13 ++++--------- test/mitmproxy/test_optmanager.py | 5 +++++ 2 files changed, 9 insertions(+), 9 deletions(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index fd777d74..61421ba7 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -54,15 +54,6 @@ class OptManager(metaclass=_DefaultsMeta): # ._initialized = True as the final operation. instance = super().__new__(cls) instance.__dict__["_opts"] = {} - - defaults = {} - for klass in reversed(inspect.getmro(cls)): - for p in inspect.signature(klass.__init__).parameters.values(): - if p.kind in (p.KEYWORD_ONLY, p.POSITIONAL_OR_KEYWORD): - if not p.default == p.empty: - defaults[p.name] = p.default - instance.__dict__["_defaults"] = defaults - return instance def __init__(self): @@ -143,6 +134,10 @@ class OptManager(metaclass=_DefaultsMeta): """ self.update(**self._defaults) + @classmethod + def default(klass, opt): + return copy.deepcopy(klass._defaults[opt]) + def update(self, **kwargs): updated = set(kwargs.keys()) for k, v in kwargs.items(): diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 97124368..0c98daea 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -30,6 +30,11 @@ class TD2(TD): def test_defaults(): + assert TD2.default("one") == "done" + assert TD2.default("two") == "dtwo" + assert TD2.default("three") == "dthree" + assert TD2.default("four") == "dfour" + o = TD2() assert o._defaults == { "one": "done", -- cgit v1.2.3 From 6aacd27ab26e6831ad6f9cfe0b1f555617321a25 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 5 Dec 2016 07:18:53 +1300 Subject: Add YAML serialization of options This uses ruamel.yaml. The library seems well-supported, and can do in-place modification of config files that retains user comments and file structure. --- mitmproxy/optmanager.py | 4 ---- 1 file changed, 4 deletions(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 61421ba7..2ce3c48b 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -124,10 +124,6 @@ class OptManager(metaclass=_DefaultsMeta): def keys(self): return set(self._opts.keys()) - @classmethod - def default(klass, opt): - return copy.deepcopy(klass._defaults[opt]) - def reset(self): """ Restore defaults for all options. -- cgit v1.2.3 From d742d4fb8c8d601c19f7aa5ff746a4167d864f7e Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 5 Dec 2016 15:55:43 +1300 Subject: Replace configargparse with argparse, enable new options loading This commit ditches configargparse and enables the new config file format. The default location is ~/.mitmproxy/config.yaml. Unifying all of the various Options objects will follow in the next patch. --- .gitignore | 2 +- mitmproxy/optmanager.py | 1 + mitmproxy/platform/windows.py | 7 +- mitmproxy/tools/cmdline.py | 168 ++++++++++++++++++------------------------ mitmproxy/tools/main.py | 66 +++++++++++------ setup.py | 1 - test/mitmproxy/test_proxy.py | 3 +- 7 files changed, 126 insertions(+), 122 deletions(-) diff --git a/.gitignore b/.gitignore index e942ea94..91caf620 100644 --- a/.gitignore +++ b/.gitignore @@ -1,6 +1,6 @@ .DS_Store MANIFEST -*/tmp +**/tmp /venv* *.py[cdo] *.swp diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 2ce3c48b..081836ce 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -238,6 +238,7 @@ class OptManager(metaclass=_DefaultsMeta): OptionsError. """ for p in paths: + p = os.path.expanduser(p) if os.path.exists(p) and os.path.isfile(p): txt = open(p, "r").read() self.load(txt) diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index a59fe25f..1c90a7a0 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -7,7 +7,7 @@ import struct import threading import time -import configargparse +import argparse import pydivert import pydivert.consts import pickle @@ -386,8 +386,9 @@ class TransparentProxy: if __name__ == "__main__": - parser = configargparse.ArgumentParser( - description="Windows Transparent Proxy") + parser = argparse.ArgumentParser( + description="Windows Transparent Proxy" + ) parser.add_argument( '--mode', choices=[ diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index f8246199..1cdde727 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -1,6 +1,7 @@ -import configargparse -import os +import argparse import re +import os + from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import options @@ -11,6 +12,9 @@ from mitmproxy import version from mitmproxy.addons import view +CONFIG_PATH = os.path.join(options.CA_DIR, "config.yaml") + + class ParseException(Exception): pass @@ -113,13 +117,13 @@ def get_common_options(args): stream_large_bodies = human.parse_size(stream_large_bodies) reps = [] - for i in args.replace: + for i in args.replace or []: try: p = parse_replace_hook(i) except ParseException as e: raise exceptions.OptionsError(e) reps.append(p) - for i in args.replace_file: + for i in args.replace_file or []: try: patt, rex, path = parse_replace_hook(i) except ParseException as e: @@ -133,7 +137,7 @@ def get_common_options(args): reps.append((patt, rex, v)) setheaders = [] - for i in args.setheader: + for i in args.setheader or []: try: p = parse_setheader(i) except ParseException as e: @@ -154,7 +158,7 @@ def get_common_options(args): # Proxy config certs = [] - for i in args.certs: + for i in args.certs or []: parts = i.split("=", 1) if len(parts) == 1: parts = ["*", parts[0]] @@ -287,8 +291,7 @@ def basic_options(parser): ) parser.add_argument( "--anticache", - action="store_true", dest="anticache", default=False, - + action="store_true", dest="anticache", help=""" Strip out request headers that might cause the server to return 304-not-modified. @@ -296,12 +299,12 @@ def basic_options(parser): ) parser.add_argument( "--cadir", - action="store", type=str, dest="cadir", default=options.CA_DIR, + action="store", type=str, dest="cadir", help="Location of the default mitmproxy CA files. (%s)" % options.CA_DIR ) parser.add_argument( "--host", - action="store_true", dest="showhost", default=False, + action="store_true", dest="showhost", help="Use the Host header to construct URLs for display." ) parser.add_argument( @@ -311,12 +314,12 @@ def basic_options(parser): ) parser.add_argument( "-r", "--read-flows", - action="store", dest="rfile", default=None, + action="store", dest="rfile", help="Read flows from file." ) parser.add_argument( "-s", "--script", - action="append", type=str, dest="scripts", default=[], + action="append", type=str, dest="scripts", metavar='"script.py --bar"', help=""" Run a script. Surround with quotes to pass script arguments. Can be @@ -327,18 +330,17 @@ def basic_options(parser): "-t", "--stickycookie", action="store", dest="stickycookie_filt", - default=None, metavar="FILTER", help="Set sticky cookie filter. Matched against requests." ) parser.add_argument( "-u", "--stickyauth", - action="store", dest="stickyauth_filt", default=None, metavar="FILTER", + action="store", dest="stickyauth_filt", metavar="FILTER", help="Set sticky auth filter. Matched against requests." ) parser.add_argument( "-v", "--verbose", - action="store_const", dest="verbose", default=2, const=3, + action="store_const", dest="verbose", const=3, help="Increase log verbosity." ) streamfile = parser.add_mutually_exclusive_group() @@ -354,19 +356,19 @@ def basic_options(parser): ) parser.add_argument( "-z", "--anticomp", - action="store_true", dest="anticomp", default=False, + action="store_true", dest="anticomp", help="Try to convince servers to send us un-compressed data." ) parser.add_argument( "-Z", "--body-size-limit", - action="store", dest="body_size_limit", default=None, + action="store", dest="body_size_limit", metavar="SIZE", help="Byte size limit of HTTP request and response bodies." " Understands k/m/g suffixes, i.e. 3m for 3 megabytes." ) parser.add_argument( "--stream", - action="store", dest="stream_large_bodies", default=None, + action="store", dest="stream_large_bodies", metavar="SIZE", help=""" Stream data to the client if response body exceeds the given @@ -383,7 +385,6 @@ def proxy_modes(parser): action="store", type=str, dest="reverse_proxy", - default=None, help=""" Forward all requests to upstream HTTP server: http[s]://host[:port]. Clients can always connect both @@ -393,12 +394,12 @@ def proxy_modes(parser): ) group.add_argument( "--socks", - action="store_true", dest="socks_proxy", default=False, + action="store_true", dest="socks_proxy", help="Set SOCKS5 proxy mode." ) group.add_argument( "-T", "--transparent", - action="store_true", dest="transparent_proxy", default=False, + action="store_true", dest="transparent_proxy", help="Set transparent proxy mode." ) group.add_argument( @@ -406,7 +407,6 @@ def proxy_modes(parser): action="store", type=str, dest="upstream_proxy", - default=None, help="Forward all requests to upstream proxy server: http://host[:port]" ) @@ -415,12 +415,12 @@ def proxy_options(parser): group = parser.add_argument_group("Proxy Options") group.add_argument( "-b", "--bind-address", - action="store", type=str, dest="addr", default='', + action="store", type=str, dest="addr", help="Address to bind proxy to (defaults to all interfaces)" ) group.add_argument( "-I", "--ignore", - action="append", type=str, dest="ignore_hosts", default=[], + action="append", type=str, dest="ignore_hosts", metavar="HOST", help=""" Ignore host and forward all traffic without processing it. In @@ -433,7 +433,7 @@ def proxy_options(parser): ) group.add_argument( "--tcp", - action="append", type=str, dest="tcp_hosts", default=[], + action="append", type=str, dest="tcp_hosts", metavar="HOST", help=""" Generic TCP SSL proxy mode for all hosts that match the pattern. @@ -448,7 +448,7 @@ def proxy_options(parser): ) group.add_argument( "-p", "--port", - action="store", type=int, dest="port", default=options.LISTEN_PORT, + action="store", type=int, dest="port", help="Proxy service port." ) group.add_argument( @@ -467,7 +467,7 @@ def proxy_options(parser): parser.add_argument( "--upstream-auth", - action="store", dest="upstream_auth", default=None, + action="store", dest="upstream_auth", type=str, help=""" Add HTTP Basic authentcation to upstream proxy and reverse proxy @@ -491,7 +491,7 @@ def proxy_options(parser): ) group.add_argument( "--upstream-bind-address", - action="store", type=str, dest="upstream_bind_address", default='', + action="store", type=str, dest="upstream_bind_address", help="Address to bind upstream requests to (defaults to none)" ) @@ -502,7 +502,6 @@ def proxy_ssl_options(parser): group.add_argument( "--cert", dest='certs', - default=[], type=str, metavar="SPEC", action="append", @@ -514,56 +513,55 @@ def proxy_ssl_options(parser): 'as the first entry. Can be passed multiple times.') group.add_argument( "--ciphers-client", action="store", - type=str, dest="ciphers_client", default=options.DEFAULT_CLIENT_CIPHERS, help="Set supported ciphers for client connections. (OpenSSL Syntax)" ) group.add_argument( "--ciphers-server", action="store", - type=str, dest="ciphers_server", default=None, + type=str, dest="ciphers_server", help="Set supported ciphers for server connections. (OpenSSL Syntax)" ) group.add_argument( "--client-certs", action="store", - type=str, dest="clientcerts", default=None, + type=str, dest="clientcerts", help="Client certificate file or directory." ) group.add_argument( - "--no-upstream-cert", default=False, + "--no-upstream-cert", action="store_true", dest="no_upstream_cert", help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--add-upstream-certs-to-client-chain", default=False, + "--add-upstream-certs-to-client-chain", action="store_true", dest="add_upstream_certs_to_client_chain", help="Add all certificates of the upstream server to the certificate chain " "that will be served to the proxy client, as extras." ) group.add_argument( - "--insecure", default=False, + "--insecure", action="store_true", dest="ssl_insecure", help="Do not verify upstream server SSL/TLS certificates." ) group.add_argument( - "--upstream-trusted-cadir", default=None, action="store", + "--upstream-trusted-cadir", action="store", dest="ssl_verify_upstream_trusted_cadir", help="Path to a directory of trusted CA certificates for upstream " "server verification prepared using the c_rehash tool." ) group.add_argument( - "--upstream-trusted-ca", default=None, action="store", + "--upstream-trusted-ca", action="store", dest="ssl_verify_upstream_trusted_ca", help="Path to a PEM formatted trusted CA certificate." ) group.add_argument( "--ssl-version-client", dest="ssl_version_client", - default="secure", action="store", + action="store", choices=tcp.sslversion_choices.keys(), help="Set supported SSL/TLS versions for client connections. " "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." ) group.add_argument( "--ssl-version-server", dest="ssl_version_server", - default="secure", action="store", + action="store", choices=tcp.sslversion_choices.keys(), help="Set supported SSL/TLS versions for server connections. " "SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+." @@ -574,12 +572,12 @@ def onboarding_app(parser): group = parser.add_argument_group("Onboarding App") group.add_argument( "--noapp", - action="store_false", dest="app", default=True, + action="store_false", dest="app", help="Disable the mitmproxy onboarding app." ) group.add_argument( "--app-host", - action="store", dest="app_host", default=options.APP_HOST, metavar="host", + action="store", dest="app_host", help=""" Domain to serve the onboarding app from. For transparent mode, use an IP when a DNS entry for the app domain is not present. Default: @@ -590,7 +588,6 @@ def onboarding_app(parser): "--app-port", action="store", dest="app_port", - default=options.APP_PORT, type=int, metavar="80", help="Port to serve the onboarding app from." @@ -601,7 +598,7 @@ def client_replay(parser): group = parser.add_argument_group("Client Replay") group.add_argument( "-c", "--client-replay", - action="append", dest="client_replay", default=[], metavar="PATH", + action="append", dest="client_replay", metavar="PATH", help="Replay client requests from a saved file." ) @@ -610,23 +607,23 @@ def server_replay(parser): group = parser.add_argument_group("Server Replay") group.add_argument( "-S", "--server-replay", - action="append", dest="server_replay", default=[], metavar="PATH", + action="append", dest="server_replay", metavar="PATH", help="Replay server responses from a saved file." ) group.add_argument( "-k", "--replay-kill-extra", - action="store_true", dest="replay_kill_extra", default=False, + action="store_true", dest="replay_kill_extra", help="Kill extra requests during replay." ) group.add_argument( "--server-replay-use-header", - action="append", dest="server_replay_use_headers", type=str, default=[], + action="append", dest="server_replay_use_headers", type=str, help="Request headers to be considered during replay. " "Can be passed multiple times." ) group.add_argument( "--norefresh", - action="store_true", dest="norefresh", default=False, + action="store_true", dest="norefresh", help=""" Disable response refresh, which updates times in cookies and headers for replayed responses. @@ -634,21 +631,21 @@ def server_replay(parser): ) group.add_argument( "--no-pop", - action="store_true", dest="server_replay_nopop", default=False, + action="store_true", dest="server_replay_nopop", help="Disable response pop from response flow. " "This makes it possible to replay same response multiple times." ) payload = group.add_mutually_exclusive_group() payload.add_argument( "--replay-ignore-content", - action="store_true", dest="server_replay_ignore_content", default=False, + action="store_true", dest="server_replay_ignore_content", help=""" Ignore request's content while searching for a saved flow to replay """ ) payload.add_argument( "--replay-ignore-payload-param", - action="append", dest="server_replay_ignore_payload_params", type=str, default=[], + action="append", dest="server_replay_ignore_payload_params", type=str, help=""" Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to be ignored while searching for a saved flow to replay. @@ -658,7 +655,7 @@ def server_replay(parser): group.add_argument( "--replay-ignore-param", - action="append", dest="server_replay_ignore_params", type=str, default=[], + action="append", dest="server_replay_ignore_params", type=str, help=""" Request's parameters to be ignored while searching for a saved flow to replay. Can be passed multiple times. @@ -668,7 +665,6 @@ def server_replay(parser): "--replay-ignore-host", action="store_true", dest="server_replay_ignore_host", - default=False, help="Ignore request's destination host while searching for a saved flow to replay") @@ -683,13 +679,13 @@ def replacements(parser): ) group.add_argument( "--replace", - action="append", type=str, dest="replace", default=[], + action="append", type=str, dest="replace", metavar="PATTERN", help="Replacement pattern." ) group.add_argument( "--replace-from-file", - action="append", type=str, dest="replace_file", default=[], + action="append", type=str, dest="replace_file", metavar="PATH", help=""" Replacement pattern, where the replacement clause is a path to a @@ -709,7 +705,7 @@ def set_headers(parser): ) group.add_argument( "--setheader", - action="append", type=str, dest="setheader", default=[], + action="append", type=str, dest="setheader", metavar="PATTERN", help="Header set pattern." ) @@ -747,6 +743,15 @@ def proxy_authentication(parser): def common_options(parser): + parser.add_argument( + "--conf", + type=str, dest="conf", default=CONFIG_PATH, + metavar="PATH", + help=""" + Configuration file + """ + ) + basic_options(parser) proxy_modes(parser) proxy_options(parser) @@ -764,26 +769,17 @@ def mitmproxy(): # platforms. from .console import palettes - parser = configargparse.ArgumentParser( - usage="%(prog)s [options]", - args_for_setting_config_path=["--conf"], - default_config_files=[ - os.path.join(options.CA_DIR, "common.conf"), - os.path.join(options.CA_DIR, "mitmproxy.conf") - ], - add_config_file_help=True, - add_env_var_help=True - ) + parser = argparse.ArgumentParser(usage="%(prog)s [options]") common_options(parser) parser.add_argument( - "--palette", type=str, default=palettes.DEFAULT, + "--palette", type=str, action="store", dest="palette", choices=sorted(palettes.palettes.keys()), help="Select color palette: " + ", ".join(palettes.palettes.keys()) ) parser.add_argument( "--palette-transparent", - action="store_true", dest="palette_transparent", default=False, + action="store_true", dest="palette_transparent", help="Set transparent background for palette." ) parser.add_argument( @@ -798,7 +794,7 @@ def mitmproxy(): ) parser.add_argument( "--order", - type=str, dest="order", default=None, + type=str, dest="order", choices=[o[1] for o in view.orders], help="Flow sort order." ) @@ -813,33 +809,24 @@ def mitmproxy(): ) group.add_argument( "-i", "--intercept", action="store", - type=str, dest="intercept", default=None, + type=str, dest="intercept", help="Intercept filter expression." ) group.add_argument( "-f", "--filter", action="store", - type=str, dest="filter", default=None, + type=str, dest="filter", help="Filter view expression." ) return parser def mitmdump(): - parser = configargparse.ArgumentParser( - usage="%(prog)s [options] [filter]", - args_for_setting_config_path=["--conf"], - default_config_files=[ - os.path.join(options.CA_DIR, "common.conf"), - os.path.join(options.CA_DIR, "mitmdump.conf") - ], - add_config_file_help=True, - add_env_var_help=True - ) + parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]") common_options(parser) parser.add_argument( "--keepserving", - action="store_true", dest="keepserving", default=False, + action="store_true", dest="keepserving", help=""" Continue serving after client playback or file read. We exit by default. @@ -847,7 +834,7 @@ def mitmdump(): ) parser.add_argument( "-d", "--detail", - action="count", dest="flow_detail", default=1, + action="count", dest="flow_detail", help="Increase flow detail display level. Can be passed multiple times." ) parser.add_argument( @@ -862,16 +849,7 @@ def mitmdump(): def mitmweb(): - parser = configargparse.ArgumentParser( - usage="%(prog)s [options]", - args_for_setting_config_path=["--conf"], - default_config_files=[ - os.path.join(options.CA_DIR, "common.conf"), - os.path.join(options.CA_DIR, "mitmweb.conf") - ], - add_config_file_help=True, - add_env_var_help=True - ) + parser = argparse.ArgumentParser(usage="%(prog)s [options]") group = parser.add_argument_group("Mitmweb") group.add_argument( @@ -881,13 +859,13 @@ def mitmweb(): ) group.add_argument( "--wport", - action="store", type=int, dest="wport", default=8081, + action="store", type=int, dest="wport", metavar="PORT", help="Mitmweb port." ) group.add_argument( "--wiface", - action="store", dest="wiface", default="127.0.0.1", + action="store", dest="wiface", metavar="IFACE", help="Mitmweb interface." ) @@ -904,7 +882,7 @@ def mitmweb(): ) group.add_argument( "-i", "--intercept", action="store", - type=str, dest="intercept", default=None, + type=str, dest="intercept", help="Intercept filter expression." ) return parser diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index c3b1e3a9..a6b75db1 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -17,6 +17,14 @@ from mitmproxy.utils import version_check # noqa from mitmproxy.utils import debug # noqa +def notnone(d): + ret = {} + for k, v in d.items(): + if v is not None: + ret[k] = v + return ret + + def assert_utf8_env(): spec = "" for i in ["LANG", "LC_CTYPE", "LC_ALL"]: @@ -63,17 +71,21 @@ def mitmproxy(args=None): # pragma: no cover args = parser.parse_args(args) try: - console_options = console.master.Options( - **cmdline.get_common_options(args) + console_options = console.master.Options() + console_options.load_paths(args.conf) + console_options.update(**notnone(cmdline.get_common_options(args))) + console_options.update( + **notnone(dict( + palette = args.palette, + palette_transparent = args.palette_transparent, + eventlog = args.eventlog, + focus_follow = args.focus_follow, + intercept = args.intercept, + filter = args.filter, + no_mouse = args.no_mouse, + order = args.order, + )) ) - console_options.palette = args.palette - console_options.palette_transparent = args.palette_transparent - console_options.eventlog = args.eventlog - console_options.focus_follow = args.focus_follow - console_options.intercept = args.intercept - console_options.filter = args.filter - console_options.no_mouse = args.no_mouse - console_options.order = args.order server = process_options(parser, console_options, args) m = console.master.ConsoleMaster(console_options, server) @@ -98,10 +110,17 @@ def mitmdump(args=None): # pragma: no cover master = None try: - dump_options = dump.Options(**cmdline.get_common_options(args)) - dump_options.flow_detail = args.flow_detail - dump_options.keepserving = args.keepserving - dump_options.filtstr = " ".join(args.filter) if args.filter else None + dump_options = dump.Options() + dump_options.load_paths(args.conf) + dump_options.update(**notnone(cmdline.get_common_options(args))) + dump_options.update( + **notnone(dict( + flow_detail = args.flow_detail, + keepserving = args.keepserving, + filtstr = " ".join(args.filter) if args.filter else None, + )) + ) + server = process_options(parser, dump_options, args) master = dump.DumpMaster(dump_options, server) @@ -130,13 +149,18 @@ def mitmweb(args=None): # pragma: no cover args = parser.parse_args(args) try: - web_options = web.master.Options(**cmdline.get_common_options(args)) - web_options.intercept = args.intercept - web_options.open_browser = args.open_browser - web_options.wdebug = args.wdebug - web_options.wiface = args.wiface - web_options.wport = args.wport - + web_options = web.master.Options() + web_options.load_paths(args.conf) + web_options.update(**notnone(cmdline.get_common_options(args))) + web_options.update( + **notnone(dict( + intercept = args.intercept, + open_browser = args.open_browser, + wdebug = args.wdebug, + wiface = args.wiface, + wport = args.wport, + )) + ) server = process_options(parser, web_options, args) m = web.master.WebMaster(web_options, server) except exceptions.OptionsError as e: diff --git a/setup.py b/setup.py index 35f7edb3..927fbc5e 100644 --- a/setup.py +++ b/setup.py @@ -62,7 +62,6 @@ setup( "blinker>=1.4, <1.5", "click>=6.2, <7.0", "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! - "configargparse>=0.10, <0.12", "construct>=2.8, <2.9", "cryptography>=1.3, <1.7", "cssutils>=1.0.1, <1.1", diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 177bac1f..91da47a0 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -5,6 +5,7 @@ import argparse from OpenSSL import SSL from mitmproxy.tools import cmdline +from mitmproxy.tools import main from mitmproxy import options from mitmproxy.proxy import ProxyConfig from mitmproxy import connections @@ -76,7 +77,7 @@ class TestProcessProxyOptions: cmdline.common_options(parser) args = parser.parse_args(args=args) opts = cmdline.get_common_options(args) - pconf = config.ProxyConfig(options.Options(**opts)) + pconf = config.ProxyConfig(options.Options(**main.notnone(opts))) return parser, pconf def assert_err(self, err, *args): -- cgit v1.2.3 From a617e3b5f717b95d1ffc8c87dd70712e36445be9 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Mon, 5 Dec 2016 17:49:16 +1300 Subject: Amalgamate all the Options objects --- mitmproxy/options.py | 47 ++++++++++++++++++++++++++++++++++- mitmproxy/tools/console/master.py | 28 --------------------- mitmproxy/tools/main.py | 7 +++--- mitmproxy/tools/web/master.py | 21 ---------------- test/mitmproxy/console/test_master.py | 11 ++++---- test/mitmproxy/test_web_app.py | 3 ++- test/mitmproxy/test_web_master.py | 5 ++-- 7 files changed, 61 insertions(+), 61 deletions(-) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 06ea30ee..157b0168 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -82,7 +82,29 @@ class Options(optmanager.OptManager): ssl_insecure: bool = False, ssl_verify_upstream_trusted_cadir: Optional[str] = None, ssl_verify_upstream_trusted_ca: Optional[str] = None, - tcp_hosts: Sequence[str] = [] + tcp_hosts: Sequence[str] = [], + + intercept: Optional[str] = None, + + # Console options + eventlog: bool = False, + focus_follow: bool = False, + filter: Optional[str] = None, + palette: Optional[str] = "dark", + palette_transparent: bool = False, + no_mouse: bool = False, + order: Optional[str] = None, + order_reversed: bool = False, + + # Web options + open_browser: bool = True, + wdebug: bool = False, + wport: int = 8081, + wiface: str = "127.0.0.1", + + # Dump options + filtstr: Optional[str] = None, + flow_detail: int = 1 ) -> None: # We could replace all assignments with clever metaprogramming, # but type hints are a much more valueable asset. @@ -146,4 +168,27 @@ class Options(optmanager.OptManager): self.ssl_verify_upstream_trusted_cadir = ssl_verify_upstream_trusted_cadir self.ssl_verify_upstream_trusted_ca = ssl_verify_upstream_trusted_ca self.tcp_hosts = tcp_hosts + + self.intercept = intercept + + # Console options + self.eventlog = eventlog + self.focus_follow = focus_follow + self.filter = filter + self.palette = palette + self.palette_transparent = palette_transparent + self.no_mouse = no_mouse + self.order = order + self.order_reversed = order_reversed + + # Web options + self.open_browser = open_browser + self.wdebug = wdebug + self.wport = wport + self.wiface = wiface + + # Dump options + self.filtstr = filtstr + self.flow_detail = flow_detail + super().__init__() diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 27f5cb58..9b6cc6ab 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -11,7 +11,6 @@ import tempfile import traceback import urwid -from typing import Optional from mitmproxy import addons from mitmproxy import controller @@ -39,33 +38,6 @@ from mitmproxy.net import tcp EVENTLOG_SIZE = 500 -class Options(mitmproxy.options.Options): - def __init__( - self, - *, # all args are keyword-only. - eventlog: bool = False, - focus_follow: bool = False, - intercept: Optional[str] = None, - filter: Optional[str] = None, - palette: Optional[str] = palettes.DEFAULT, - palette_transparent: bool = False, - no_mouse: bool = False, - order: Optional[str] = None, - order_reversed: bool = False, - **kwargs - ): - self.eventlog = eventlog - self.focus_follow = focus_follow - self.intercept = intercept - self.filter = filter - self.palette = palette - self.palette_transparent = palette_transparent - self.no_mouse = no_mouse - self.order = order - self.order_reversed = order_reversed - super().__init__(**kwargs) - - class ConsoleMaster(master.Master): palette = [] diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index a6b75db1..3c0e44bc 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -11,6 +11,7 @@ import signal # noqa from mitmproxy.tools import cmdline # noqa from mitmproxy import exceptions # noqa +from mitmproxy import options # noqa from mitmproxy.proxy import config # noqa from mitmproxy.proxy import server # noqa from mitmproxy.utils import version_check # noqa @@ -71,7 +72,7 @@ def mitmproxy(args=None): # pragma: no cover args = parser.parse_args(args) try: - console_options = console.master.Options() + console_options = options.Options() console_options.load_paths(args.conf) console_options.update(**notnone(cmdline.get_common_options(args))) console_options.update( @@ -110,7 +111,7 @@ def mitmdump(args=None): # pragma: no cover master = None try: - dump_options = dump.Options() + dump_options = options.Options() dump_options.load_paths(args.conf) dump_options.update(**notnone(cmdline.get_common_options(args))) dump_options.update( @@ -149,7 +150,7 @@ def mitmweb(args=None): # pragma: no cover args = parser.parse_args(args) try: - web_options = web.master.Options() + web_options = options.Options() web_options.load_paths(args.conf) web_options.update(**notnone(cmdline.get_common_options(args))) web_options.update( diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 36bdcb07..edb12467 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -1,5 +1,4 @@ import webbrowser -from typing import Optional import tornado.httpserver import tornado.ioloop @@ -7,7 +6,6 @@ from mitmproxy import addons from mitmproxy import exceptions from mitmproxy import log from mitmproxy import master -from mitmproxy import options from mitmproxy.addons import eventstore from mitmproxy.addons import intercept from mitmproxy.addons import termlog @@ -15,25 +13,6 @@ from mitmproxy.addons import view from mitmproxy.tools.web import app -class Options(options.Options): - def __init__( - self, - *, # all args are keyword-only. - intercept: Optional[str] = None, - open_browser: bool = True, - wdebug: bool = False, - wport: int = 8081, - wiface: str = "127.0.0.1", - **kwargs - ) -> None: - self.intercept = intercept - self.open_browser = open_browser - self.wdebug = wdebug - self.wport = wport - self.wiface = wiface - super().__init__(**kwargs) - - class WebMaster(master.Master): def __init__(self, options, server): super().__init__(options, server) diff --git a/test/mitmproxy/console/test_master.py b/test/mitmproxy/console/test_master.py index eb840187..fb3c2527 100644 --- a/test/mitmproxy/console/test_master.py +++ b/test/mitmproxy/console/test_master.py @@ -2,6 +2,7 @@ from mitmproxy.test import tflow import mitmproxy.test.tutils from mitmproxy.tools import console from mitmproxy import proxy +from mitmproxy import options from mitmproxy.tools.console import common from .. import mastertest @@ -20,14 +21,14 @@ def test_format_keyvals(): def test_options(): - assert console.master.Options(replay_kill_extra=True) + assert options.Options(replay_kill_extra=True) class TestMaster(mastertest.MasterTest): - def mkmaster(self, **options): - if "verbosity" not in options: - options["verbosity"] = 0 - o = console.master.Options(**options) + def mkmaster(self, **opts): + if "verbosity" not in opts: + opts["verbosity"] = 0 + o = options.Options(**opts) return console.master.ConsoleMaster(o, proxy.DummyServer()) def test_basic(self): diff --git a/test/mitmproxy/test_web_app.py b/test/mitmproxy/test_web_app.py index be195528..2cab5bf4 100644 --- a/test/mitmproxy/test_web_app.py +++ b/test/mitmproxy/test_web_app.py @@ -4,6 +4,7 @@ import mock import tornado.testing from mitmproxy import exceptions from mitmproxy import proxy +from mitmproxy import options from mitmproxy.test import tflow from mitmproxy.tools.web import app from mitmproxy.tools.web import master as webmaster @@ -17,7 +18,7 @@ def json(resp: httpclient.HTTPResponse): class TestApp(tornado.testing.AsyncHTTPTestCase): def get_app(self): - o = webmaster.Options() + o = options.Options() m = webmaster.WebMaster(o, proxy.DummyServer()) f = tflow.tflow(resp=True) f.id = "42" diff --git a/test/mitmproxy/test_web_master.py b/test/mitmproxy/test_web_master.py index 298b14eb..08dce8f3 100644 --- a/test/mitmproxy/test_web_master.py +++ b/test/mitmproxy/test_web_master.py @@ -1,11 +1,12 @@ from mitmproxy.tools.web import master from mitmproxy import proxy +from mitmproxy import options from . import mastertest class TestWebMaster(mastertest.MasterTest): - def mkmaster(self, **options): - o = master.Options(**options) + def mkmaster(self, **opts): + o = options.Options(**opts) return master.WebMaster(o, proxy.DummyServer(o)) def test_basic(self): -- cgit v1.2.3 From b2695dbc6a170cb21771b001059efa7dc5201722 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 8 Dec 2016 10:18:50 +1300 Subject: Minor tweaks --- mitmproxy/optmanager.py | 13 +++++++------ mitmproxy/tools/cmdline.py | 1 + test/mitmproxy/addons/test_dumper.py | 24 ++++++++++++------------ test/mitmproxy/test_optmanager.py | 2 +- 4 files changed, 21 insertions(+), 19 deletions(-) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 081836ce..e497c3b6 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -180,12 +180,13 @@ class OptManager(metaclass=_DefaultsMeta): Save to path. If the destination file exists, modify it in-place. """ if os.path.exists(path) and os.path.isfile(path): - data = open(path, "r").read() + with open(path, "r") as f: + data = f.read() else: data = "" data = self.serialize(data, defaults) - fp = open(path, "w") - fp.write(data) + with open(path, "w") as f: + f.write(data) def serialize(self, text, defaults=False): """ @@ -228,8 +229,7 @@ class OptManager(metaclass=_DefaultsMeta): this object. May raise OptionsError if the config file is invalid. """ data = self._load(text) - for k, v in data.items(): - setattr(self, k, v) + self.update(**data) def load_paths(self, *paths): """ @@ -240,7 +240,8 @@ class OptManager(metaclass=_DefaultsMeta): for p in paths: p = os.path.expanduser(p) if os.path.exists(p) and os.path.isfile(p): - txt = open(p, "r").read() + with open(p, "r") as f: + txt = f.read() self.load(txt) def __repr__(self): diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 1cdde727..925491d7 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -513,6 +513,7 @@ def proxy_ssl_options(parser): 'as the first entry. Can be passed multiple times.') group.add_argument( "--ciphers-client", action="store", + type=str, dest="ciphers_client", help="Set supported ciphers for client connections. (OpenSSL Syntax)" ) group.add_argument( diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index f87df329..760efa08 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -31,37 +31,37 @@ def test_simple(): sio = io.StringIO() d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - ctx.configure(d, flow_detail = 0) + ctx.configure(d, flow_detail=0) d.response(tflow.tflow(resp=True)) assert not sio.getvalue() sio.truncate(0) - ctx.configure(d, flow_detail = 1) + ctx.configure(d, flow_detail=1) d.response(tflow.tflow(resp=True)) assert sio.getvalue() sio.truncate(0) - ctx.configure(d, flow_detail = 1) + ctx.configure(d, flow_detail=1) d.error(tflow.tflow(err=True)) assert sio.getvalue() sio.truncate(0) - ctx.configure(d, flow_detail = 4) + ctx.configure(d, flow_detail=4) d.response(tflow.tflow(resp=True)) assert sio.getvalue() sio.truncate(0) - ctx.configure(d, flow_detail = 4) + ctx.configure(d, flow_detail=4) d.response(tflow.tflow(resp=True)) assert "<<" in sio.getvalue() sio.truncate(0) - ctx.configure(d, flow_detail = 4) + ctx.configure(d, flow_detail=4) d.response(tflow.tflow(err=True)) assert "<<" in sio.getvalue() sio.truncate(0) - ctx.configure(d, flow_detail = 4) + ctx.configure(d, flow_detail=4) flow = tflow.tflow() flow.request = tutils.treq() flow.request.stickycookie = True @@ -74,7 +74,7 @@ def test_simple(): assert sio.getvalue() sio.truncate(0) - ctx.configure(d, flow_detail = 4) + ctx.configure(d, flow_detail=4) flow = tflow.tflow(resp=tutils.tresp(content=b"{")) flow.response.headers["content-type"] = "application/json" flow.response.status_code = 400 @@ -82,7 +82,7 @@ def test_simple(): assert sio.getvalue() sio.truncate(0) - ctx.configure(d, flow_detail = 4) + ctx.configure(d, flow_detail=4) flow = tflow.tflow() flow.request.content = None flow.response = http.HTTPResponse.wrap(tutils.tresp()) @@ -100,7 +100,7 @@ def test_echo_body(): sio = io.StringIO() d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - ctx.configure(d, flow_detail = 3) + ctx.configure(d, flow_detail=3) d._echo_message(f.response) t = sio.getvalue() assert "cut off" in t @@ -110,7 +110,7 @@ def test_echo_request_line(): sio = io.StringIO() d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - ctx.configure(d, flow_detail = 3, showhost = True) + ctx.configure(d, flow_detail=3, showhost=True) f = tflow.tflow(client_conn=None, server_conn=True, resp=True) f.request.is_replay = True d._echo_request_line(f) @@ -146,7 +146,7 @@ def test_tcp(): sio = io.StringIO() d = dumper.Dumper(sio) with taddons.context(options=dump.Options()) as ctx: - ctx.configure(d, flow_detail = 3, showhost = True) + ctx.configure(d, flow_detail=3, showhost=True) f = tflow.ttcpflow(client_conn=True, server_conn=True) d.tcp_message(f) assert "it's me" in sio.getvalue() diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index 0c98daea..c6eb2534 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -15,7 +15,7 @@ class TO(optmanager.OptManager): class TD(optmanager.OptManager): - def __init__(self, one="done", two="dtwo", three="error"): + def __init__(self, *, one="done", two="dtwo", three="error"): self.one = one self.two = two self.three = three -- cgit v1.2.3