From 917c701562794b59b7f4015e9165a7ef7ed50f79 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 25 Oct 2016 17:34:30 -0700 Subject: make options keyword-only --- mitmproxy/options.py | 1 + mitmproxy/tools/console/master.py | 1 + mitmproxy/tools/dump.py | 1 + mitmproxy/tools/web/master.py | 1 + 4 files changed, 4 insertions(+) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 26b7030e..8e885735 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -23,6 +23,7 @@ DEFAULT_CLIENT_CIPHERS = "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA class Options(optmanager.OptManager): def __init__( self, + *, # all args are keyword-only. # TODO: rename to onboarding_app_* app: bool = True, app_host: str = APP_HOST, diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 909c83da..92959256 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -203,6 +203,7 @@ class ConsoleState(state.State): class Options(mitmproxy.options.Options): def __init__( self, + *, # all args are keyword-only. eventlog: bool = False, follow: bool = False, intercept: bool = False, diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index e92482f3..72895102 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -18,6 +18,7 @@ class DumpError(Exception): class Options(options.Options): def __init__( self, + *, # all args are keyword-only. keepserving: bool = False, filtstr: Optional[str] = None, flow_detail: int = 1, diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 619582f3..9cf17faf 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -94,6 +94,7 @@ class WebState(state.State): class Options(options.Options): def __init__( self, + *, # all args are keyword-only. intercept: Optional[str] = None, wdebug: bool = bool, wport: int = 8081, -- cgit v1.2.3 From b1bdae3d1cbcf7c4c427cac5163faf75150f4cff Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 25 Oct 2016 20:45:48 -0700 Subject: typecheck options, fix current inconsistencies --- mitmproxy/addons/clientplayback.py | 2 +- mitmproxy/options.py | 16 ++--- mitmproxy/optmanager.py | 13 +++- mitmproxy/tools/cmdline.py | 10 ++-- mitmproxy/tools/console/master.py | 2 +- mitmproxy/tools/dump.py | 2 +- mitmproxy/tools/web/master.py | 2 +- mitmproxy/utils/typecheck.py | 54 +++++++++++++++++ setup.py | 2 +- test/mitmproxy/test_utils_data.py | 7 --- test/mitmproxy/test_utils_debug.py | 23 ------- test/mitmproxy/test_utils_human.py | 46 -------------- test/mitmproxy/test_utils_strutils.py | 96 ------------------------------ test/mitmproxy/test_utils_version_check.py | 25 -------- test/mitmproxy/utils/__init__.py | 0 test/mitmproxy/utils/test_data.py | 8 +++ test/mitmproxy/utils/test_debug.py | 23 +++++++ test/mitmproxy/utils/test_human.py | 46 ++++++++++++++ test/mitmproxy/utils/test_strutils.py | 96 ++++++++++++++++++++++++++++++ test/mitmproxy/utils/test_typecheck.py | 48 +++++++++++++++ test/mitmproxy/utils/test_version_check.py | 25 ++++++++ 21 files changed, 330 insertions(+), 216 deletions(-) create mode 100644 mitmproxy/utils/typecheck.py delete mode 100644 test/mitmproxy/test_utils_data.py delete mode 100644 test/mitmproxy/test_utils_debug.py delete mode 100644 test/mitmproxy/test_utils_human.py delete mode 100644 test/mitmproxy/test_utils_strutils.py delete mode 100644 test/mitmproxy/test_utils_version_check.py create mode 100644 test/mitmproxy/utils/__init__.py create mode 100644 test/mitmproxy/utils/test_data.py create mode 100644 test/mitmproxy/utils/test_debug.py create mode 100644 test/mitmproxy/utils/test_human.py create mode 100644 test/mitmproxy/utils/test_strutils.py create mode 100644 test/mitmproxy/utils/test_typecheck.py create mode 100644 test/mitmproxy/utils/test_version_check.py diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 7f2b53ac..e69cd27a 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -21,7 +21,7 @@ class ClientPlayback: def configure(self, options, updated): if "client_replay" in updated: if options.client_replay: - ctx.log.info(options.client_replay) + ctx.log.info("Client Replay: {}".format(options.client_replay)) try: flows = io.read_flows_from_paths(options.client_replay) except exceptions.FlowReadException as e: diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 8e885735..03547189 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -30,7 +30,7 @@ class Options(optmanager.OptManager): app_port: int = APP_PORT, anticache: bool = False, anticomp: bool = False, - client_replay: Optional[str] = None, + client_replay: Sequence[str] = (), replay_kill_extra: bool = False, keepserving: bool = True, no_server: bool = False, @@ -42,12 +42,12 @@ class Options(optmanager.OptManager): replacements: Sequence[Tuple[str, str, str]] = (), server_replay_use_headers: Sequence[str] = (), setheaders: Sequence[Tuple[str, str, str]] = (), - server_replay: Sequence[str] = None, + server_replay: Sequence[str] = (), stickycookie: Optional[str] = None, stickyauth: Optional[str] = None, - stream_large_bodies: Optional[str] = None, + stream_large_bodies: Optional[int] = None, verbosity: int = 2, - outfile: Tuple[str, str] = None, + outfile: Optional[Tuple[str, str]] = None, server_replay_ignore_content: bool = False, server_replay_ignore_params: Sequence[str] = (), server_replay_ignore_payload_params: Sequence[str] = (), @@ -72,13 +72,13 @@ class Options(optmanager.OptManager): rawtcp: bool = False, websockets: bool = False, spoof_source_address: bool = False, - upstream_server: str = "", - upstream_auth: str = "", + upstream_server: Optional[str] = None, + upstream_auth: Optional[str] = None, ssl_version_client: str = "secure", ssl_version_server: str = "secure", ssl_insecure: bool = False, - ssl_verify_upstream_trusted_cadir: str = None, - ssl_verify_upstream_trusted_ca: str = None, + ssl_verify_upstream_trusted_cadir: Optional[str] = None, + ssl_verify_upstream_trusted_ca: Optional[str] = None, tcp_hosts: Sequence[str] = () ): # We could replace all assignments with clever metaprogramming, diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 6683e41d..20492f82 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -3,6 +3,7 @@ import blinker import pprint from mitmproxy import exceptions +from mitmproxy.utils import typecheck """ The base implementation for Options. @@ -58,10 +59,19 @@ class OptManager: def __setattr__(self, attr, value): if not self._initialized: + self._typecheck(attr, value) self._opts[attr] = value return self.update(**{attr: value}) + def _typecheck(self, attr, value): + expected_type = typecheck.get_arg_type_from_constructor_annotation( + type(self), attr + ) + if expected_type is None: + return # no type info :( + typecheck.check_type(attr, value, expected_type) + def keys(self): return set(self._opts.keys()) @@ -70,9 +80,10 @@ class OptManager: def update(self, **kwargs): updated = set(kwargs.keys()) - for k in kwargs: + for k, v in kwargs.items(): if k not in self._opts: raise KeyError("No such option: %s" % k) + self._typecheck(k, v) with self.rollback(updated): self._opts.update(kwargs) self.changed.send(self, updated=updated) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index 2f9ea15c..55adb7fa 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -591,7 +591,7 @@ def client_replay(parser): group = parser.add_argument_group("Client Replay") group.add_argument( "-c", "--client-replay", - action="append", dest="client_replay", default=None, metavar="PATH", + action="append", dest="client_replay", default=[], metavar="PATH", help="Replay client requests from a saved file." ) @@ -600,7 +600,7 @@ def server_replay(parser): group = parser.add_argument_group("Server Replay") group.add_argument( "-S", "--server-replay", - action="append", dest="server_replay", default=None, metavar="PATH", + action="append", dest="server_replay", default=[], metavar="PATH", help="Replay server responses from a saved file." ) group.add_argument( @@ -610,7 +610,7 @@ def server_replay(parser): ) group.add_argument( "--server-replay-use-header", - action="append", dest="server_replay_use_headers", type=str, + action="append", dest="server_replay_use_headers", type=str, default=[], help="Request headers to be considered during replay. " "Can be passed multiple times." ) @@ -638,7 +638,7 @@ def server_replay(parser): ) payload.add_argument( "--replay-ignore-payload-param", - action="append", dest="server_replay_ignore_payload_params", type=str, + action="append", dest="server_replay_ignore_payload_params", type=str, default=[], help=""" Request's payload parameters (application/x-www-form-urlencoded or multipart/form-data) to be ignored while searching for a saved flow to replay. @@ -648,7 +648,7 @@ def server_replay(parser): group.add_argument( "--replay-ignore-param", - action="append", dest="server_replay_ignore_params", type=str, + action="append", dest="server_replay_ignore_params", type=str, default=[], help=""" Request's parameters to be ignored while searching for a saved flow to replay. Can be passed multiple times. diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 92959256..c128e42b 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -206,7 +206,7 @@ class Options(mitmproxy.options.Options): *, # all args are keyword-only. eventlog: bool = False, follow: bool = False, - intercept: bool = False, + intercept: Optional[str] = None, filter: Optional[str] = None, palette: Optional[str] = None, palette_transparent: bool = False, diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index 72895102..837959bc 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -18,7 +18,7 @@ class DumpError(Exception): class Options(options.Options): def __init__( self, - *, # all args are keyword-only. + *, # all args are keyword-only. keepserving: bool = False, filtstr: Optional[str] = None, flow_detail: int = 1, diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 9cf17faf..75842422 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -96,7 +96,7 @@ class Options(options.Options): self, *, # all args are keyword-only. intercept: Optional[str] = None, - wdebug: bool = bool, + wdebug: bool = False, wport: int = 8081, wiface: str = "127.0.0.1", wauthenticator: Optional[authentication.PassMan] = None, diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py new file mode 100644 index 00000000..ce57cff1 --- /dev/null +++ b/mitmproxy/utils/typecheck.py @@ -0,0 +1,54 @@ +import typing + + +def check_type(attr_name: str, value: typing.Any, typeinfo: type) -> None: + """ + This function checks if the provided value is an instance of typeinfo + and raises a TypeError otherwise. + + The following types from the typing package have specialized support: + + - Union + - Tuple + - TextIO + """ + # If we realize that we need to extend this list substantially, it may make sense + # to use typeguard for this, but right now it's not worth the hassle for 16 lines of code. + + e = TypeError("Expected {} for {}, but got {}.".format( + typeinfo, + attr_name, + type(value) + )) + + if isinstance(typeinfo, typing.UnionMeta): + for T in typeinfo.__union_params__: + try: + check_type(attr_name, value, T) + except TypeError: + pass + else: + return + raise e + if isinstance(typeinfo, typing.TupleMeta): + check_type(attr_name, value, tuple) + if len(typeinfo.__tuple_params__) != len(value): + raise e + for i, (x, T) in enumerate(zip(value, typeinfo.__tuple_params__)): + check_type("{}[{}]".format(attr_name, i), x, T) + return + if typeinfo == typing.TextIO: + if hasattr(value, "read"): + return + + if not isinstance(value, typeinfo): + raise e + + +def get_arg_type_from_constructor_annotation(cls: type, attr: str) -> typing.Optional[type]: + """ + Returns the first type annotation for attr in the class hierarchy. + """ + for c in cls.mro(): + if attr in getattr(c.__init__, "__annotations__", ()): + return c.__init__.__annotations__[attr] diff --git a/setup.py b/setup.py index 1351ba73..4165b367 100644 --- a/setup.py +++ b/setup.py @@ -67,7 +67,7 @@ setup( "cryptography>=1.3, <1.6", "cssutils>=1.0.1, <1.1", "Flask>=0.10.1, <0.12", - "h2>=2.4.1, <3", + "h2>=2.4.1, <2.5", "html2text>=2016.1.8, <=2016.9.19", "hyperframe>=4.0.1, <5", "jsbeautifier>=1.6.3, <1.7", diff --git a/test/mitmproxy/test_utils_data.py b/test/mitmproxy/test_utils_data.py deleted file mode 100644 index c6e4420e..00000000 --- a/test/mitmproxy/test_utils_data.py +++ /dev/null @@ -1,7 +0,0 @@ -from mitmproxy.utils import data -from . import tutils - - -def test_pkg_data(): - assert data.pkg_data.path("tools/console") - tutils.raises("does not exist", data.pkg_data.path, "nonexistent") diff --git a/test/mitmproxy/test_utils_debug.py b/test/mitmproxy/test_utils_debug.py deleted file mode 100644 index 9acf8192..00000000 --- a/test/mitmproxy/test_utils_debug.py +++ /dev/null @@ -1,23 +0,0 @@ -import io - -from mitmproxy.utils import debug - - -def test_dump_info(): - cs = io.StringIO() - debug.dump_info(None, None, file=cs, testing=True) - assert cs.getvalue() - - -def test_dump_stacks(): - cs = io.StringIO() - debug.dump_stacks(None, None, file=cs, testing=True) - assert cs.getvalue() - - -def test_sysinfo(): - assert debug.sysinfo() - - -def test_register_info_dumpers(): - debug.register_info_dumpers() diff --git a/test/mitmproxy/test_utils_human.py b/test/mitmproxy/test_utils_human.py deleted file mode 100644 index 443c8f66..00000000 --- a/test/mitmproxy/test_utils_human.py +++ /dev/null @@ -1,46 +0,0 @@ -import time -from mitmproxy.utils import human -from mitmproxy.test import tutils - - -def test_format_timestamp(): - assert human.format_timestamp(time.time()) - - -def test_format_timestamp_with_milli(): - assert human.format_timestamp_with_milli(time.time()) - - -def test_parse_size(): - assert human.parse_size("0") == 0 - assert human.parse_size("0b") == 0 - assert human.parse_size("1") == 1 - assert human.parse_size("1k") == 1024 - assert human.parse_size("1m") == 1024**2 - assert human.parse_size("1g") == 1024**3 - tutils.raises(ValueError, human.parse_size, "1f") - tutils.raises(ValueError, human.parse_size, "ak") - - -def test_pretty_size(): - assert human.pretty_size(0) == "0b" - assert human.pretty_size(100) == "100b" - assert human.pretty_size(1024) == "1k" - assert human.pretty_size(1024 + (1024 / 2.0)) == "1.5k" - assert human.pretty_size(1024 * 1024) == "1m" - assert human.pretty_size(10 * 1024 * 1024) == "10m" - - -def test_pretty_duration(): - assert human.pretty_duration(0.00001) == "0ms" - assert human.pretty_duration(0.0001) == "0ms" - assert human.pretty_duration(0.001) == "1ms" - assert human.pretty_duration(0.01) == "10ms" - assert human.pretty_duration(0.1) == "100ms" - assert human.pretty_duration(1) == "1.00s" - assert human.pretty_duration(10) == "10.0s" - assert human.pretty_duration(100) == "100s" - assert human.pretty_duration(1000) == "1000s" - assert human.pretty_duration(10000) == "10000s" - assert human.pretty_duration(1.123) == "1.12s" - assert human.pretty_duration(0.123) == "123ms" diff --git a/test/mitmproxy/test_utils_strutils.py b/test/mitmproxy/test_utils_strutils.py deleted file mode 100644 index 84281c6b..00000000 --- a/test/mitmproxy/test_utils_strutils.py +++ /dev/null @@ -1,96 +0,0 @@ -from mitmproxy.utils import strutils -from mitmproxy.test import tutils - - -def test_always_bytes(): - assert strutils.always_bytes(bytes(range(256))) == bytes(range(256)) - assert strutils.always_bytes("foo") == b"foo" - with tutils.raises(ValueError): - strutils.always_bytes(u"\u2605", "ascii") - with tutils.raises(TypeError): - strutils.always_bytes(42, "ascii") - - -def test_native(): - with tutils.raises(TypeError): - strutils.native(42) - assert strutils.native(u"foo") == u"foo" - assert strutils.native(b"foo") == u"foo" - - -def test_escape_control_characters(): - assert strutils.escape_control_characters(u"one") == u"one" - assert strutils.escape_control_characters(u"\00ne") == u".ne" - assert strutils.escape_control_characters(u"\nne") == u"\nne" - assert strutils.escape_control_characters(u"\nne", False) == u".ne" - assert strutils.escape_control_characters(u"\u2605") == u"\u2605" - assert ( - strutils.escape_control_characters(bytes(bytearray(range(128))).decode()) == - u'.........\t\n..\r.................. !"#$%&\'()*+,-./0123456789:;<' - u'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.' - ) - assert ( - strutils.escape_control_characters(bytes(bytearray(range(128))).decode(), False) == - u'................................ !"#$%&\'()*+,-./0123456789:;<' - u'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.' - ) - - with tutils.raises(ValueError): - strutils.escape_control_characters(b"foo") - - -def test_bytes_to_escaped_str(): - assert strutils.bytes_to_escaped_str(b"foo") == "foo" - assert strutils.bytes_to_escaped_str(b"\b") == r"\x08" - assert strutils.bytes_to_escaped_str(br"&!?=\)") == r"&!?=\\)" - assert strutils.bytes_to_escaped_str(b'\xc3\xbc') == r"\xc3\xbc" - assert strutils.bytes_to_escaped_str(b"'") == r"'" - assert strutils.bytes_to_escaped_str(b'"') == r'"' - - assert strutils.bytes_to_escaped_str(b"'", escape_single_quotes=True) == r"\'" - assert strutils.bytes_to_escaped_str(b'"', escape_single_quotes=True) == r'"' - - assert strutils.bytes_to_escaped_str(b"\r\n\t") == "\\r\\n\\t" - assert strutils.bytes_to_escaped_str(b"\r\n\t", True) == "\r\n\t" - - assert strutils.bytes_to_escaped_str(b"\n", True) == "\n" - assert strutils.bytes_to_escaped_str(b"\\n", True) == "\\ \\ n".replace(" ", "") - assert strutils.bytes_to_escaped_str(b"\\\n", True) == "\\ \\ \n".replace(" ", "") - assert strutils.bytes_to_escaped_str(b"\\\\n", True) == "\\ \\ \\ \\ n".replace(" ", "") - - with tutils.raises(ValueError): - strutils.bytes_to_escaped_str(u"such unicode") - - -def test_escaped_str_to_bytes(): - assert strutils.escaped_str_to_bytes("foo") == b"foo" - assert strutils.escaped_str_to_bytes("\x08") == b"\b" - assert strutils.escaped_str_to_bytes("&!?=\\\\)") == br"&!?=\)" - assert strutils.escaped_str_to_bytes(u"\\x08") == b"\b" - assert strutils.escaped_str_to_bytes(u"&!?=\\\\)") == br"&!?=\)" - assert strutils.escaped_str_to_bytes(u"\u00fc") == b'\xc3\xbc' - - with tutils.raises(ValueError): - strutils.escaped_str_to_bytes(b"very byte") - - -def test_is_mostly_bin(): - assert not strutils.is_mostly_bin(b"foo\xFF") - assert strutils.is_mostly_bin(b"foo" + b"\xFF" * 10) - assert not strutils.is_mostly_bin("") - - -def test_is_xml(): - assert not strutils.is_xml(b"foo") - assert strutils.is_xml(b"?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.' + ) + assert ( + strutils.escape_control_characters(bytes(bytearray(range(128))).decode(), False) == + u'................................ !"#$%&\'()*+,-./0123456789:;<' + u'=>?@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_`abcdefghijklmnopqrstuvwxyz{|}~.' + ) + + with tutils.raises(ValueError): + strutils.escape_control_characters(b"foo") + + +def test_bytes_to_escaped_str(): + assert strutils.bytes_to_escaped_str(b"foo") == "foo" + assert strutils.bytes_to_escaped_str(b"\b") == r"\x08" + assert strutils.bytes_to_escaped_str(br"&!?=\)") == r"&!?=\\)" + assert strutils.bytes_to_escaped_str(b'\xc3\xbc') == r"\xc3\xbc" + assert strutils.bytes_to_escaped_str(b"'") == r"'" + assert strutils.bytes_to_escaped_str(b'"') == r'"' + + assert strutils.bytes_to_escaped_str(b"'", escape_single_quotes=True) == r"\'" + assert strutils.bytes_to_escaped_str(b'"', escape_single_quotes=True) == r'"' + + assert strutils.bytes_to_escaped_str(b"\r\n\t") == "\\r\\n\\t" + assert strutils.bytes_to_escaped_str(b"\r\n\t", True) == "\r\n\t" + + assert strutils.bytes_to_escaped_str(b"\n", True) == "\n" + assert strutils.bytes_to_escaped_str(b"\\n", True) == "\\ \\ n".replace(" ", "") + assert strutils.bytes_to_escaped_str(b"\\\n", True) == "\\ \\ \n".replace(" ", "") + assert strutils.bytes_to_escaped_str(b"\\\\n", True) == "\\ \\ \\ \\ n".replace(" ", "") + + with tutils.raises(ValueError): + strutils.bytes_to_escaped_str(u"such unicode") + + +def test_escaped_str_to_bytes(): + assert strutils.escaped_str_to_bytes("foo") == b"foo" + assert strutils.escaped_str_to_bytes("\x08") == b"\b" + assert strutils.escaped_str_to_bytes("&!?=\\\\)") == br"&!?=\)" + assert strutils.escaped_str_to_bytes(u"\\x08") == b"\b" + assert strutils.escaped_str_to_bytes(u"&!?=\\\\)") == br"&!?=\)" + assert strutils.escaped_str_to_bytes(u"\u00fc") == b'\xc3\xbc' + + with tutils.raises(ValueError): + strutils.escaped_str_to_bytes(b"very byte") + + +def test_is_mostly_bin(): + assert not strutils.is_mostly_bin(b"foo\xFF") + assert strutils.is_mostly_bin(b"foo" + b"\xFF" * 10) + assert not strutils.is_mostly_bin("") + + +def test_is_xml(): + assert not strutils.is_xml(b"foo") + assert strutils.is_xml(b"