diff options
69 files changed, 893 insertions, 659 deletions
diff --git a/.travis.yml b/.travis.yml index efb28bc7..e64bf6d4 100644 --- a/.travis.yml +++ b/.travis.yml @@ -49,7 +49,9 @@ matrix: env: TOXENV=docs - language: node_js node_js: "node" - before_install: npm install -g yarn + before_install: + - curl -o- -L https://yarnpkg.com/install.sh | bash + - export PATH=$HOME/.yarn/bin:$PATH install: cd web && yarn script: npm test cache: diff --git a/docs/features/replacements.rst b/docs/features/replacements.rst index 215f0ddb..39dccca2 100644 --- a/docs/features/replacements.rst +++ b/docs/features/replacements.rst @@ -48,14 +48,14 @@ In practice, it's pretty common for the replacement literal to be long and complex. For instance, it might be an XSS exploit that weighs in at hundreds or thousands of characters. To cope with this, there's a variation of the replacement hook specifier that lets you load the replacement text from a file. -So, you might start **mitmdump** as follows: +To specify a file as replacement, prefix the file path with ``@``. +You might start **mitmdump** as follows: ->>> mitmdump --replace-from-file :~q:foo:~/xss-exploit +>>> mitmdump --replacements :~q:foo:@~/xss-exploit This will load the replacement text from the file ``~/xss-exploit``. -Both the ``--replace`` and ``--replace-from-file`` flags can be passed multiple -times. +The ``--replacements`` flag can be passed multiple times. Interactively @@ -66,7 +66,6 @@ replacement hooks using a built-in editor. The context-sensitive help (:kbd:`?`) complete usage information. ================== ======================= -command-line ``--replace``, - ``--replace-from-file`` +command-line ``--replacements`` mitmproxy shortcut :kbd:`O` then :kbd:`R` ================== ======================= diff --git a/examples/complex/dns_spoofing.py b/examples/complex/dns_spoofing.py index 2fd6b699..ca2bcd35 100644 --- a/examples/complex/dns_spoofing.py +++ b/examples/complex/dns_spoofing.py @@ -1,11 +1,12 @@ """ -This script makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect -connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the -Host header of the HTTP request. -Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't -know the actual target and cannot construct a certificate that looks valid. -Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well. -Using transparent mode is the better option most of the time. +This script makes it possible to use mitmproxy in scenarios where IP spoofing +has been used to redirect connections to mitmproxy. The way this works is that +we rely on either the TLS Server Name Indication (SNI) or the Host header of the +HTTP request. Of course, this is not foolproof - if an HTTPS connection comes +without SNI, we don't know the actual target and cannot construct a certificate +that looks valid. Similarly, if there's no Host header or a spoofed Host header, +we're out of luck as well. Using transparent mode is the better option most of +the time. Usage: mitmproxy @@ -53,5 +54,5 @@ class Rerouter: flow.request.port = port -def start(): +def start(opts): return Rerouter() diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index 86a33684..9a86e45e 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -25,7 +25,7 @@ HAR = {} SERVERS_SEEN = set() -def start(): +def start(opts): """ Called once on script startup before any other events. """ diff --git a/examples/complex/remote_debug.py b/examples/complex/remote_debug.py index fb864f78..ae0dffc1 100644 --- a/examples/complex/remote_debug.py +++ b/examples/complex/remote_debug.py @@ -14,6 +14,6 @@ Usage: """ -def start(): +def start(opts): import pydevd pydevd.settrace("localhost", port=5678, stdoutToServer=True, stderrToServer=True) diff --git a/examples/complex/tls_passthrough.py b/examples/complex/tls_passthrough.py index 40c1051d..6dba7ca1 100644 --- a/examples/complex/tls_passthrough.py +++ b/examples/complex/tls_passthrough.py @@ -112,7 +112,7 @@ class TlsFeedback(TlsLayer): tls_strategy = None -def start(): +def start(opts): global tls_strategy if len(sys.argv) == 2: tls_strategy = ProbabilisticStrategy(float(sys.argv[1])) diff --git a/examples/simple/add_header_class.py b/examples/simple/add_header_class.py index 6443798a..9270be09 100644 --- a/examples/simple/add_header_class.py +++ b/examples/simple/add_header_class.py @@ -3,5 +3,5 @@ class AddHeader: flow.response.headers["newheader"] = "foo" -def start(): +def start(opts): return AddHeader() diff --git a/examples/simple/custom_contentview.py b/examples/simple/custom_contentview.py index 1f3a38ec..4bc17af0 100644 --- a/examples/simple/custom_contentview.py +++ b/examples/simple/custom_contentview.py @@ -20,7 +20,7 @@ class ViewSwapCase(contentviews.View): view = ViewSwapCase() -def start(): +def start(opts): contentviews.add(view) diff --git a/examples/simple/custom_option.py b/examples/simple/custom_option.py new file mode 100644 index 00000000..324d27e7 --- /dev/null +++ b/examples/simple/custom_option.py @@ -0,0 +1,11 @@ +from mitmproxy import ctx + + +def start(options): + ctx.log.info("Registering option 'custom'") + options.add_option("custom", bool, False, "A custom option") + + +def configure(options, updated): + if "custom" in updated: + ctx.log.info("custom option value: %s" % options.custom) diff --git a/examples/simple/filter_flows.py b/examples/simple/filter_flows.py index 29d0a9b8..24e8b6c1 100644 --- a/examples/simple/filter_flows.py +++ b/examples/simple/filter_flows.py @@ -17,7 +17,7 @@ class Filter: print(flow) -def start(): +def start(opts): if len(sys.argv) != 2: raise ValueError("Usage: -s 'filt.py FILTER'") return Filter(sys.argv[1]) diff --git a/examples/simple/io_write_dumpfile.py b/examples/simple/io_write_dumpfile.py index ff1fd0f4..311950af 100644 --- a/examples/simple/io_write_dumpfile.py +++ b/examples/simple/io_write_dumpfile.py @@ -23,7 +23,7 @@ class Writer: self.w.add(flow) -def start(): +def start(opts): if len(sys.argv) != 2: raise ValueError('Usage: -s "flowriter.py filename"') return Writer(sys.argv[1]) diff --git a/examples/simple/log_events.py b/examples/simple/log_events.py index ab1baf75..a81892aa 100644 --- a/examples/simple/log_events.py +++ b/examples/simple/log_events.py @@ -7,6 +7,6 @@ If you want to help us out: https://github.com/mitmproxy/mitmproxy/issues/1530 : from mitmproxy import ctx -def start(): +def start(opts): ctx.log.info("This is some informative text.") ctx.log.error("This is an error.") diff --git a/examples/simple/modify_body_inject_iframe.py b/examples/simple/modify_body_inject_iframe.py index e3d5fee9..ab5abf27 100644 --- a/examples/simple/modify_body_inject_iframe.py +++ b/examples/simple/modify_body_inject_iframe.py @@ -23,7 +23,7 @@ class Injector: flow.response.content = str(html).encode("utf8") -def start(): +def start(opts): if len(sys.argv) != 2: raise ValueError('Usage: -s "iframe_injector.py url"') return Injector(sys.argv[1]) diff --git a/examples/simple/script_arguments.py b/examples/simple/script_arguments.py index 70851192..b46a1960 100644 --- a/examples/simple/script_arguments.py +++ b/examples/simple/script_arguments.py @@ -9,7 +9,7 @@ class Replacer: flow.response.replace(self.src, self.dst) -def start(): +def start(opts): parser = argparse.ArgumentParser() parser.add_argument("src", type=str) parser.add_argument("dst", type=str) diff --git a/examples/simple/wsgi_flask_app.py b/examples/simple/wsgi_flask_app.py index f95c41e5..db3b1adf 100644 --- a/examples/simple/wsgi_flask_app.py +++ b/examples/simple/wsgi_flask_app.py @@ -14,7 +14,7 @@ def hello_world(): return 'Hello World!' -def start(): +def start(opts): # Host app at the magic domain "proxapp" on port 80. Requests to this # domain and port combination will now be routed to the WSGI app instance. return wsgiapp.WSGIApp(app, "proxapp", 80) diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py index db8e0cd7..43e76510 100644 --- a/mitmproxy/addonmanager.py +++ b/mitmproxy/addonmanager.py @@ -1,4 +1,5 @@ from mitmproxy import exceptions +from mitmproxy import eventsequence import pprint @@ -10,7 +11,7 @@ class AddonManager: def __init__(self, master): self.chain = [] self.master = master - master.options.changed.connect(self._options_update) + master.options.changed.connect(self.configure_all) def clear(self): """ @@ -29,22 +30,14 @@ class AddonManager: if name == _get_name(i): return i - def _options_update(self, options, updated): - for i in self.chain: - with self.master.handlecontext(): - self.invoke_with_context(i, "configure", options, updated) + def configure_all(self, options, updated): + self.invoke_all_with_context("configure", options, updated) def startup(self, s): """ Run startup events on addon. """ - self.invoke_with_context(s, "start") - self.invoke_with_context( - s, - "configure", - self.master.options, - self.master.options.keys() - ) + self.invoke_with_context(s, "start", self.master.options) def add(self, *addons): """ @@ -62,8 +55,7 @@ class AddonManager: self.invoke_with_context(addon, "done") def done(self): - for i in self.chain: - self.invoke_with_context(i, "done") + self.invoke_all_with_context("done") def __len__(self): return len(self.chain) @@ -75,7 +67,14 @@ class AddonManager: with self.master.handlecontext(): self.invoke(addon, name, *args, **kwargs) + def invoke_all_with_context(self, name, *args, **kwargs): + with self.master.handlecontext(): + for i in self.chain: + self.invoke(i, name, *args, **kwargs) + def invoke(self, addon, name, *args, **kwargs): + if name not in eventsequence.Events: # prama: no cover + raise NotImplementedError("Unknown event") func = getattr(addon, name, None) if func: if not callable(func): diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py index 80e3b2cb..7a45106c 100644 --- a/mitmproxy/addons/__init__.py +++ b/mitmproxy/addons/__init__.py @@ -30,7 +30,6 @@ def default_addons(): onboarding.Onboarding(), proxyauth.ProxyAuth(), replace.Replace(), - replace.ReplaceFile(), script.ScriptLoader(), serverplayback.ServerPlayback(), setheaders.SetHeaders(), diff --git a/mitmproxy/addons/replace.py b/mitmproxy/addons/replace.py index 0d0c3aa5..d6c11ca4 100644 --- a/mitmproxy/addons/replace.py +++ b/mitmproxy/addons/replace.py @@ -1,3 +1,4 @@ +import os import re from mitmproxy import exceptions @@ -42,7 +43,7 @@ def parse_hook(s): return patt, a, b -class _ReplaceBase: +class Replace: def __init__(self): self.lst = [] @@ -51,12 +52,12 @@ class _ReplaceBase: .replacements is a list of tuples (fpat, rex, s): fpatt: a string specifying a filter pattern. - rex: a regular expression, as bytes. - s: the replacement string, as bytes + rex: a regular expression, as string. + s: the replacement string """ - if self.optionName in updated: + if "replacements" in updated: lst = [] - for rep in getattr(options, self.optionName): + for rep in options.replacements: fpatt, rex, s = parse_hook(rep) flt = flowfilter.parse(fpatt) @@ -65,11 +66,16 @@ class _ReplaceBase: "Invalid filter pattern: %s" % fpatt ) try: + # We should ideally escape here before trying to compile re.compile(rex) except re.error as e: raise exceptions.OptionsError( "Invalid regular expression: %s - %s" % (rex, str(e)) ) + if s.startswith("@") and not os.path.isfile(s[1:]): + raise exceptions.OptionsError( + "Invalid file path: {}".format(s[1:]) + ) lst.append((rex, s, flt)) self.lst = lst @@ -89,21 +95,13 @@ class _ReplaceBase: if not flow.reply.has_message: self.execute(flow) - -class Replace(_ReplaceBase): - optionName = "replacements" - def replace(self, obj, rex, s): + if s.startswith("@"): + s = os.path.expanduser(s[1:]) + try: + with open(s, "rb") as f: + s = f.read() + except IOError: + ctx.log.warn("Could not read replacement file: %s" % s) + return obj.replace(rex, s, flags=re.DOTALL) - - -class ReplaceFile(_ReplaceBase): - optionName = "replacement_files" - - def replace(self, obj, rex, s): - try: - v = open(s, "rb").read() - except IOError as e: - ctx.log.warn("Could not read replacement file: %s" % s) - return - obj.replace(rex, v, flags=re.DOTALL) diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index a7d3a312..cfbe5284 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -170,22 +170,23 @@ class Script: def load_script(self): self.ns = load_script(self.path, self.args) - ret = self.run("start") + ret = self.run("start", self.last_options) if ret: self.ns = ret - self.run("start") + self.run("start", self.last_options) def tick(self): if self.should_reload.is_set(): self.should_reload.clear() ctx.log.info("Reloading script: %s" % self.name) self.ns = load_script(self.path, self.args) - self.start() + self.start(self.last_options) self.configure(self.last_options, self.last_options.keys()) else: self.run("tick") - def start(self): + def start(self, opts): + self.last_options = opts self.load_script() def configure(self, options, updated): @@ -209,6 +210,12 @@ class ScriptLoader: """ An addon that manages loading scripts from options. """ + def __init__(self): + self.is_running = False + + def running(self): + self.is_running = True + def run_once(self, command, flows): try: sc = Script(command) @@ -267,3 +274,10 @@ class ScriptLoader: for s in newscripts: ctx.master.addons.startup(s) + if self.is_running: + # If we're already running, we configure and tell the addon + # we're up and running. + ctx.master.addons.invoke_with_context( + s, "configure", options, options.keys() + ) + ctx.master.addons.invoke_with_context(s, "running") diff --git a/mitmproxy/addons/termstatus.py b/mitmproxy/addons/termstatus.py new file mode 100644 index 00000000..7b05f409 --- /dev/null +++ b/mitmproxy/addons/termstatus.py @@ -0,0 +1,23 @@ +from mitmproxy import ctx + +""" + A tiny addon to print the proxy status to terminal. Eventually this could + also print some stats on exit. +""" + + +class TermStatus: + def __init__(self): + self.server = False + + def configure(self, options, updated): + if "server" in updated: + self.server = options.server + + def running(self): + if self.server: + ctx.log.info( + "Proxy server listening at http://{}:{}".format( + *ctx.master.server.address, + ) + ) diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index 6485eed7..618b34de 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -384,9 +384,6 @@ class SSLCert(serializable.Serializable): def __eq__(self, other): return self.digest("sha256") == other.digest("sha256") - def __ne__(self, other): - return not self.__eq__(other) - def get_state(self): return self.to_pem() diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py index 9359b67d..01721a71 100644 --- a/mitmproxy/connections.py +++ b/mitmproxy/connections.py @@ -1,6 +1,7 @@ import time import os +import uuid from mitmproxy import stateobject from mitmproxy import certs @@ -41,6 +42,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): self.clientcert = None self.ssl_established = None + self.id = str(uuid.uuid4()) self.mitmcert = None self.timestamp_start = time.time() self.timestamp_end = None @@ -73,6 +75,14 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): port=self.address[1], ) + def __eq__(self, other): + if isinstance(other, ClientConnection): + return self.id == other.id + return False + + def __hash__(self): + return hash(self.id) + @property def tls_established(self): return self.ssl_established @@ -82,6 +92,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): self.ssl_established = value _stateobject_attributes = dict( + id=str, address=tuple, ssl_established=bool, clientcert=certs.SSLCert, @@ -110,6 +121,7 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): @classmethod def make_dummy(cls, address): return cls.from_state(dict( + id=str(uuid.uuid4()), address=address, clientcert=None, mitmcert=None, @@ -165,6 +177,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def __init__(self, address, source_address=None, spoof_source_address=None): tcp.TCPClient.__init__(self, address, source_address, spoof_source_address) + self.id = str(uuid.uuid4()) self.alpn_proto_negotiated = None self.tls_version = None self.via = None @@ -196,6 +209,14 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): port=self.address[1], ) + def __eq__(self, other): + if isinstance(other, ServerConnection): + return self.id == other.id + return False + + def __hash__(self): + return hash(self.id) + @property def tls_established(self): return self.ssl_established @@ -205,6 +226,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.ssl_established = value _stateobject_attributes = dict( + id=str, address=tuple, ip_address=tuple, source_address=tuple, @@ -228,6 +250,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): @classmethod def make_dummy(cls, address): return cls.from_state(dict( + id=str(uuid.uuid4()), address=address, ip_address=address, cert=None, diff --git a/mitmproxy/eventsequence.py b/mitmproxy/eventsequence.py index 5872f607..bc6660e0 100644 --- a/mitmproxy/eventsequence.py +++ b/mitmproxy/eventsequence.py @@ -33,6 +33,7 @@ Events = frozenset([ "done", "log", "start", + "running", "tick", ]) diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index 6b68c8b2..bcc55559 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -93,7 +93,7 @@ class Flow(stateobject.StateObject): def get_state(self): d = super().get_state() - d.update(version=version.IVERSION) + d.update(version=version.FLOW_FORMAT_VERSION) if self._backup and self._backup != d: d.update(backup=self._backup) return d @@ -112,7 +112,6 @@ class Flow(stateobject.StateObject): def copy(self): f = super().copy() - f.id = str(uuid.uuid4()) f.live = False if self.reply is not None: f.reply = controller.DummyReply() diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py index 7c4f95f7..2c7fc52f 100644 --- a/mitmproxy/flowfilter.py +++ b/mitmproxy/flowfilter.py @@ -319,10 +319,14 @@ class FDomain(_Rex): code = "d" help = "Domain" flags = re.IGNORECASE + is_binary = False @only(http.HTTPFlow) def __call__(self, f): - return bool(self.re.search(f.request.data.host)) + return bool( + self.re.search(f.request.host) or + self.re.search(f.request.pretty_host) + ) class FUrl(_Rex): @@ -339,7 +343,7 @@ class FUrl(_Rex): @only(http.HTTPFlow) def __call__(self, f): - return self.re.search(f.request.url) + return self.re.search(f.request.pretty_url) class FSrc(_Rex): diff --git a/mitmproxy/io_compat.py b/mitmproxy/io_compat.py index 16cbc9fe..7d839ffd 100644 --- a/mitmproxy/io_compat.py +++ b/mitmproxy/io_compat.py @@ -1,8 +1,8 @@ """ This module handles the import of mitmproxy flows generated by old versions. """ - -from typing import Any +import uuid +from typing import Any, Dict from mitmproxy import version from mitmproxy.utils import strutils @@ -82,6 +82,9 @@ def convert_018_019(data): def convert_019_100(data): + # convert_unicode needs to be called for every dual release and the first py3-only release + data = convert_unicode(data) + data["version"] = (1, 0, 0) return data @@ -105,6 +108,30 @@ def convert_200_300(data): return data +def convert_300_4(data): + data["version"] = 4 + return data + + +client_connections = {} +server_connections = {} + + +def convert_4_5(data): + data["version"] = 5 + client_conn_key = ( + data["client_conn"]["timestamp_start"], + *data["client_conn"]["address"] + ) + server_conn_key = ( + data["server_conn"]["timestamp_start"], + *data["server_conn"]["source_address"] + ) + data["client_conn"]["id"] = client_connections.setdefault(client_conn_key, str(uuid.uuid4())) + data["server_conn"]["id"] = server_connections.setdefault(server_conn_key, str(uuid.uuid4())) + return data + + def _convert_dict_keys(o: Any) -> Any: if isinstance(o, dict): return {strutils.always_str(k): _convert_dict_keys(v) for k, v in o.items()} @@ -155,19 +182,33 @@ converters = { (0, 19): convert_019_100, (1, 0): convert_100_200, (2, 0): convert_200_300, + (3, 0): convert_300_4, + 4: convert_4_5, } -def migrate_flow(flow_data): +def migrate_flow(flow_data: Dict[str, Any]) -> Dict[str, Any]: while True: - flow_version = tuple(flow_data.get(b"version", flow_data.get("version"))) - if flow_version[:2] == version.IVERSION[:2]: + flow_version = flow_data.get(b"version", flow_data.get("version")) + + # Historically, we used the mitmproxy minor version tuple as the flow format version. + if not isinstance(flow_version, int): + flow_version = tuple(flow_version)[:2] + + if flow_version == version.FLOW_FORMAT_VERSION: break - elif flow_version[:2] in converters: - flow_data = converters[flow_version[:2]](flow_data) + elif flow_version in converters: + flow_data = converters[flow_version](flow_data) else: - v = ".".join(str(i) for i in flow_version) + should_upgrade = ( + isinstance(flow_version, int) + and flow_version > version.FLOW_FORMAT_VERSION + ) raise ValueError( - "{} cannot read files serialized with version {}.".format(version.MITMPROXY, v) + "{} cannot read files with flow format version {}{}.".format( + version.MITMPROXY, + flow_version, + ", please update mitmproxy" if should_upgrade else "" + ) ) return flow_data diff --git a/mitmproxy/master.py b/mitmproxy/master.py index 8855452c..79747a97 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -42,6 +42,7 @@ class Master: self.event_queue = queue.Queue() self.should_exit = threading.Event() self.server = server + self.first_tick = True channel = controller.Channel(self.event_queue, self.should_exit) server.set_channel(channel) @@ -86,6 +87,9 @@ class Master: self.shutdown() def tick(self, timeout): + if self.first_tick: + self.first_tick = False + self.addons.invoke_all_with_context("running") with self.handlecontext(): self.addons("tick") changed = False diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index c0a78ea9..506674d6 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -13,9 +13,6 @@ class MessageData(serializable.Serializable): return self.__dict__ == other.__dict__ return False - def __ne__(self, other): - return not self.__eq__(other) - def set_state(self, state): for k, v in state.items(): if k == "headers": @@ -39,9 +36,6 @@ class Message(serializable.Serializable): return self.data == other.data return False - def __ne__(self, other): - return not self.__eq__(other) - def get_state(self): return self.data.get_state() diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 6dd8616b..036b3d29 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -78,7 +78,7 @@ class Options(optmanager.OptManager): "Kill extra requests during replay." ) self.add_option( - "keepserving", bool, True, + "keepserving", bool, False, "Continue serving after client playback or file read." ) self.add_option( @@ -121,13 +121,6 @@ class Options(optmanager.OptManager): """ ) self.add_option( - "replacement_files", Sequence[str], [], - """ - Replacement pattern, where the replacement clause is a path to a - file. - """ - ) - self.add_option( "server_replay_use_headers", Sequence[str], [], "Request headers to be considered during replay." ) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 9553bd32..495354f4 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -81,8 +81,6 @@ class _Option: class OptManager: """ OptManager is the base class from which Options objects are derived. - Note that the __init__ method of all child classes must force all - arguments to be positional only, by including a "*" argument. .changed is a blinker Signal that triggers whenever options are updated. If any handler in the chain raises an exceptions.OptionsError @@ -176,15 +174,29 @@ class OptManager: o.reset() self.changed.send(self._options.keys()) + def update_known(self, **kwargs): + """ + Update and set all known options from kwargs. Returns a dictionary + of unknown options. + """ + known, unknown = {}, {} + for k, v in kwargs.items(): + if k in self._options: + known[k] = v + else: + unknown[k] = v + updated = set(known.keys()) + if updated: + with self.rollback(updated): + for k, v in known.items(): + self._options[k].set(v) + self.changed.send(self, updated=updated) + return unknown + def update(self, **kwargs): - updated = set(kwargs.keys()) - with self.rollback(updated): - for k, v in kwargs.items(): - if k not in self._options: - raise KeyError("No such option: %s" % k) - self._options[k].set(v) - self.changed.send(self, updated=updated) - return self + u = self.update_known(**kwargs) + if u: + raise KeyError("Unknown options: %s" % ", ".join(u.keys())) def setter(self, attr): """ @@ -222,83 +234,6 @@ class OptManager: """ return self._options[option].has_changed() - def save(self, path, defaults=False): - """ - Save to path. If the destination file exists, modify it in-place. - """ - if os.path.exists(path) and os.path.isfile(path): - with open(path, "r") as f: - data = f.read() - else: - data = "" - data = self.serialize(data, defaults) - with open(path, "w") as f: - f.write(data) - - def serialize(self, text, defaults=False): - """ - Performs a round-trip serialization. If text is not None, it is - treated as a previous serialization that should be modified - in-place. - - - If "defaults" is False, only options with non-default values are - serialized. Default values in text are preserved. - - Unknown options in text are removed. - - Raises OptionsError if text is invalid. - """ - data = self._load(text) - for k in self.keys(): - if defaults or self.has_changed(k): - data[k] = getattr(self, k) - for k in list(data.keys()): - if k not in self._options: - del data[k] - return ruamel.yaml.round_trip_dump(data) - - def _load(self, text): - if not text: - return {} - try: - data = ruamel.yaml.load(text, ruamel.yaml.RoundTripLoader) - except ruamel.yaml.error.YAMLError as v: - snip = v.problem_mark.get_snippet() - raise exceptions.OptionsError( - "Config error at line %s:\n%s\n%s" % - (v.problem_mark.line + 1, snip, v.problem) - ) - if isinstance(data, str): - raise exceptions.OptionsError("Config error - no keys found.") - return data - - def load(self, text): - """ - Load configuration from text, over-writing options already set in - this object. May raise OptionsError if the config file is invalid. - """ - data = self._load(text) - try: - self.update(**data) - except KeyError as v: - raise exceptions.OptionsError(v) - - def load_paths(self, *paths): - """ - Load paths in order. Each path takes precedence over the previous - path. Paths that don't exist are ignored, errors raise an - OptionsError. - """ - for p in paths: - p = os.path.expanduser(p) - if os.path.exists(p) and os.path.isfile(p): - with open(p, "r") as f: - txt = f.read() - try: - self.load(txt) - except exceptions.OptionsError as e: - raise exceptions.OptionsError( - "Error reading %s: %s" % (p, e) - ) - def merge(self, opts): """ Merge a dict of options into this object. Options that have None @@ -324,23 +259,33 @@ class OptManager: options=options ) - def set(self, spec): + def set(self, *spec): + vals = {} + for i in spec: + vals.update(self._setspec(i)) + self.update(**vals) + + def _setspec(self, spec): + d = {} + parts = spec.split("=", maxsplit=1) if len(parts) == 1: optname, optval = parts[0], None else: optname, optval = parts[0], parts[1] + if optname not in self._options: + raise exceptions.OptionsError("No such option %s" % optname) o = self._options[optname] if o.typespec in (str, typing.Optional[str]): - setattr(self, optname, optval) + d[optname] = optval elif o.typespec in (int, typing.Optional[int]): if optval: try: optval = int(optval) except ValueError: raise exceptions.OptionsError("Not an integer: %s" % optval) - setattr(self, optname, optval) + d[optname] = optval elif o.typespec == bool: if not optval or optval == "true": v = True @@ -350,18 +295,15 @@ class OptManager: raise exceptions.OptionsError( "Boolean must be \"true\", \"false\", or have the value " "omitted (a synonym for \"true\")." ) - setattr(self, optname, v) + d[optname] = v elif o.typespec == typing.Sequence[str]: if not optval: - setattr(self, optname, []) + d[optname] = [] else: - setattr( - self, - optname, - getattr(self, optname) + [optval] - ) + d[optname] = getattr(self, optname) + [optval] else: # pragma: no cover raise NotImplementedError("Unsupported option type: %s", o.typespec) + return d def make_parser(self, parser, optname, metavar=None, short=None): o = self._options[optname] @@ -430,7 +372,7 @@ class OptManager: raise ValueError("Unsupported option type: %s", o.typespec) -def dump(opts): +def dump_defaults(opts): """ Dumps an annotated file with all options. """ @@ -461,3 +403,88 @@ def dump(opts): ) s.yaml_set_comment_before_after_key(k, before = "\n" + txt) return ruamel.yaml.round_trip_dump(s) + + +def parse(text): + if not text: + return {} + try: + data = ruamel.yaml.load(text, ruamel.yaml.RoundTripLoader) + except ruamel.yaml.error.YAMLError as v: + snip = v.problem_mark.get_snippet() + raise exceptions.OptionsError( + "Config error at line %s:\n%s\n%s" % + (v.problem_mark.line + 1, snip, v.problem) + ) + if isinstance(data, str): + raise exceptions.OptionsError("Config error - no keys found.") + return data + + +def load(opts, text): + """ + Load configuration from text, over-writing options already set in + this object. May raise OptionsError if the config file is invalid. + + Returns a dictionary of all unknown options. + """ + data = parse(text) + return opts.update_known(**data) + + +def load_paths(opts, *paths): + """ + Load paths in order. Each path takes precedence over the previous + path. Paths that don't exist are ignored, errors raise an + OptionsError. + + Returns a dictionary of unknown options. + """ + ret = {} + for p in paths: + p = os.path.expanduser(p) + if os.path.exists(p) and os.path.isfile(p): + with open(p, "r") as f: + txt = f.read() + try: + ret.update(load(opts, txt)) + except exceptions.OptionsError as e: + raise exceptions.OptionsError( + "Error reading %s: %s" % (p, e) + ) + return ret + + +def serialize(opts, text, defaults=False): + """ + Performs a round-trip serialization. If text is not None, it is + treated as a previous serialization that should be modified + in-place. + + - If "defaults" is False, only options with non-default values are + serialized. Default values in text are preserved. + - Unknown options in text are removed. + - Raises OptionsError if text is invalid. + """ + data = parse(text) + for k in opts.keys(): + if defaults or opts.has_changed(k): + data[k] = getattr(opts, k) + for k in list(data.keys()): + if k not in opts._options: + del data[k] + return ruamel.yaml.round_trip_dump(data) + + +def save(opts, path, defaults=False): + """ + Save to path. If the destination file exists, modify it in-place. + """ + if os.path.exists(path) and os.path.isfile(path): + with open(path, "r") as f: + data = f.read() + else: + data = "" + data = serialize(opts, data, defaults) + with open(path, "w") as f: + f.write(data) diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py index 7fbe1727..270021cb 100644 --- a/mitmproxy/test/tflow.py +++ b/mitmproxy/test/tflow.py @@ -1,4 +1,5 @@ import io +import uuid from mitmproxy.net import websockets from mitmproxy.test import tutils @@ -146,6 +147,7 @@ def tclient_conn(): @return: mitmproxy.proxy.connection.ClientConnection """ c = connections.ClientConnection.from_state(dict( + id=str(uuid.uuid4()), address=("address", 22), clientcert=None, mitmcert=None, @@ -169,6 +171,7 @@ def tserver_conn(): @return: mitmproxy.proxy.connection.ServerConnection """ c = connections.ServerConnection.from_state(dict( + id=str(uuid.uuid4()), address=("address", 22), source_address=("address", 22), ip_address=None, diff --git a/mitmproxy/test/tutils.py b/mitmproxy/test/tutils.py index 7b311492..80e5b6fd 100644 --- a/mitmproxy/test/tutils.py +++ b/mitmproxy/test/tutils.py @@ -1,9 +1,5 @@ -from io import BytesIO -import tempfile -import os import time -import shutil -from contextlib import contextmanager +from io import BytesIO from mitmproxy.utils import data from mitmproxy.net import tcp @@ -13,18 +9,6 @@ from mitmproxy.net import http test_data = data.Data(__name__).push("../../test/") -@contextmanager -def tmpdir(*args, **kwargs): - orig_workdir = os.getcwd() - temp_workdir = tempfile.mkdtemp(*args, **kwargs) - os.chdir(temp_workdir) - - yield temp_workdir - - os.chdir(orig_workdir) - shutil.rmtree(temp_workdir) - - def treader(bytes): """ Construct a tcp.Read object from bytes. diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index aaefd10a..da091c12 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -23,7 +23,7 @@ def common_options(parser, opts): parser.add_argument( '--options', action='store_true', - help="Dump all options", + help="Show all options and their default values", ) parser.add_argument( "--conf", @@ -93,7 +93,6 @@ def common_options(parser, opts): # Replacements group = parser.add_argument_group("Replacements") opts.make_parser(group, "replacements", metavar="PATTERN", short="R") - opts.make_parser(group, "replacement_files", metavar="PATTERN") # Set headers group = parser.add_argument_group("Set Headers") diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index 2d24cf86..0d9929ae 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -1,5 +1,8 @@ +import os import re + import urwid + from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy.addons import script @@ -87,6 +90,9 @@ class ReplaceEditor(base.GridEditor): re.compile(val) except re.error: return "Invalid regular expression." + elif col == 2: + if val.startswith("@") and not os.path.isfile(os.path.expanduser(val[1:])): + return "Invalid file path" return False diff --git a/mitmproxy/tools/console/options.py b/mitmproxy/tools/console/options.py index 33e3ec38..79bb53c2 100644 --- a/mitmproxy/tools/console/options.py +++ b/mitmproxy/tools/console/options.py @@ -1,6 +1,7 @@ import urwid from mitmproxy import contentviews +from mitmproxy import optmanager from mitmproxy.tools.console import common from mitmproxy.tools.console import grideditor from mitmproxy.tools.console import select @@ -173,7 +174,7 @@ class Options(urwid.WidgetWrap): return super().keypress(size, key) def do_save(self, path): - self.master.options.save(path) + optmanager.save(self.master.options, path) return "Saved" def save(self): diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index e70ce2f9..4bfe2dc4 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -3,7 +3,7 @@ from mitmproxy import exceptions from mitmproxy import addons from mitmproxy import options from mitmproxy import master -from mitmproxy.addons import dumper, termlog +from mitmproxy.addons import dumper, termlog, termstatus class DumpMaster(master.Master): @@ -18,17 +18,11 @@ class DumpMaster(master.Master): master.Master.__init__(self, options, server) self.has_errored = False if with_termlog: - self.addons.add(termlog.TermLog()) + self.addons.add(termlog.TermLog(), termstatus.TermStatus()) self.addons.add(*addons.default_addons()) if with_dumper: self.addons.add(dumper.Dumper()) - if self.options.server: - self.add_log( - "Proxy server listening at http://{}:{}".format(server.address[0], server.address[1]), - "info" - ) - if options.rfile: try: self.load_flows_file(options.rfile) diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 17c1abbb..35567b62 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -39,15 +39,10 @@ def process_options(parser, opts, args): if args.version: print(debug.dump_system_info()) sys.exit(0) - if args.options: - print(optmanager.dump(opts)) - sys.exit(0) - if args.quiet: + if args.quiet or args.options: + args.verbosity = 0 args.flow_detail = 0 - for i in args.setoptions: - opts.set(i) - adict = {} for n in dir(args): if n in opts: @@ -74,9 +69,17 @@ def run(MasterKlass, args): # pragma: no cover args = parser.parse_args(args) master = None try: - opts.load_paths(args.conf) + unknown = optmanager.load_paths(opts, args.conf) server = process_options(parser, opts, args) master = MasterKlass(opts, server) + master.addons.configure_all(opts, opts.keys()) + remaining = opts.update_known(**unknown) + if remaining and opts.verbosity > 1: + print("Ignored options: %s" % remaining) + if args.options: + print(optmanager.dump_defaults(opts)) + sys.exit(0) + opts.set(*args.setoptions) def cleankill(*args, **kwargs): master.shutdown() diff --git a/mitmproxy/types/multidict.py b/mitmproxy/types/multidict.py index 31a1f22b..c4f42580 100644 --- a/mitmproxy/types/multidict.py +++ b/mitmproxy/types/multidict.py @@ -4,7 +4,7 @@ from collections.abc import MutableMapping from mitmproxy.types import serializable -class _MultiDict(MutableMapping, serializable.Serializable, metaclass=ABCMeta): +class _MultiDict(MutableMapping, metaclass=ABCMeta): def __repr__(self): fields = ( repr(field) @@ -67,9 +67,6 @@ class _MultiDict(MutableMapping, serializable.Serializable, metaclass=ABCMeta): return self.fields == other.fields return False - def __ne__(self, other): - return not self.__eq__(other) - def get_all(self, key): """ Return the list of all values for a given key. @@ -174,18 +171,8 @@ class _MultiDict(MutableMapping, serializable.Serializable, metaclass=ABCMeta): coll.append([key, values]) return coll - def get_state(self): - return self.fields - - def set_state(self, state): - self.fields = tuple(tuple(x) for x in state) - - @classmethod - def from_state(cls, state): - return cls(state) - -class MultiDict(_MultiDict): +class MultiDict(_MultiDict, serializable.Serializable): def __init__(self, fields=()): super().__init__() self.fields = tuple( @@ -200,6 +187,16 @@ class MultiDict(_MultiDict): def _kconv(key): return key + def get_state(self): + return self.fields + + def set_state(self, state): + self.fields = tuple(tuple(x) for x in state) + + @classmethod + def from_state(cls, state): + return cls(state) + class MultiDictView(_MultiDict): """ @@ -230,3 +227,6 @@ class MultiDictView(_MultiDict): @fields.setter def fields(self, value): self._setter(value) + + def copy(self): + return MultiDict(self.fields) diff --git a/mitmproxy/types/serializable.py b/mitmproxy/types/serializable.py index 49892ffc..cd8539b0 100644 --- a/mitmproxy/types/serializable.py +++ b/mitmproxy/types/serializable.py @@ -1,4 +1,5 @@ import abc +import uuid class Serializable(metaclass=abc.ABCMeta): @@ -29,4 +30,7 @@ class Serializable(metaclass=abc.ABCMeta): raise NotImplementedError() def copy(self): - return self.from_state(self.get_state()) + state = self.get_state() + if isinstance(state, dict) and "id" in state: + state["id"] = str(uuid.uuid4()) + return self.from_state(state) diff --git a/mitmproxy/version.py b/mitmproxy/version.py index d23b2d19..006ec868 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -3,5 +3,9 @@ VERSION = ".".join(str(i) for i in IVERSION) PATHOD = "pathod " + VERSION MITMPROXY = "mitmproxy " + VERSION +# Serialization format version. This is displayed nowhere, it just needs to be incremented by one +# for each change the the file format. +FLOW_FORMAT_VERSION = 5 + if __name__ == "__main__": print(VERSION) diff --git a/test/mitmproxy/addons/test_clientplayback.py b/test/mitmproxy/addons/test_clientplayback.py index 6b8b7c90..c22b3589 100644 --- a/test/mitmproxy/addons/test_clientplayback.py +++ b/test/mitmproxy/addons/test_clientplayback.py @@ -1,9 +1,7 @@ -import os import pytest from unittest import mock from mitmproxy.test import tflow -from mitmproxy.test import tutils from mitmproxy import io from mitmproxy import exceptions @@ -49,14 +47,13 @@ class TestClientPlayback: cp.tick() assert cp.current_thread is None - def test_configure(self): + def test_configure(self, tmpdir): cp = clientplayback.ClientPlayback() with taddons.context() as tctx: - with tutils.tmpdir() as td: - path = os.path.join(td, "flows") - tdump(path, [tflow.tflow()]) - tctx.configure(cp, client_replay=[path]) - tctx.configure(cp, client_replay=[]) - tctx.configure(cp) - with pytest.raises(exceptions.OptionsError): - tctx.configure(cp, client_replay=["nonexistent"]) + path = str(tmpdir.join("flows")) + tdump(path, [tflow.tflow()]) + tctx.configure(cp, client_replay=[path]) + tctx.configure(cp, client_replay=[]) + tctx.configure(cp) + with pytest.raises(exceptions.OptionsError): + tctx.configure(cp, client_replay=["nonexistent"]) diff --git a/test/mitmproxy/addons/test_replace.py b/test/mitmproxy/addons/test_replace.py index 8c280c51..7d590b35 100644 --- a/test/mitmproxy/addons/test_replace.py +++ b/test/mitmproxy/addons/test_replace.py @@ -1,11 +1,8 @@ -import os.path import pytest -from mitmproxy.test import tflow -from mitmproxy.test import tutils -from .. import tservers from mitmproxy.addons import replace from mitmproxy.test import taddons +from mitmproxy.test import tflow class TestReplace: @@ -34,7 +31,7 @@ class TestReplace: with taddons.context() as tctx: tctx.configure( r, - replacements = [ + replacements=[ "/~q/foo/bar", "/~s/foo/bar", ] @@ -49,55 +46,57 @@ class TestReplace: r.response(f) assert f.response.content == b"bar" - -class TestUpstreamProxy(tservers.HTTPUpstreamProxyTest): - ssl = False - def test_order(self): - sa = replace.Replace() - self.proxy.tmaster.addons.add(sa) - - self.proxy.tmaster.options.replacements = [ - "/~q/foo/bar", - "/~q/bar/baz", - "/~q/foo/oh noes!", - "/~s/baz/ORLY" - ] - p = self.pathoc() - with p.connect(): - req = p.request("get:'%s/p/418:b\"foo\"'" % self.server.urlbase) - assert req.content == b"ORLY" - assert req.status_code == 418 + r = replace.Replace() + with taddons.context() as tctx: + tctx.configure( + r, + replacements=[ + "/foo/bar", + "/bar/baz", + "/foo/oh noes!", + "/bar/oh noes!", + ] + ) + f = tflow.tflow() + f.request.content = b"foo" + r.request(f) + assert f.request.content == b"baz" class TestReplaceFile: - def test_simple(self): - r = replace.ReplaceFile() - with tutils.tmpdir() as td: - rp = os.path.join(td, "replacement") - with open(rp, "w") as f: - f.write("bar") - with taddons.context() as tctx: + def test_simple(self, tmpdir): + r = replace.Replace() + with taddons.context() as tctx: + tmpfile = tmpdir.join("replacement") + tmpfile.write("bar") + tctx.configure( + r, + replacements=["/~q/foo/@" + str(tmpfile)] + ) + f = tflow.tflow() + f.request.content = b"foo" + r.request(f) + assert f.request.content == b"bar" + + def test_nonexistent(self, tmpdir): + r = replace.Replace() + with taddons.context() as tctx: + with pytest.raises(Exception, match="Invalid file path"): tctx.configure( r, - replacement_files = [ - "/~q/foo/" + rp, - "/~s/foo/" + rp, - "/~b nonexistent/nonexistent/nonexistent", - ] + replacements=["/~q/foo/@nonexistent"] ) - f = tflow.tflow() - f.request.content = b"foo" - r.request(f) - assert f.request.content == b"bar" - - f = tflow.tflow(resp=True) - f.response.content = b"foo" - r.response(f) - assert f.response.content == b"bar" - f = tflow.tflow() - f.request.content = b"nonexistent" - assert not tctx.master.event_log - r.request(f) - assert tctx.master.event_log + tmpfile = tmpdir.join("replacement") + tmpfile.write("bar") + tctx.configure( + r, + replacements=["/~q/foo/@" + str(tmpfile)] + ) + tmpfile.remove() + f = tflow.tflow() + f.request.content = b"foo" + assert not tctx.master.event_log + r.request(f) + assert tctx.master.event_log diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index 5f196ebf..4c1b2e43 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -68,13 +68,12 @@ class TestParseCommand: with pytest.raises(ValueError): script.parse_command(" ") - def test_no_script_file(self): + def test_no_script_file(self, tmpdir): with pytest.raises(Exception, match="not found"): script.parse_command("notfound") - with tutils.tmpdir() as dir: - with pytest.raises(Exception, match="Not a file"): - script.parse_command(dir) + with pytest.raises(Exception, match="Not a file"): + script.parse_command(str(tmpdir)) def test_parse_args(self): with utils.chdir(tutils.test_data.dirname): @@ -117,9 +116,7 @@ class TestScript: ) ) sc.load_script() - assert sc.ns.call_log == [ - ("solo", "start", (), {}), - ] + assert sc.ns.call_log[0][0:2] == ("solo", "start") sc.ns.call_log = [] f = tflow.tflow(resp=True) @@ -128,28 +125,26 @@ class TestScript: recf = sc.ns.call_log[0] assert recf[1] == "request" - def test_reload(self): + def test_reload(self, tmpdir): with taddons.context() as tctx: - with tutils.tmpdir(): - with open("foo.py", "w"): - pass - sc = script.Script("foo.py") - tctx.configure(sc) - for _ in range(100): - with open("foo.py", "a") as f: - f.write(".") - sc.tick() - time.sleep(0.1) - if tctx.master.event_log: - return - raise AssertionError("Change event not detected.") + f = tmpdir.join("foo.py") + f.ensure(file=True) + sc = script.Script(str(f)) + tctx.configure(sc) + for _ in range(100): + f.write(".") + sc.tick() + time.sleep(0.1) + if tctx.master.event_log: + return + raise AssertionError("Change event not detected.") def test_exception(self): with taddons.context() as tctx: sc = script.Script( tutils.test_data.path("mitmproxy/data/addonscripts/error.py") ) - sc.start() + sc.start(tctx.options) f = tflow.tflow(resp=True) sc.request(f) assert tctx.master.event_log[0][0] == "error" @@ -165,7 +160,7 @@ class TestScript: "mitmproxy/data/addonscripts/addon.py" ) ) - sc.start() + sc.start(tctx.options) tctx.configure(sc) assert sc.ns.event_log == [ 'scriptstart', 'addonstart', 'addonconfigure' @@ -228,24 +223,31 @@ class TestScriptLoader: assert len(m.addons) == 1 def test_dupes(self): - o = options.Options(scripts=["one", "one"]) - m = master.Master(o, proxy.DummyServer()) sc = script.ScriptLoader() - with pytest.raises(exceptions.OptionsError): - m.addons.add(o, sc) + with taddons.context() as tctx: + tctx.master.addons.add(sc) + with pytest.raises(exceptions.OptionsError): + tctx.configure( + sc, + scripts = ["one", "one"] + ) def test_nonexistent(self): - o = options.Options(scripts=["nonexistent"]) - m = master.Master(o, proxy.DummyServer()) sc = script.ScriptLoader() - with pytest.raises(exceptions.OptionsError): - m.addons.add(o, sc) + with taddons.context() as tctx: + tctx.master.addons.add(sc) + with pytest.raises(exceptions.OptionsError): + tctx.configure( + sc, + scripts = ["nonexistent"] + ) def test_order(self): rec = tutils.test_data.path("mitmproxy/data/addonscripts/recorder.py") sc = script.ScriptLoader() with taddons.context() as tctx: tctx.master.addons.add(sc) + sc.running() tctx.configure( sc, scripts = [ @@ -256,9 +258,17 @@ class TestScriptLoader: ) debug = [(i[0], i[1]) for i in tctx.master.event_log if i[0] == "debug"] assert debug == [ - ('debug', 'a start'), ('debug', 'a configure'), - ('debug', 'b start'), ('debug', 'b configure'), - ('debug', 'c start'), ('debug', 'c configure') + ('debug', 'a start'), + ('debug', 'a configure'), + ('debug', 'a running'), + + ('debug', 'b start'), + ('debug', 'b configure'), + ('debug', 'b running'), + + ('debug', 'c start'), + ('debug', 'c configure'), + ('debug', 'c running'), ] tctx.master.event_log = [] tctx.configure( @@ -287,4 +297,5 @@ class TestScriptLoader: ('debug', 'b done'), ('debug', 'x start'), ('debug', 'x configure'), + ('debug', 'x running'), ] diff --git a/test/mitmproxy/addons/test_serverplayback.py b/test/mitmproxy/addons/test_serverplayback.py index e2afa516..54e4d281 100644 --- a/test/mitmproxy/addons/test_serverplayback.py +++ b/test/mitmproxy/addons/test_serverplayback.py @@ -1,10 +1,8 @@ -import os import urllib import pytest -from mitmproxy.test import tutils -from mitmproxy.test import tflow from mitmproxy.test import taddons +from mitmproxy.test import tflow import mitmproxy.test.tutils from mitmproxy.addons import serverplayback @@ -19,15 +17,14 @@ def tdump(path, flows): w.add(i) -def test_config(): +def test_config(tmpdir): s = serverplayback.ServerPlayback() - with tutils.tmpdir() as p: - with taddons.context() as tctx: - fpath = os.path.join(p, "flows") - tdump(fpath, [tflow.tflow(resp=True)]) - tctx.configure(s, server_replay=[fpath]) - with pytest.raises(exceptions.OptionsError): - tctx.configure(s, server_replay=[p]) + with taddons.context() as tctx: + fpath = str(tmpdir.join("flows")) + tdump(fpath, [tflow.tflow(resp=True)]) + tctx.configure(s, server_replay=[fpath]) + with pytest.raises(exceptions.OptionsError): + tctx.configure(s, server_replay=[str(tmpdir)]) def test_tick(): diff --git a/test/mitmproxy/addons/test_streamfile.py b/test/mitmproxy/addons/test_streamfile.py index 4105c1fc..3f78521c 100644 --- a/test/mitmproxy/addons/test_streamfile.py +++ b/test/mitmproxy/addons/test_streamfile.py @@ -1,9 +1,7 @@ -import os.path import pytest -from mitmproxy.test import tflow -from mitmproxy.test import tutils from mitmproxy.test import taddons +from mitmproxy.test import tflow from mitmproxy import io from mitmproxy import exceptions @@ -11,19 +9,17 @@ from mitmproxy import options from mitmproxy.addons import streamfile -def test_configure(): +def test_configure(tmpdir): sa = streamfile.StreamFile() with taddons.context(options=options.Options()) as tctx: - with tutils.tmpdir() as tdir: - p = os.path.join(tdir, "foo") - with pytest.raises(exceptions.OptionsError): - tctx.configure(sa, streamfile=tdir) - with pytest.raises(Exception, match="Invalid filter"): - tctx.configure(sa, streamfile=p, filtstr="~~") - tctx.configure(sa, filtstr="foo") - assert sa.filt - tctx.configure(sa, filtstr=None) - assert not sa.filt + with pytest.raises(exceptions.OptionsError): + tctx.configure(sa, streamfile=str(tmpdir)) + with pytest.raises(Exception, match="Invalid filter"): + tctx.configure(sa, streamfile=str(tmpdir.join("foo")), filtstr="~~") + tctx.configure(sa, filtstr="foo") + assert sa.filt + tctx.configure(sa, filtstr=None) + assert not sa.filt def rd(p): @@ -31,36 +27,34 @@ def rd(p): return list(x.stream()) -def test_tcp(): +def test_tcp(tmpdir): sa = streamfile.StreamFile() with taddons.context() as tctx: - with tutils.tmpdir() as tdir: - p = os.path.join(tdir, "foo") - tctx.configure(sa, streamfile=p) + p = str(tmpdir.join("foo")) + tctx.configure(sa, streamfile=p) - tt = tflow.ttcpflow() - sa.tcp_start(tt) - sa.tcp_end(tt) - tctx.configure(sa, streamfile=None) - assert rd(p) + tt = tflow.ttcpflow() + sa.tcp_start(tt) + sa.tcp_end(tt) + tctx.configure(sa, streamfile=None) + assert rd(p) -def test_simple(): +def test_simple(tmpdir): sa = streamfile.StreamFile() with taddons.context() as tctx: - with tutils.tmpdir() as tdir: - p = os.path.join(tdir, "foo") + p = str(tmpdir.join("foo")) - tctx.configure(sa, streamfile=p) + tctx.configure(sa, streamfile=p) - f = tflow.tflow(resp=True) - sa.request(f) - sa.response(f) - tctx.configure(sa, streamfile=None) - assert rd(p)[0].response + f = tflow.tflow(resp=True) + sa.request(f) + sa.response(f) + tctx.configure(sa, streamfile=None) + assert rd(p)[0].response - tctx.configure(sa, streamfile="+" + p) - f = tflow.tflow() - sa.request(f) - tctx.configure(sa, streamfile=None) - assert not rd(p)[1].response + tctx.configure(sa, streamfile="+" + p) + f = tflow.tflow() + sa.request(f) + tctx.configure(sa, streamfile=None) + assert not rd(p)[1].response diff --git a/test/mitmproxy/addons/test_termstatus.py b/test/mitmproxy/addons/test_termstatus.py new file mode 100644 index 00000000..01c14814 --- /dev/null +++ b/test/mitmproxy/addons/test_termstatus.py @@ -0,0 +1,12 @@ +from mitmproxy.addons import termstatus +from mitmproxy.test import taddons + + +def test_configure(): + ts = termstatus.TermStatus() + with taddons.context() as ctx: + ts.running() + assert not ctx.master.event_log + ctx.configure(ts, server=True) + ts.running() + assert ctx.master.event_log diff --git a/test/mitmproxy/data/addonscripts/addon.py b/test/mitmproxy/data/addonscripts/addon.py index 84173cb6..f34f41cb 100644 --- a/test/mitmproxy/data/addonscripts/addon.py +++ b/test/mitmproxy/data/addonscripts/addon.py @@ -6,7 +6,7 @@ class Addon: def event_log(self): return event_log - def start(self): + def start(self, opts): event_log.append("addonstart") def configure(self, options, updated): @@ -17,6 +17,6 @@ def configure(options, updated): event_log.append("addonconfigure") -def start(): +def start(opts): event_log.append("scriptstart") return Addon() diff --git a/test/mitmproxy/data/addonscripts/concurrent_decorator_class.py b/test/mitmproxy/data/addonscripts/concurrent_decorator_class.py index bd047c99..10ba24cd 100644 --- a/test/mitmproxy/data/addonscripts/concurrent_decorator_class.py +++ b/test/mitmproxy/data/addonscripts/concurrent_decorator_class.py @@ -9,5 +9,5 @@ class ConcurrentClass: time.sleep(0.1) -def start(): +def start(opts): return ConcurrentClass() diff --git a/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py b/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py index 756869c8..7bc28182 100644 --- a/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py +++ b/test/mitmproxy/data/addonscripts/concurrent_decorator_err.py @@ -2,5 +2,5 @@ from mitmproxy.script import concurrent @concurrent -def start(): +def start(opts): pass diff --git a/test/mitmproxy/data/addonscripts/recorder.py b/test/mitmproxy/data/addonscripts/recorder.py index 6b9b6ea8..aff524a8 100644 --- a/test/mitmproxy/data/addonscripts/recorder.py +++ b/test/mitmproxy/data/addonscripts/recorder.py @@ -22,5 +22,5 @@ class CallLogger: raise AttributeError -def start(): +def start(opts): return CallLogger(*sys.argv[1:]) diff --git a/test/mitmproxy/net/http/test_message.py b/test/mitmproxy/net/http/test_message.py index 034bd600..b75bc7c2 100644 --- a/test/mitmproxy/net/http/test_message.py +++ b/test/mitmproxy/net/http/test_message.py @@ -38,14 +38,12 @@ def _test_decoded_attr(message, attr): class TestMessageData: - def test_eq_ne(self): + def test_eq(self): data = tutils.tresp(timestamp_start=42, timestamp_end=42).data same = tutils.tresp(timestamp_start=42, timestamp_end=42).data assert data == same - assert not data != same other = tutils.tresp(content=b"foo").data - assert not data == other assert data != other assert data != 0 @@ -61,10 +59,8 @@ class TestMessage: resp = tutils.tresp(timestamp_start=42, timestamp_end=42) same = tutils.tresp(timestamp_start=42, timestamp_end=42) assert resp == same - assert not resp != same other = tutils.tresp(timestamp_start=0, timestamp_end=0) - assert not resp == other assert resp != other assert resp != 0 diff --git a/test/mitmproxy/net/test_tcp.py b/test/mitmproxy/net/test_tcp.py index cf010f6e..8b26784a 100644 --- a/test/mitmproxy/net/test_tcp.py +++ b/test/mitmproxy/net/test_tcp.py @@ -11,8 +11,8 @@ from OpenSSL import SSL from mitmproxy import certs from mitmproxy.net import tcp -from mitmproxy.test import tutils from mitmproxy import exceptions +from mitmproxy.test import tutils from . import tservers from ...conftest import requires_alpn @@ -783,25 +783,24 @@ class TestSSLKeyLogger(tservers.ServerTestBase): cipher_list="AES256-SHA" ) - def test_log(self): + def test_log(self, tmpdir): testval = b"echo!\n" _logfun = tcp.log_ssl_key - with tutils.tmpdir() as d: - logfile = os.path.join(d, "foo", "bar", "logfile") - tcp.log_ssl_key = tcp.SSLKeyLogger(logfile) + logfile = str(tmpdir.join("foo", "bar", "logfile")) + tcp.log_ssl_key = tcp.SSLKeyLogger(logfile) - c = tcp.TCPClient(("127.0.0.1", self.port)) - with c.connect(): - c.convert_to_ssl() - c.wfile.write(testval) - c.wfile.flush() - assert c.rfile.readline() == testval - c.finish() - - tcp.log_ssl_key.close() - with open(logfile, "rb") as f: - assert f.read().count(b"CLIENT_RANDOM") == 2 + c = tcp.TCPClient(("127.0.0.1", self.port)) + with c.connect(): + c.convert_to_ssl() + c.wfile.write(testval) + c.wfile.flush() + assert c.rfile.readline() == testval + c.finish() + + tcp.log_ssl_key.close() + with open(logfile, "rb") as f: + assert f.read().count(b"CLIENT_RANDOM") == 2 tcp.log_ssl_key = _logfun diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index aa45761a..16efe415 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -302,6 +302,9 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin): class TestHTTPAuth(tservers.HTTPProxyTest): def test_auth(self): self.master.addons.add(proxyauth.ProxyAuth()) + self.master.addons.configure_all( + self.master.options, self.master.options.keys() + ) self.master.options.proxyauth = "test:test" assert self.pathod("202").status_code == 407 p = self.pathoc() diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index e81c023d..a9b6f0c4 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -24,7 +24,7 @@ class TestConcurrent(tservers.MasterTest): "mitmproxy/data/addonscripts/concurrent_decorator.py" ) ) - sc.start() + sc.start(tctx.options) f1, f2 = tflow.tflow(), tflow.tflow() tctx.cycle(sc, f1) @@ -42,7 +42,7 @@ class TestConcurrent(tservers.MasterTest): "mitmproxy/data/addonscripts/concurrent_decorator_err.py" ) ) - sc.start() + sc.start(tctx.options) assert "decorator not supported" in tctx.master.event_log[0][1] def test_concurrent_class(self): @@ -52,7 +52,7 @@ class TestConcurrent(tservers.MasterTest): "mitmproxy/data/addonscripts/concurrent_decorator_class.py" ) ) - sc.start() + sc.start(tctx.options) f1, f2 = tflow.tflow(), tflow.tflow() tctx.cycle(sc, f1) diff --git a/test/mitmproxy/test_addonmanager.py b/test/mitmproxy/test_addonmanager.py index 17402e26..3e5f71c6 100644 --- a/test/mitmproxy/test_addonmanager.py +++ b/test/mitmproxy/test_addonmanager.py @@ -10,12 +10,12 @@ from mitmproxy import proxy class TAddon: def __init__(self, name): self.name = name - self.noop_member = True + self.tick = True def __repr__(self): return "Addon(%s)" % self.name - def noop(self): + def done(self): pass @@ -30,6 +30,6 @@ def test_simple(): assert not a.chain a.add(TAddon("one")) - a("noop") + a("done") with pytest.raises(exceptions.AddonError): - a("noop_member") + a("tick") diff --git a/test/mitmproxy/test_certs.py b/test/mitmproxy/test_certs.py index 9bd3ad25..88c49561 100644 --- a/test/mitmproxy/test_certs.py +++ b/test/mitmproxy/test_certs.py @@ -34,118 +34,106 @@ from mitmproxy.test import tutils class TestCertStore: - def test_create_explicit(self): - with tutils.tmpdir() as d: - ca = certs.CertStore.from_store(d, "test") - assert ca.get_cert(b"foo", []) - - ca2 = certs.CertStore.from_store(d, "test") - assert ca2.get_cert(b"foo", []) - - assert ca.default_ca.get_serial_number() == ca2.default_ca.get_serial_number() - - def test_create_no_common_name(self): - with tutils.tmpdir() as d: - ca = certs.CertStore.from_store(d, "test") - assert ca.get_cert(None, [])[0].cn is None - - def test_create_tmp(self): - with tutils.tmpdir() as d: - ca = certs.CertStore.from_store(d, "test") - assert ca.get_cert(b"foo.com", []) - assert ca.get_cert(b"foo.com", []) - assert ca.get_cert(b"*.foo.com", []) - - r = ca.get_cert(b"*.foo.com", []) - assert r[1] == ca.default_privatekey - - def test_sans(self): - with tutils.tmpdir() as d: - ca = certs.CertStore.from_store(d, "test") - c1 = ca.get_cert(b"foo.com", [b"*.bar.com"]) - ca.get_cert(b"foo.bar.com", []) - # assert c1 == c2 - c3 = ca.get_cert(b"bar.com", []) - assert not c1 == c3 - - def test_sans_change(self): - with tutils.tmpdir() as d: - ca = certs.CertStore.from_store(d, "test") - ca.get_cert(b"foo.com", [b"*.bar.com"]) - cert, key, chain_file = ca.get_cert(b"foo.bar.com", [b"*.baz.com"]) - assert b"*.baz.com" in cert.altnames - - def test_expire(self): - with tutils.tmpdir() as d: - ca = certs.CertStore.from_store(d, "test") - ca.STORE_CAP = 3 - ca.get_cert(b"one.com", []) - ca.get_cert(b"two.com", []) - ca.get_cert(b"three.com", []) - - assert (b"one.com", ()) in ca.certs - assert (b"two.com", ()) in ca.certs - assert (b"three.com", ()) in ca.certs - - ca.get_cert(b"one.com", []) - - assert (b"one.com", ()) in ca.certs - assert (b"two.com", ()) in ca.certs - assert (b"three.com", ()) in ca.certs - - ca.get_cert(b"four.com", []) - - assert (b"one.com", ()) not in ca.certs - assert (b"two.com", ()) in ca.certs - assert (b"three.com", ()) in ca.certs - assert (b"four.com", ()) in ca.certs - - def test_overrides(self): - with tutils.tmpdir() as d: - ca1 = certs.CertStore.from_store(os.path.join(d, "ca1"), "test") - ca2 = certs.CertStore.from_store(os.path.join(d, "ca2"), "test") - assert not ca1.default_ca.get_serial_number( - ) == ca2.default_ca.get_serial_number() - - dc = ca2.get_cert(b"foo.com", [b"sans.example.com"]) - dcp = os.path.join(d, "dc") - f = open(dcp, "wb") - f.write(dc[0].to_pem()) - f.close() - ca1.add_cert_file(b"foo.com", dcp) - - ret = ca1.get_cert(b"foo.com", []) - assert ret[0].serial == dc[0].serial - - def test_create_dhparams(self): - with tutils.tmpdir() as d: - filename = os.path.join(d, "dhparam.pem") - certs.CertStore.load_dhparam(filename) - assert os.path.exists(filename) + def test_create_explicit(self, tmpdir): + ca = certs.CertStore.from_store(str(tmpdir), "test") + assert ca.get_cert(b"foo", []) + + ca2 = certs.CertStore.from_store(str(tmpdir), "test") + assert ca2.get_cert(b"foo", []) + + assert ca.default_ca.get_serial_number() == ca2.default_ca.get_serial_number() + + def test_create_no_common_name(self, tmpdir): + ca = certs.CertStore.from_store(str(tmpdir), "test") + assert ca.get_cert(None, [])[0].cn is None + + def test_create_tmp(self, tmpdir): + ca = certs.CertStore.from_store(str(tmpdir), "test") + assert ca.get_cert(b"foo.com", []) + assert ca.get_cert(b"foo.com", []) + assert ca.get_cert(b"*.foo.com", []) + + r = ca.get_cert(b"*.foo.com", []) + assert r[1] == ca.default_privatekey + + def test_sans(self, tmpdir): + ca = certs.CertStore.from_store(str(tmpdir), "test") + c1 = ca.get_cert(b"foo.com", [b"*.bar.com"]) + ca.get_cert(b"foo.bar.com", []) + # assert c1 == c2 + c3 = ca.get_cert(b"bar.com", []) + assert not c1 == c3 + + def test_sans_change(self, tmpdir): + ca = certs.CertStore.from_store(str(tmpdir), "test") + ca.get_cert(b"foo.com", [b"*.bar.com"]) + cert, key, chain_file = ca.get_cert(b"foo.bar.com", [b"*.baz.com"]) + assert b"*.baz.com" in cert.altnames + + def test_expire(self, tmpdir): + ca = certs.CertStore.from_store(str(tmpdir), "test") + ca.STORE_CAP = 3 + ca.get_cert(b"one.com", []) + ca.get_cert(b"two.com", []) + ca.get_cert(b"three.com", []) + + assert (b"one.com", ()) in ca.certs + assert (b"two.com", ()) in ca.certs + assert (b"three.com", ()) in ca.certs + + ca.get_cert(b"one.com", []) + + assert (b"one.com", ()) in ca.certs + assert (b"two.com", ()) in ca.certs + assert (b"three.com", ()) in ca.certs + + ca.get_cert(b"four.com", []) + + assert (b"one.com", ()) not in ca.certs + assert (b"two.com", ()) in ca.certs + assert (b"three.com", ()) in ca.certs + assert (b"four.com", ()) in ca.certs + + def test_overrides(self, tmpdir): + ca1 = certs.CertStore.from_store(str(tmpdir.join("ca1")), "test") + ca2 = certs.CertStore.from_store(str(tmpdir.join("ca2")), "test") + assert not ca1.default_ca.get_serial_number() == ca2.default_ca.get_serial_number() + + dc = ca2.get_cert(b"foo.com", [b"sans.example.com"]) + dcp = tmpdir.join("dc") + dcp.write(dc[0].to_pem()) + ca1.add_cert_file(b"foo.com", str(dcp)) + + ret = ca1.get_cert(b"foo.com", []) + assert ret[0].serial == dc[0].serial + + def test_create_dhparams(self, tmpdir): + filename = str(tmpdir.join("dhparam.pem")) + certs.CertStore.load_dhparam(filename) + assert os.path.exists(filename) class TestDummyCert: - def test_with_ca(self): - with tutils.tmpdir() as d: - ca = certs.CertStore.from_store(d, "test") - r = certs.dummy_cert( - ca.default_privatekey, - ca.default_ca, - b"foo.com", - [b"one.com", b"two.com", b"*.three.com", b"127.0.0.1"] - ) - assert r.cn == b"foo.com" - assert r.altnames == [b'one.com', b'two.com', b'*.three.com'] - - r = certs.dummy_cert( - ca.default_privatekey, - ca.default_ca, - None, - [] - ) - assert r.cn is None - assert r.altnames == [] + def test_with_ca(self, tmpdir): + ca = certs.CertStore.from_store(str(tmpdir), "test") + r = certs.dummy_cert( + ca.default_privatekey, + ca.default_ca, + b"foo.com", + [b"one.com", b"two.com", b"*.three.com", b"127.0.0.1"] + ) + assert r.cn == b"foo.com" + assert r.altnames == [b'one.com', b'two.com', b'*.three.com'] + + r = certs.dummy_cert( + ca.default_privatekey, + ca.default_ca, + None, + [] + ) + assert r.cn is None + assert r.altnames == [] class TestSSLCert: @@ -172,7 +160,6 @@ class TestSSLCert: assert c2.to_pem() assert c2.has_expired is not None - assert not c1 == c2 assert c1 != c2 def test_err_broken_sans(self): diff --git a/test/mitmproxy/test_connections.py b/test/mitmproxy/test_connections.py index 0083f57c..67a6552f 100644 --- a/test/mitmproxy/test_connections.py +++ b/test/mitmproxy/test_connections.py @@ -66,8 +66,18 @@ class TestClientConnection: assert c.timestamp_start == 42 c3 = c.copy() + assert c3.get_state() != c.get_state() + c.id = c3.id = "foo" assert c3.get_state() == c.get_state() + def test_eq(self): + c = tflow.tclient_conn() + c2 = c.copy() + assert c == c + assert c != c2 + assert c != 42 + assert hash(c) != hash(c2) + class TestServerConnection: @@ -147,6 +157,21 @@ class TestServerConnection: with pytest.raises(ValueError, matches='sni must be str, not '): c.establish_ssl(None, b'foobar') + def test_state(self): + c = tflow.tserver_conn() + c2 = c.copy() + assert c2.get_state() != c.get_state() + c.id = c2.id = "foo" + assert c2.get_state() == c.get_state() + + def test_eq(self): + c = tflow.tserver_conn() + c2 = c.copy() + assert c == c + assert c != c2 + assert c != 42 + assert hash(c) != hash(c2) + class TestClientConnectionTLS: diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index 668d0d4a..f20e0c8c 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -1,5 +1,4 @@ import json -import os import shlex import pytest @@ -142,30 +141,26 @@ class TestHARDump: with pytest.raises(ScriptError): tscript("complex/har_dump.py") - def test_simple(self): - with tutils.tmpdir() as tdir: - path = os.path.join(tdir, "somefile") + def test_simple(self, tmpdir): + path = str(tmpdir.join("somefile")) - m, sc = tscript("complex/har_dump.py", shlex.quote(path)) - m.addons.invoke(m, "response", self.flow()) - m.addons.remove(sc) - - with open(path, "r") as inp: - har = json.load(inp) + m, sc = tscript("complex/har_dump.py", shlex.quote(path)) + m.addons.invoke(m, "response", self.flow()) + m.addons.remove(sc) + with open(path, "r") as inp: + har = json.load(inp) assert len(har["log"]["entries"]) == 1 - def test_base64(self): - with tutils.tmpdir() as tdir: - path = os.path.join(tdir, "somefile") - - m, sc = tscript("complex/har_dump.py", shlex.quote(path)) - m.addons.invoke(m, "response", self.flow(resp_content=b"foo" + b"\xFF" * 10)) - m.addons.remove(sc) + def test_base64(self, tmpdir): + path = str(tmpdir.join("somefile")) - with open(path, "r") as inp: - har = json.load(inp) + m, sc = tscript("complex/har_dump.py", shlex.quote(path)) + m.addons.invoke(m, "response", self.flow(resp_content=b"foo" + b"\xFF" * 10)) + m.addons.remove(sc) + with open(path, "r") as inp: + har = json.load(inp) assert har["log"]["entries"][0]["response"]["content"]["encoding"] == "base64" def test_format_cookies(self): @@ -187,7 +182,7 @@ class TestHARDump: f = format_cookies([("n", "v", CA([("expires", "Mon, 24-Aug-2037 00:00:00 GMT")]))])[0] assert f['expires'] - def test_binary(self): + def test_binary(self, tmpdir): f = self.flow() f.request.method = "POST" @@ -196,14 +191,12 @@ class TestHARDump: f.response.headers["random-junk"] = bytes(range(256)) f.response.content = bytes(range(256)) - with tutils.tmpdir() as tdir: - path = os.path.join(tdir, "somefile") - - m, sc = tscript("complex/har_dump.py", shlex.quote(path)) - m.addons.invoke(m, "response", f) - m.addons.remove(sc) + path = str(tmpdir.join("somefile")) - with open(path, "r") as inp: - har = json.load(inp) + m, sc = tscript("complex/har_dump.py", shlex.quote(path)) + m.addons.invoke(m, "response", f) + m.addons.remove(sc) + with open(path, "r") as inp: + har = json.load(inp) assert len(har["log"]["entries"]) == 1 diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index db33cddd..df392829 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -1,5 +1,4 @@ import copy -import os import pytest import typing import argparse @@ -7,7 +6,6 @@ import argparse from mitmproxy import options from mitmproxy import optmanager from mitmproxy import exceptions -from mitmproxy.test import tutils class TO(optmanager.OptManager): @@ -85,10 +83,11 @@ def test_options(): with pytest.raises(TypeError): TO(nonexistent = "value") - with pytest.raises(Exception, match="No such option"): + with pytest.raises(Exception, match="Unknown options"): o.nonexistent = "value" - with pytest.raises(Exception, match="No such option"): + with pytest.raises(Exception, match="Unknown options"): o.update(nonexistent = "value") + assert o.update_known(nonexistent = "value") == {"nonexistent": "value"} rec = [] @@ -201,62 +200,63 @@ def test_simple(): def test_serialize(): o = TD2() o.three = "set" - assert "dfour" in o.serialize(None, defaults=True) + assert "dfour" in optmanager.serialize(o, None, defaults=True) - data = o.serialize(None) + data = optmanager.serialize(o, None) assert "dfour" not in data o2 = TD2() - o2.load(data) + optmanager.load(o2, data) assert o2 == o t = """ unknown: foo """ - data = o.serialize(t) + data = optmanager.serialize(o, t) o2 = TD2() - o2.load(data) + optmanager.load(o2, data) assert o2 == o t = "invalid: foo\ninvalid" with pytest.raises(Exception, match="Config error"): - o2.load(t) + optmanager.load(o2, t) t = "invalid" with pytest.raises(Exception, match="Config error"): - o2.load(t) + optmanager.load(o2, t) t = "" - o2.load(t) - - with pytest.raises(exceptions.OptionsError, matches='No such option: foobar'): - o2.load("foobar: '123'") + optmanager.load(o2, t) + assert optmanager.load(o2, "foobar: '123'") == {"foobar": "123"} def test_serialize_defaults(): o = options.Options() - assert o.serialize(None, defaults=True) + assert optmanager.serialize(o, None, defaults=True) -def test_saving(): +def test_saving(tmpdir): o = TD2() o.three = "set" - with tutils.tmpdir() as tdir: - dst = os.path.join(tdir, "conf") - o.save(dst, defaults=True) + dst = str(tmpdir.join("conf")) + optmanager.save(o, dst, defaults=True) + + o2 = TD2() + optmanager.load_paths(o2, dst) + o2.three = "foo" + optmanager.save(o2, dst, defaults=True) - o2 = TD2() - o2.load_paths(dst) - o2.three = "foo" - o2.save(dst, defaults=True) + optmanager.load_paths(o, dst) + assert o.three == "foo" - o.load_paths(dst) - assert o.three == "foo" + with open(dst, 'a') as f: + f.write("foobar: '123'") + assert optmanager.load_paths(o, dst) == {"foobar": "123"} - with open(dst, 'a') as f: - f.write("foobar: '123'") - with pytest.raises(exceptions.OptionsError, matches=''): - o.load_paths(dst) + with open(dst, 'a') as f: + f.write("'''") + with pytest.raises(exceptions.OptionsError): + optmanager.load_paths(o, dst) def test_merge(): @@ -283,9 +283,9 @@ def test_option(): assert o2 != o -def test_dump(): +def test_dump_defaults(): o = options.Options() - assert optmanager.dump(o) + assert optmanager.dump_defaults(o) class TTypes(optmanager.OptManager): @@ -349,3 +349,6 @@ def test_set(): assert opts.seqstr == ["foo", "bar"] opts.set("seqstr") assert opts.seqstr == [] + + with pytest.raises(exceptions.OptionsError): + opts.set("nonexistent=wobble") diff --git a/test/mitmproxy/tools/console/test_master.py b/test/mitmproxy/tools/console/test_master.py index 0bf3734b..6c716ad1 100644 --- a/test/mitmproxy/tools/console/test_master.py +++ b/test/mitmproxy/tools/console/test_master.py @@ -28,7 +28,9 @@ class TestMaster(tservers.MasterTest): if "verbosity" not in opts: opts["verbosity"] = 1 o = options.Options(**opts) - return console.master.ConsoleMaster(o, proxy.DummyServer()) + m = console.master.ConsoleMaster(o, proxy.DummyServer()) + m.addons.configure_all(o, o.keys()) + return m def test_basic(self): m = self.mkmaster() diff --git a/test/mitmproxy/tools/test_dump.py b/test/mitmproxy/tools/test_dump.py index 2542ec4b..a15bf583 100644 --- a/test/mitmproxy/tools/test_dump.py +++ b/test/mitmproxy/tools/test_dump.py @@ -1,4 +1,3 @@ -import os import pytest from unittest import mock @@ -9,7 +8,6 @@ from mitmproxy import controller from mitmproxy import options from mitmproxy.tools import dump -from mitmproxy.test import tutils from .. import tservers @@ -19,18 +17,17 @@ class TestDumpMaster(tservers.MasterTest): m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=False, with_dumper=False) return m - def test_read(self): - with tutils.tmpdir() as t: - p = os.path.join(t, "read") - self.flowfile(p) - self.dummy_cycle( - self.mkmaster(None, rfile=p), - 1, b"", - ) - with pytest.raises(exceptions.OptionsError): - self.mkmaster(None, rfile="/nonexistent") - with pytest.raises(exceptions.OptionsError): - self.mkmaster(None, rfile="test_dump.py") + def test_read(self, tmpdir): + p = str(tmpdir.join("read")) + self.flowfile(p) + self.dummy_cycle( + self.mkmaster(None, rfile=p), + 1, b"", + ) + with pytest.raises(exceptions.OptionsError): + self.mkmaster(None, rfile="/nonexistent") + with pytest.raises(exceptions.OptionsError): + self.mkmaster(None, rfile="test_dump.py") def test_has_error(self): m = self.mkmaster(None) diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index a8aaa358..c47411ee 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -79,6 +79,8 @@ class TestMaster(master.Master): self.state = TestState() self.addons.add(self.state) self.addons.add(*addons) + self.addons.configure_all(self.options, self.options.keys()) + self.addons.invoke_all_with_context("running") def clear_log(self): self.tlog = [] diff --git a/test/mitmproxy/types/test_multidict.py b/test/mitmproxy/types/test_multidict.py index 9b13c5cd..c76cd753 100644 --- a/test/mitmproxy/types/test_multidict.py +++ b/test/mitmproxy/types/test_multidict.py @@ -93,11 +93,6 @@ class TestMultiDict: md1.fields = md1.fields[1:] + md1.fields[:1] assert not (md1 == md2) - def test_ne(self): - assert not TMultiDict() != TMultiDict() - assert TMultiDict() != self._multi() - assert TMultiDict() != 42 - def test_hash(self): """ If a class defines mutable objects and implements an __eq__() method, @@ -205,3 +200,12 @@ class TestMultiDictView: tv["c"] = "b" assert p.vals == (("a", "b"), ("c", "b")) assert tv["a"] == "b" + + def test_copy(self): + p = TParent() + tv = multidict.MultiDictView(p.getter, p.setter) + c = tv.copy() + assert isinstance(c, multidict.MultiDict) + assert tv.items() == c.items() + c["foo"] = "bar" + assert tv.items() != c.items() diff --git a/test/mitmproxy/types/test_serializable.py b/test/mitmproxy/types/test_serializable.py index dd4a3778..390d17e1 100644 --- a/test/mitmproxy/types/test_serializable.py +++ b/test/mitmproxy/types/test_serializable.py @@ -1,3 +1,5 @@ +import copy + from mitmproxy.types import serializable @@ -6,17 +8,17 @@ class SerializableDummy(serializable.Serializable): self.i = i def get_state(self): - return self.i + return copy.copy(self.i) def set_state(self, i): self.i = i - def from_state(self, state): - return type(self)(state) + @classmethod + def from_state(cls, state): + return cls(state) class TestSerializable: - def test_copy(self): a = SerializableDummy(42) assert a.i == 42 @@ -26,3 +28,12 @@ class TestSerializable: a.set_state(1) assert a.i == 1 assert b.i == 42 + + def test_copy_id(self): + a = SerializableDummy({ + "id": "foo", + "foo": 42 + }) + b = a.copy() + assert a.get_state()["id"] != b.get_state()["id"] + assert a.get_state()["foo"] == b.get_state()["foo"] diff --git a/test/pathod/language/test_base.py b/test/pathod/language/test_base.py index 85e9e53b..ec460b07 100644 --- a/test/pathod/language/test_base.py +++ b/test/pathod/language/test_base.py @@ -1,11 +1,8 @@ -import os import pytest from pathod import language from pathod.language import base, exceptions -from mitmproxy.test import tutils - def parse_request(s): return language.parse_pathoc(s).next() @@ -137,24 +134,22 @@ class TestTokValueFile: v = base.TokValue.parseString("<path")[0] assert v.path == "path" - def test_access_control(self): + def test_access_control(self, tmpdir): v = base.TokValue.parseString("<path")[0] - with tutils.tmpdir() as t: - p = os.path.join(t, "path") - with open(p, "wb") as f: - f.write(b"x" * 10000) - - assert v.get_generator(language.Settings(staticdir=t)) - - v = base.TokValue.parseString("<path2")[0] - with pytest.raises(exceptions.FileAccessDenied): - v.get_generator(language.Settings(staticdir=t)) - with pytest.raises(Exception, match="access disabled"): - v.get_generator(language.Settings()) - - v = base.TokValue.parseString("</outside")[0] - with pytest.raises(Exception, match="outside"): - v.get_generator(language.Settings(staticdir=t)) + f = tmpdir.join("path") + f.write(b"x" * 10000) + + assert v.get_generator(language.Settings(staticdir=str(tmpdir))) + + v = base.TokValue.parseString("<path2")[0] + with pytest.raises(exceptions.FileAccessDenied): + v.get_generator(language.Settings(staticdir=str(tmpdir))) + with pytest.raises(Exception, match="access disabled"): + v.get_generator(language.Settings()) + + v = base.TokValue.parseString("</outside")[0] + with pytest.raises(Exception, match="outside"): + v.get_generator(language.Settings(staticdir=str(tmpdir))) def test_spec(self): v = base.TokValue.parseString("<'one two'")[0] diff --git a/test/pathod/language/test_generators.py b/test/pathod/language/test_generators.py index b3ce0335..dc15aaa1 100644 --- a/test/pathod/language/test_generators.py +++ b/test/pathod/language/test_generators.py @@ -1,7 +1,4 @@ -import os - from pathod.language import generators -from mitmproxy.test import tutils def test_randomgenerator(): @@ -15,23 +12,20 @@ def test_randomgenerator(): assert len(g[1000:1001]) == 0 -def test_filegenerator(): - with tutils.tmpdir() as t: - path = os.path.join(t, "foo") - f = open(path, "wb") - f.write(b"x" * 10000) - f.close() - g = generators.FileGenerator(path) - assert len(g) == 10000 - assert g[0] == b"x" - assert g[-1] == b"x" - assert g[0:5] == b"xxxxx" - assert len(g[1:10]) == 9 - assert len(g[10000:10001]) == 0 - assert repr(g) - # remove all references to FileGenerator instance to close the file - # handle. - del g +def test_filegenerator(tmpdir): + f = tmpdir.join("foo") + f.write(b"x" * 10000) + g = generators.FileGenerator(str(f)) + assert len(g) == 10000 + assert g[0] == b"x" + assert g[-1] == b"x" + assert g[0:5] == b"xxxxx" + assert len(g[1:10]) == 9 + assert len(g[10000:10001]) == 0 + assert repr(g) + # remove all references to FileGenerator instance to close the file + # handle. + del g def test_transform_generator(): diff --git a/web/src/js/__tests__/ducks/utils/storeSpec.js b/web/src/js/__tests__/ducks/utils/storeSpec.js new file mode 100644 index 00000000..6bfea2c7 --- /dev/null +++ b/web/src/js/__tests__/ducks/utils/storeSpec.js @@ -0,0 +1,86 @@ +jest.unmock('../../../ducks/utils/store') + +import reduceStore, * as storeActions from '../../../ducks/utils/store' + +describe('store reducer', () => { + it('should return initial state', () => { + expect(reduceStore(undefined, {})).toEqual({ + byId: {}, + list: [], + listIndex: {}, + view: [], + viewIndex: {}, + }) + }) + + it('should handle add action', () => { + let a = {id: 1}, + b = {id: 9}, + state = reduceStore(undefined, {}) + expect(state = reduceStore(state, storeActions.add(a))).toEqual({ + byId: { 1: a }, + listIndex: { 1: 0 }, + list: [ a ], + view: [ a ], + viewIndex: { 1: 0 }, + }) + + expect(reduceStore(state, storeActions.add(b))).toEqual({ + byId: { 1: a, 9: b }, + listIndex: { 1: 0, 9: 1 }, + list: [ a, b ], + view: [ a, b ], + viewIndex: { 1: 0, 9: 1 }, + }) + }) + + it('should not add the item with duplicated id', () => { + let a = {id: 1}, + state = reduceStore(undefined, storeActions.add(a)) + expect(reduceStore(state, storeActions.add(a))).toEqual(state) + }) + + it('should handle update action', () => { + let a = {id: 1, foo: "foo"}, + updated = {...a, foo: "bar"}, + state = reduceStore(undefined, storeActions.add(a)) + expect(reduceStore(state, storeActions.update(updated))).toEqual({ + byId: { 1: updated }, + list: [ updated ], + listIndex: { 1: 0 }, + view: [ updated ], + viewIndex: { 1: 0 }, + }) + }) + + it('should handle update action with filter', () => { + let a = {id: 0}, + b = {id: 1}, + state = reduceStore(undefined, storeActions.add(a)) + state = reduceStore(state, storeActions.add(b)) + expect(reduceStore(state, storeActions.update(b, + item => {return item.id < 1}))).toEqual({ + byId: { 0: a, 1: b }, + list: [ a, b ], + listIndex: { 0: 0, 1: 1 }, + view: [ a ], + viewIndex: { 0: 0 } + }) + }) + + it('should handle update action with sort', () => { + let a = {id: 2}, + b = {id: 3}, + state = reduceStore(undefined, storeActions.add(a)) + state = reduceStore(state, storeActions.add(b)) + expect(reduceStore(state, storeActions.update(a, undefined, + (a, b) => {return b.id - a.id}))).toEqual({ + // sort by id in descending order + byId: { 2: a, 3: b }, + list: [ a, b ], + listIndex: {2: 0, 3: 1}, + view: [ b, a ], + viewIndex: { 2: 1, 3: 0 }, + }) + }) +}) diff --git a/web/src/js/ducks/ui/flow.js b/web/src/js/ducks/ui/flow.js index 8cb6e170..ba604ea2 100644 --- a/web/src/js/ducks/ui/flow.js +++ b/web/src/js/ducks/ui/flow.js @@ -60,7 +60,7 @@ export default function reducer(state = defaultState, action) { // There is no explicit "stop edit" event. // We stop editing when we receive an update for // the currently edited flow from the server - if (action.flow.id === state.modifiedFlow.id) { + if (action.data.id === state.modifiedFlow.id) { return { ...state, modifiedFlow: false, @@ -148,7 +148,7 @@ export function setContent(content){ return { type: SET_CONTENT, content } } -export function stopEdit(flow, modifiedFlow) { - let diff = getDiff(flow, modifiedFlow) - return {type: flowsActions.UPDATE, flow, diff } +export function stopEdit(data, modifiedFlow) { + let diff = getDiff(data, modifiedFlow) + return {type: flowsActions.UPDATE, data, diff } } diff --git a/web/src/js/filt/filt.peg b/web/src/js/filt/filt.peg index b2576661..12959474 100644 --- a/web/src/js/filt/filt.peg +++ b/web/src/js/filt/filt.peg @@ -96,7 +96,7 @@ function responseBody(regex){ function domain(regex){ regex = new RegExp(regex, "i"); function domainFilter(flow){ - return flow.request && regex.test(flow.request.host); + return flow.request && (regex.test(flow.request.host) || regex.test(flow.request.pretty_host)); } domainFilter.desc = "domain matches " + regex; return domainFilter; |