diff options
56 files changed, 723 insertions, 665 deletions
diff --git a/examples/complex/flowbasic.py b/examples/complex/flowbasic.py deleted file mode 100644 index dafdd084..00000000 --- a/examples/complex/flowbasic.py +++ /dev/null @@ -1,43 +0,0 @@ -#!/usr/bin/env python3 -""" - This example shows how to build a proxy based on mitmproxy's Flow - primitives. - - Heads Up: In the majority of cases, you want to use inline scripts. - - Note that request and response messages are not automatically replied to, - so we need to implement handlers to do this. -""" -from mitmproxy import controller, options, master -from mitmproxy.proxy import ProxyServer, ProxyConfig - - -class MyMaster(master.Master): - def run(self): - try: - master.Master.run(self) - except KeyboardInterrupt: - self.shutdown() - - @controller.handler - def request(self, f): - print("request", f) - - @controller.handler - def response(self, f): - print("response", f) - - @controller.handler - def error(self, f): - print("error", f) - - @controller.handler - def log(self, l): - print("log", l.msg) - - -opts = options.Options(cadir="~/.mitmproxy/") -config = ProxyConfig(opts) -server = ProxyServer(config) -m = MyMaster(opts, server) -m.run() diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py index 43e76510..670c4f24 100644 --- a/mitmproxy/addonmanager.py +++ b/mitmproxy/addonmanager.py @@ -1,5 +1,7 @@ from mitmproxy import exceptions from mitmproxy import eventsequence +from mitmproxy import controller +from . import ctx import pprint @@ -31,31 +33,27 @@ class AddonManager: return i def configure_all(self, options, updated): - self.invoke_all_with_context("configure", options, updated) - - def startup(self, s): - """ - Run startup events on addon. - """ - self.invoke_with_context(s, "start", self.master.options) + self.trigger("configure", options, updated) def add(self, *addons): """ Add addons to the end of the chain, and run their startup events. """ self.chain.extend(addons) - for i in addons: - self.startup(i) + with self.master.handlecontext(): + for i in addons: + self.invoke_addon(i, "start", self.master.options) def remove(self, addon): """ Remove an addon from the chain, and run its done events. """ self.chain = [i for i in self.chain if i is not addon] - self.invoke_with_context(addon, "done") + with self.master.handlecontext(): + self.invoke_addon(addon, "done") def done(self): - self.invoke_all_with_context("done") + self.trigger("done") def __len__(self): return len(self.chain) @@ -63,18 +61,43 @@ class AddonManager: def __str__(self): return pprint.pformat([str(i) for i in self.chain]) - def invoke_with_context(self, addon, name, *args, **kwargs): - with self.master.handlecontext(): - self.invoke(addon, name, *args, **kwargs) + def handle_lifecycle(self, name, message): + """ + Handle a lifecycle event. + """ + if not hasattr(message, "reply"): # pragma: no cover + raise exceptions.ControlException( + "Message %s has no reply attribute" % message + ) - def invoke_all_with_context(self, name, *args, **kwargs): - with self.master.handlecontext(): - for i in self.chain: - self.invoke(i, name, *args, **kwargs) + # We can use DummyReply objects multiple times. We only clear them up on + # the next handler so that we can access value and state in the + # meantime. + if isinstance(message.reply, controller.DummyReply): + message.reply.reset() - def invoke(self, addon, name, *args, **kwargs): - if name not in eventsequence.Events: # prama: no cover - raise NotImplementedError("Unknown event") + self.trigger(name, message) + + if message.reply.state != "taken": + message.reply.take() + if not message.reply.has_message: + message.reply.ack() + message.reply.commit() + + if isinstance(message.reply, controller.DummyReply): + message.reply.mark_reset() + + def invoke_addon(self, addon, name, *args, **kwargs): + """ + Invoke an event on an addon. This method must run within an + established handler context. + """ + if not ctx.master: + raise exceptions.AddonError( + "invoke_addon called without a handler context." + ) + if name not in eventsequence.Events: + name = "event_" + name func = getattr(addon, name, None) if func: if not callable(func): @@ -83,9 +106,13 @@ class AddonManager: ) func(*args, **kwargs) - def __call__(self, name, *args, **kwargs): - for i in self.chain: - try: - self.invoke(i, name, *args, **kwargs) - except exceptions.AddonHalt: - return + def trigger(self, name, *args, **kwargs): + """ + Establish a handler context and trigger an event across all addons + """ + with self.master.handlecontext(): + for i in self.chain: + try: + self.invoke_addon(i, name, *args, **kwargs) + except exceptions.AddonHalt: + return diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py index 7a45106c..b4367d78 100644 --- a/mitmproxy/addons/__init__.py +++ b/mitmproxy/addons/__init__.py @@ -8,6 +8,7 @@ from mitmproxy.addons import disable_h2c from mitmproxy.addons import onboarding from mitmproxy.addons import proxyauth from mitmproxy.addons import replace +from mitmproxy.addons import readfile from mitmproxy.addons import script from mitmproxy.addons import serverplayback from mitmproxy.addons import setheaders @@ -37,5 +38,6 @@ def default_addons(): stickycookie.StickyCookie(), streambodies.StreamBodies(), streamfile.StreamFile(), + readfile.ReadFile(), upstream_auth.UpstreamAuth(), ] diff --git a/mitmproxy/addons/check_alpn.py b/mitmproxy/addons/check_alpn.py index c288d788..cb3c87e3 100644 --- a/mitmproxy/addons/check_alpn.py +++ b/mitmproxy/addons/check_alpn.py @@ -1,5 +1,6 @@ import mitmproxy from mitmproxy.net import tcp +from mitmproxy import ctx class CheckALPN: @@ -9,9 +10,8 @@ class CheckALPN: def configure(self, options, updated): self.failed = mitmproxy.ctx.master.options.http2 and not tcp.HAS_ALPN if self.failed: - mitmproxy.ctx.master.add_log( + ctx.log.warn( "HTTP/2 is disabled because ALPN support missing!\n" "OpenSSL 1.0.2+ required to support HTTP/2 connections.\n" - "Use --no-http2 to silence this warning.", - "warn", + "Use --no-http2 to silence this warning." ) diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 34c6c9c9..3345e65a 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -10,7 +10,6 @@ class ClientPlayback: def __init__(self): self.flows = None self.current_thread = None - self.keepserving = False self.has_replayed = False def count(self) -> int: @@ -32,7 +31,6 @@ class ClientPlayback: self.load(flows) else: self.flows = None - self.keepserving = options.keepserving def tick(self): if self.current_thread and not self.current_thread.is_alive(): @@ -41,5 +39,5 @@ class ClientPlayback: self.current_thread = ctx.master.replay_request(self.flows.pop(0)) self.has_replayed = True if self.has_replayed: - if not self.flows and not self.current_thread and not self.keepserving: - ctx.master.shutdown() + if not self.flows and not self.current_thread: + ctx.master.addons.trigger("processing_complete") diff --git a/mitmproxy/addons/keepserving.py b/mitmproxy/addons/keepserving.py new file mode 100644 index 00000000..9c975a7b --- /dev/null +++ b/mitmproxy/addons/keepserving.py @@ -0,0 +1,7 @@ +from mitmproxy import ctx + + +class KeepServing: + def event_processing_complete(self): + if not ctx.master.options.keepserving: + ctx.master.shutdown() diff --git a/mitmproxy/addons/readfile.py b/mitmproxy/addons/readfile.py new file mode 100644 index 00000000..03dcd084 --- /dev/null +++ b/mitmproxy/addons/readfile.py @@ -0,0 +1,46 @@ +import os.path + +from mitmproxy import ctx +from mitmproxy import io +from mitmproxy import exceptions + + +class ReadFile: + """ + An addon that handles reading from file on startup. + """ + def __init__(self): + self.path = None + + def load_flows_file(self, path: str) -> int: + path = os.path.expanduser(path) + cnt = 0 + try: + with open(path, "rb") as f: + freader = io.FlowReader(f) + for i in freader.stream(): + cnt += 1 + ctx.master.load_flow(i) + return cnt + except (IOError, exceptions.FlowReadException) as v: + if cnt: + ctx.log.warn( + "Flow file corrupted - loaded %i flows." % cnt, + ) + else: + ctx.log.error("Flow file corrupted.") + raise exceptions.FlowReadException(v) + + def configure(self, options, updated): + if "rfile" in updated and options.rfile: + self.path = options.rfile + + def running(self): + if self.path: + try: + self.load_flows_file(self.path) + except exceptions.FlowReadException as v: + raise exceptions.OptionsError(v) + finally: + self.path = None + ctx.master.addons.trigger("processing_complete") diff --git a/mitmproxy/addons/readstdin.py b/mitmproxy/addons/readstdin.py new file mode 100644 index 00000000..93a99f01 --- /dev/null +++ b/mitmproxy/addons/readstdin.py @@ -0,0 +1,26 @@ +from mitmproxy import ctx +from mitmproxy import io +from mitmproxy import exceptions +import sys + + +class ReadStdin: + """ + An addon that reads from stdin if we're not attached to (someting like) + a tty. + """ + def running(self, stdin = sys.stdin): + if not stdin.isatty(): + ctx.log.info("Reading from stdin") + try: + stdin.buffer.read(0) + except Exception as e: + ctx.log.warn("Cannot read from stdin: {}".format(e)) + return + freader = io.FlowReader(stdin.buffer) + try: + for i in freader.stream(): + ctx.master.load_flow(i) + except exceptions.FlowReadException as e: + ctx.log.error("Error reading from stdin: %s" % e) + ctx.master.addons.trigger("processing_complete") diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index cfbe5284..4d893f1c 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -273,11 +273,11 @@ class ScriptLoader: ctx.master.addons.chain = ochain[:pos + 1] + ordered + ochain[pos + 1:] for s in newscripts: - ctx.master.addons.startup(s) + ctx.master.addons.invoke_addon(s, "start", options) if self.is_running: # If we're already running, we configure and tell the addon # we're up and running. - ctx.master.addons.invoke_with_context( + ctx.master.addons.invoke_addon( s, "configure", options, options.keys() ) - ctx.master.addons.invoke_with_context(s, "running") + ctx.master.addons.invoke_addon(s, "running") diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index f2b5f206..be2d6f2b 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -104,7 +104,7 @@ class ServerPlayback: def tick(self): if self.stop and not self.final_flow.live: - ctx.master.shutdown() + ctx.master.addons.trigger("processing_complete") def request(self, f): if self.flowmap: @@ -115,7 +115,7 @@ class ServerPlayback: if self.options.refresh_server_playback: response.refresh() f.response = response - if not self.flowmap and not self.options.keepserving: + if not self.flowmap: self.final_flow = f self.stop = True elif self.options.replay_kill_extra: diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 3a0587b0..2218327c 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -241,7 +241,7 @@ class View(collections.Sequence): self._refilter() self.sig_store_refresh.send(self) - def add(self, f: mitmproxy.flow.Flow) -> bool: + def add(self, f: mitmproxy.flow.Flow) -> None: """ Adds a flow to the state. If the flow already exists, it is ignored. @@ -371,6 +371,7 @@ class Focus: def index(self) -> typing.Optional[int]: if self.flow: return self.view.index(self.flow) + return None @index.setter def index(self, idx): diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py index c7db6690..f57b27c7 100644 --- a/mitmproxy/contentviews/__init__.py +++ b/mitmproxy/contentviews/__init__.py @@ -36,12 +36,14 @@ def get(name: str) -> Optional[View]: for i in views: if i.name.lower() == name.lower(): return i + return None def get_by_shortcut(c: str) -> Optional[View]: for i in views: if i.prompt[1] == c: return i + return None def add(view: View) -> None: diff --git a/mitmproxy/contentviews/xml_html.py b/mitmproxy/contentviews/xml_html.py index 0f2ce57d..62fb939f 100644 --- a/mitmproxy/contentviews/xml_html.py +++ b/mitmproxy/contentviews/xml_html.py @@ -128,6 +128,7 @@ def is_inline_text(a: Token, b: Token, c: Token) -> bool: if isinstance(a, Tag) and isinstance(b, Text) and isinstance(c, Tag): if a.is_opening and "\n" not in b.data and c.is_closing and a.tag == c.tag: return True + return False def is_inline(prev2: Token, prev1: Token, t: Token, next1: Token, next2: Token) -> bool: @@ -140,6 +141,7 @@ def is_inline(prev2: Token, prev1: Token, t: Token, next1: Token, next2: Token) return True # <div></div> (start tag) if isinstance(prev1, Tag) and prev1.is_opening and t.is_closing and prev1.tag == t.tag: return True # <div></div> (end tag) + return False class ElementStack: diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 868d5841..63117ef0 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -1,4 +1,3 @@ -import functools import queue from mitmproxy import exceptions @@ -14,8 +13,8 @@ class Channel: def ask(self, mtype, m): """ - Decorate a message with a reply attribute, and send it to the - master. Then wait for a response. + Decorate a message with a reply attribute, and send it to the master. + Then wait for a response. Raises: exceptions.Kill: All connections should be closed immediately. @@ -36,84 +35,42 @@ class Channel: def tell(self, mtype, m): """ - Decorate a message with a dummy reply attribute, send it to the - master, then return immediately. + Decorate a message with a dummy reply attribute, send it to the master, + then return immediately. """ m.reply = DummyReply() self.q.put((mtype, m)) -def handler(f): - @functools.wraps(f) - def wrapper(master, message): - if not hasattr(message, "reply"): - raise exceptions.ControlException("Message %s has no reply attribute" % message) - - # DummyReplys may be reused multiple times. - # We only clear them up on the next handler so that we can access value and - # state in the meantime. - if isinstance(message.reply, DummyReply): - message.reply.reset() - - # The following ensures that inheritance with wrapped handlers in the - # base class works. If we're the first handler, then responsibility for - # acking is ours. If not, it's someone else's and we ignore it. - handling = False - # We're the first handler - ack responsibility is ours - if message.reply.state == "unhandled": - handling = True - message.reply.handle() - - with master.handlecontext(): - ret = f(master, message) - if handling: - master.addons(f.__name__, message) - - # Reset the handled flag - it's common for us to feed the same object - # through handlers repeatedly, so we don't want this to persist across - # calls. - if handling and message.reply.state == "handled": - message.reply.take() - if not message.reply.has_message: - message.reply.ack() - message.reply.commit() - - # DummyReplys may be reused multiple times. - if isinstance(message.reply, DummyReply): - message.reply.mark_reset() - return ret - # Mark this function as a handler wrapper - wrapper.__dict__["__handler"] = True - return wrapper - - NO_REPLY = object() # special object we can distinguish from a valid "None" reply. class Reply: """ - Messages sent through a channel are decorated with a "reply" attribute. - This object is used to respond to the message through the return - channel. + Messages sent through a channel are decorated with a "reply" attribute. This + object is used to respond to the message through the return channel. """ def __init__(self, obj): self.obj = obj self.q = queue.Queue() # type: queue.Queue - self._state = "unhandled" # "unhandled" -> "handled" -> "taken" -> "committed" - self.value = NO_REPLY # holds the reply value. May change before things are actually commited. + self._state = "start" # "start" -> "taken" -> "committed" + + # Holds the reply value. May change before things are actually commited. + self.value = NO_REPLY @property def state(self): """ - The state the reply is currently in. A normal reply object goes sequentially through the following lifecycle: + The state the reply is currently in. A normal reply object goes + sequentially through the following lifecycle: - 1. unhandled: Initial State. - 2. handled: The reply object has been handled by the topmost handler function. - 3. taken: The reply object has been taken to be commited. - 4. committed: The reply has been sent back to the requesting party. + 1. start: Initial State. + 2. taken: The reply object has been taken to be commited. + 3. committed: The reply has been sent back to the requesting party. - This attribute is read-only and can only be modified by calling one of state transition functions. + This attribute is read-only and can only be modified by calling one of + state transition functions. """ return self._state @@ -121,47 +78,43 @@ class Reply: def has_message(self): return self.value != NO_REPLY - def handle(self): - """ - Reply are handled by controller.handlers, which may be nested. The first handler takes - responsibility and handles the reply. - """ - if self.state != "unhandled": - raise exceptions.ControlException("Reply is {}, but expected it to be unhandled.".format(self.state)) - self._state = "handled" - def take(self): """ Scripts or other parties make "take" a reply out of a normal flow. For example, intercepted flows are taken out so that the connection thread does not proceed. """ - if self.state != "handled": - raise exceptions.ControlException("Reply is {}, but expected it to be handled.".format(self.state)) + if self.state != "start": + raise exceptions.ControlException( + "Reply is {}, but expected it to be start.".format(self.state) + ) self._state = "taken" def commit(self): """ - Ultimately, messages are commited. This is done either automatically by the handler - if the message is not taken or manually by the entity which called .take(). + Ultimately, messages are commited. This is done either automatically by + if the message is not taken or manually by the entity which called + .take(). """ if self.state != "taken": - raise exceptions.ControlException("Reply is {}, but expected it to be taken.".format(self.state)) + raise exceptions.ControlException( + "Reply is {}, but expected it to be taken.".format(self.state) + ) if not self.has_message: raise exceptions.ControlException("There is no reply message.") self._state = "committed" self.q.put(self.value) def ack(self, force=False): + if self.state not in {"start", "taken"}: + raise exceptions.ControlException( + "Reply is {}, but expected it to be start or taken.".format(self.state) + ) self.send(self.obj, force) def kill(self, force=False): self.send(exceptions.Kill, force) def send(self, msg, force=False): - if self.state not in ("handled", "taken"): - raise exceptions.ControlException( - "Reply is {}, did not expect a call to .send().".format(self.state) - ) if self.has_message and not force: raise exceptions.ControlException("There is already a reply message.") self.value = msg @@ -175,7 +128,7 @@ class Reply: class DummyReply(Reply): """ A reply object that is not connected to anything. In contrast to regular - Reply objects, DummyReply objects are reset to "unhandled" at the end of an + Reply objects, DummyReply objects are reset to "start" at the end of an handler so that they can be used multiple times. Useful when we need an object to seem like it has a channel, and during testing. """ @@ -190,7 +143,7 @@ class DummyReply(Reply): def reset(self): if self._should_reset: - self._state = "unhandled" + self._state = "start" self.value = NO_REPLY def __del__(self): diff --git a/mitmproxy/exceptions.py b/mitmproxy/exceptions.py index 309b8189..9b6328ac 100644 --- a/mitmproxy/exceptions.py +++ b/mitmproxy/exceptions.py @@ -102,6 +102,9 @@ class AddonError(MitmproxyException): class AddonHalt(MitmproxyException): + """ + Raised by addons to signal that no further handlers should handle this event. + """ pass diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index bcc55559..294aba26 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -144,7 +144,7 @@ class Flow(stateobject.StateObject): @property def killable(self): - return self.reply and self.reply.state in {"handled", "taken"} + return self.reply and self.reply.state == "taken" def kill(self): """ @@ -152,8 +152,9 @@ class Flow(stateobject.StateObject): """ self.error = Error("Connection killed") self.intercepted = False - # reply.state should only be "handled" or "taken" here. - # if none of this is the case, .take() will raise an exception. + + # reply.state should be "taken" here, or .take() will raise an + # exception. if self.reply.state != "taken": self.reply.take() self.reply.kill(force=True) diff --git a/mitmproxy/master.py b/mitmproxy/master.py index 79747a97..946b25a4 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -1,8 +1,6 @@ -import os import threading import contextlib import queue -import sys from mitmproxy import addonmanager from mitmproxy import options @@ -12,10 +10,8 @@ from mitmproxy import exceptions from mitmproxy import connections from mitmproxy import http from mitmproxy import log -from mitmproxy import io from mitmproxy.proxy.protocol import http_replay from mitmproxy.types import basethread -import mitmproxy.net.http from . import ctx as mitmproxy_ctx @@ -68,8 +64,7 @@ class Master: """ level: debug, info, warn, error """ - with self.handlecontext(): - self.addons("log", log.LogEntry(e, level)) + self.addons.trigger("log", log.LogEntry(e, level)) def start(self): self.should_exit.clear() @@ -89,9 +84,8 @@ class Master: def tick(self, timeout): if self.first_tick: self.first_tick = False - self.addons.invoke_all_with_context("running") - with self.handlecontext(): - self.addons("tick") + self.addons.trigger("running") + self.addons.trigger("tick") changed = False try: mtype, obj = self.event_queue.get(timeout=timeout) @@ -99,18 +93,7 @@ class Master: raise exceptions.ControlException( "Unknown event %s" % repr(mtype) ) - handle_func = getattr(self, mtype) - if not callable(handle_func): - raise exceptions.ControlException( - "Handler %s not callable" % mtype - ) - if not handle_func.__dict__.get("__handler"): - raise exceptions.ControlException( - "Handler function %s is not decorated with controller.handler" % ( - handle_func - ) - ) - handle_func(obj) + self.addons.handle_lifecycle(mtype, obj) self.event_queue.task_done() changed = True except queue.Empty: @@ -122,27 +105,18 @@ class Master: self.should_exit.set() self.addons.done() - def create_request(self, method, scheme, host, port, path): + def create_request(self, method, url): """ - this method creates a new artificial and minimalist request also adds it to flowlist + Create a new artificial and minimalist request also adds it to flowlist. + + Raises: + ValueError, if the url is malformed. """ + req = http.HTTPRequest.make(method, url) c = connections.ClientConnection.make_dummy(("", 0)) - s = connections.ServerConnection.make_dummy((host, port)) + s = connections.ServerConnection.make_dummy((req.host, req.port)) f = http.HTTPFlow(c, s) - headers = mitmproxy.net.http.Headers() - - req = http.HTTPRequest( - "absolute", - method, - scheme, - host, - port, - path, - b"HTTP/1.1", - headers, - b"" - ) f.request = req self.load_flow(f) return f @@ -158,34 +132,7 @@ class Master: f.request.scheme = self.server.config.upstream_server.scheme f.reply = controller.DummyReply() for e, o in eventsequence.iterate(f): - getattr(self, e)(o) - - def load_flows(self, fr: io.FlowReader) -> int: - """ - Load flows from a FlowReader object. - """ - cnt = 0 - for i in fr.stream(): - cnt += 1 - self.load_flow(i) - return cnt - - def load_flows_file(self, path: str) -> int: - path = os.path.expanduser(path) - try: - if path == "-": - try: - sys.stdin.buffer.read(0) - except Exception as e: - raise IOError("Cannot read from stdin: {}".format(e)) - freader = io.FlowReader(sys.stdin.buffer) - return self.load_flows(freader) - else: - with open(path, "rb") as f: - freader = io.FlowReader(f) - return self.load_flows(freader) - except IOError as v: - raise exceptions.FlowReadException(v.strerror) + self.addons.handle_lifecycle(e, o) def replay_request( self, @@ -241,87 +188,3 @@ class Master: if block: rt.join() return rt - - @controller.handler - def log(self, l): - pass - - @controller.handler - def clientconnect(self, root_layer): - pass - - @controller.handler - def clientdisconnect(self, root_layer): - pass - - @controller.handler - def serverconnect(self, server_conn): - pass - - @controller.handler - def serverdisconnect(self, server_conn): - pass - - @controller.handler - def next_layer(self, top_layer): - pass - - @controller.handler - def http_connect(self, f): - pass - - @controller.handler - def error(self, f): - pass - - @controller.handler - def requestheaders(self, f): - pass - - @controller.handler - def request(self, f): - pass - - @controller.handler - def responseheaders(self, f): - pass - - @controller.handler - def response(self, f): - pass - - @controller.handler - def websocket_handshake(self, f): - pass - - @controller.handler - def websocket_start(self, flow): - pass - - @controller.handler - def websocket_message(self, flow): - pass - - @controller.handler - def websocket_error(self, flow): - pass - - @controller.handler - def websocket_end(self, flow): - pass - - @controller.handler - def tcp_start(self, flow): - pass - - @controller.handler - def tcp_message(self, flow): - pass - - @controller.handler - def tcp_error(self, flow): - pass - - @controller.handler - def tcp_end(self, flow): - pass diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index b961e1e4..90a1f924 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -1,6 +1,6 @@ import re import urllib -from typing import Optional +from typing import Optional, AnyStr, Dict, Iterable, Tuple, Union from mitmproxy.types import multidict from mitmproxy.utils import strutils @@ -77,6 +77,53 @@ class Request(message.Message): self.method, hostport, path ) + @classmethod + def make( + cls, + method: str, + url: str, + content: AnyStr = b"", + headers: Union[Dict[AnyStr, AnyStr], Iterable[Tuple[bytes, bytes]]] = () + ): + """ + Simplified API for creating request objects. + """ + req = cls( + "absolute", + method, + "", + "", + "", + "", + "HTTP/1.1", + (), + b"" + ) + + req.url = url + + # Headers can be list or dict, we differentiate here. + if isinstance(headers, dict): + req.headers = nheaders.Headers(**headers) + elif isinstance(headers, Iterable): + req.headers = nheaders.Headers(headers) + else: + raise TypeError("Expected headers to be an iterable or dict, but is {}.".format( + type(headers).__name__ + )) + + # Assign this manually to update the content-length header. + if isinstance(content, bytes): + req.content = content + elif isinstance(content, str): + req.text = content + else: + raise TypeError("Expected content to be str or bytes, but is {}.".format( + type(content).__name__ + )) + + return req + def replace(self, pattern, repl, flags=0, count=0): """ Replaces a regular expression pattern with repl in the headers, the diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 036b3d29..5b84ac93 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -79,7 +79,10 @@ class Options(optmanager.OptManager): ) self.add_option( "keepserving", bool, False, - "Continue serving after client playback or file read." + """ + Instructs mitmdump to continue serving after client playback, + server playback or file read. This option is ignored by interactive tools, which always keep serving. + """ ) self.add_option( "server", bool, True, diff --git a/mitmproxy/test/taddons.py b/mitmproxy/test/taddons.py index 8d6baa12..8bc174c7 100644 --- a/mitmproxy/test/taddons.py +++ b/mitmproxy/test/taddons.py @@ -3,19 +3,45 @@ import contextlib import mitmproxy.master import mitmproxy.options from mitmproxy import proxy +from mitmproxy import addonmanager from mitmproxy import eventsequence +class TestAddons(addonmanager.AddonManager): + def __init__(self, master): + super().__init__(master) + + def trigger(self, event, *args, **kwargs): + if event == "log": + self.master.logs.append(args[0]) + else: + self.master.events.append((event, args, kwargs)) + super().trigger(event, *args, **kwargs) + + class RecordingMaster(mitmproxy.master.Master): def __init__(self, *args, **kwargs): super().__init__(*args, **kwargs) - self.event_log = [] + self.addons = TestAddons(self) + self.events = [] + self.logs = [] - def add_log(self, e, level): - self.event_log.append((level, e)) + def has_log(self, txt, level=None): + for i in self.logs: + if level and i.level != level: + continue + if txt.lower() in i.msg.lower(): + return True + return False + + def has_event(self, name): + for i in self.events: + if i[0] == name: + return True + return False def clear(self): - self.event_log = [] + self.logs = [] class context: @@ -47,7 +73,7 @@ class context: Cycles the flow through the events for the flow. Stops if a reply is taken (as in flow interception). """ - f.reply._state = "handled" + f.reply._state = "start" for evt, arg in eventsequence.iterate(f): h = getattr(addon, evt, None) if h: diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index d2e28d35..5fe86975 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -1,6 +1,5 @@ import urwid -import mitmproxy.net.http.url from mitmproxy import exceptions from mitmproxy.tools.console import common from mitmproxy.tools.console import signals @@ -339,12 +338,10 @@ class FlowListBox(urwid.ListBox): def new_request(self, url, method): try: - parts = mitmproxy.net.http.url.parse(str(url)) + f = self.master.create_request(method, url) except ValueError as e: signals.status_message.send(message = "Invalid URL: " + str(e)) return - scheme, host, port, path = parts - f = self.master.create_request(method, scheme, host, port, path) self.master.view.focus.flow = f def keypress(self, size, key): diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index e75105cf..d0e23712 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -13,7 +13,6 @@ import traceback import urwid from mitmproxy import addons -from mitmproxy import controller from mitmproxy import exceptions from mitmproxy import master from mitmproxy import io @@ -40,6 +39,35 @@ class Logger: signals.add_log(evt.msg, evt.level) +class UnsupportedLog: + """ + A small addon to dump info on flow types we don't support yet. + """ + def websocket_message(self, f): + message = f.messages[-1] + signals.add_log(f.message_info(message), "info") + signals.add_log(strutils.bytes_to_escaped_str(message.content), "debug") + + def websocket_end(self, f): + signals.add_log("WebSocket connection closed by {}: {} {}, {}".format( + f.close_sender, + f.close_code, + f.close_message, + f.close_reason), "info") + + def tcp_message(self, f): + message = f.messages[-1] + direction = "->" if message.from_client else "<-" + signals.add_log("{client_host}:{client_port} {direction} tcp {direction} {server_host}:{server_port}".format( + client_host=f.client_conn.address[0], + client_port=f.client_conn.address[1], + server_host=f.server_conn.address[0], + server_port=f.server_conn.address[1], + direction=direction, + ), "info") + signals.add_log(strutils.bytes_to_escaped_str(message.content), "debug") + + class ConsoleMaster(master.Master): palette = [] @@ -63,7 +91,7 @@ class ConsoleMaster(master.Master): signals.sig_add_log.connect(self.sig_add_log) self.addons.add(Logger()) self.addons.add(*addons.default_addons()) - self.addons.add(intercept.Intercept(), self.view) + self.addons.add(intercept.Intercept(), self.view, UnsupportedLog()) def sigint_handler(*args, **kwargs): self.prompt_for_exit() @@ -256,19 +284,6 @@ class ConsoleMaster(master.Master): ) self.ab = statusbar.ActionBar() - if self.options.rfile: - ret = self.load_flows_path(self.options.rfile) - if ret and self.view.store_count(): - signals.add_log( - "File truncated or corrupted. " - "Loaded as many flows as possible.", - "error" - ) - elif ret and not self.view.store_count(): - self.shutdown() - print("Could not load file: {}".format(ret), file=sys.stderr) - sys.exit(1) - self.loop.set_alarm_in(0.01, self.ticker) self.loop.set_alarm_in( @@ -289,7 +304,10 @@ class ConsoleMaster(master.Master): print("Shutting down...", file=sys.stderr) finally: sys.stderr.flush() - self.shutdown() + super().shutdown() + + def shutdown(self): + raise urwid.ExitMainLoop def view_help(self, helpctx): signals.push_view_state.send( @@ -402,38 +420,7 @@ class ConsoleMaster(master.Master): def quit(self, a): if a != "n": - raise urwid.ExitMainLoop + self.shutdown() def clear_events(self): self.logbuffer[:] = [] - - # Handlers - @controller.handler - def websocket_message(self, f): - super().websocket_message(f) - message = f.messages[-1] - signals.add_log(f.message_info(message), "info") - signals.add_log(strutils.bytes_to_escaped_str(message.content), "debug") - - @controller.handler - def websocket_end(self, f): - super().websocket_end(f) - signals.add_log("WebSocket connection closed by {}: {} {}, {}".format( - f.close_sender, - f.close_code, - f.close_message, - f.close_reason), "info") - - @controller.handler - def tcp_message(self, f): - super().tcp_message(f) - message = f.messages[-1] - direction = "->" if message.from_client else "<-" - signals.add_log("{client_host}:{client_port} {direction} tcp {direction} {server_host}:{server_port}".format( - client_host=f.client_conn.address[0], - client_port=f.client_conn.address[1], - server_host=f.server_conn.address[0], - server_port=f.server_conn.address[1], - direction=direction, - ), "info") - signals.add_log(strutils.bytes_to_escaped_str(message.content), "debug") diff --git a/mitmproxy/tools/dump.py b/mitmproxy/tools/dump.py index 4bfe2dc4..e6f0c3df 100644 --- a/mitmproxy/tools/dump.py +++ b/mitmproxy/tools/dump.py @@ -1,9 +1,16 @@ -from mitmproxy import controller -from mitmproxy import exceptions from mitmproxy import addons from mitmproxy import options from mitmproxy import master -from mitmproxy.addons import dumper, termlog, termstatus +from mitmproxy.addons import dumper, termlog, termstatus, readstdin, keepserving + + +class ErrorCheck: + def __init__(self): + self.has_errored = False + + def log(self, e): + if e.level == "error": + self.has_errored = True class DumpMaster(master.Master): @@ -16,27 +23,14 @@ class DumpMaster(master.Master): with_dumper=True, ) -> None: master.Master.__init__(self, options, server) - self.has_errored = False + self.errorcheck = ErrorCheck() if with_termlog: self.addons.add(termlog.TermLog(), termstatus.TermStatus()) self.addons.add(*addons.default_addons()) if with_dumper: self.addons.add(dumper.Dumper()) - - if options.rfile: - try: - self.load_flows_file(options.rfile) - except exceptions.FlowReadException as v: - self.add_log("Flow file corrupted.", "error") - raise exceptions.OptionsError(v) - - @controller.handler - def log(self, e): - if e.level == "error": - self.has_errored = True - - def run(self): # pragma: no cover - if self.options.rfile and not self.options.keepserving: - self.addons.done() - return - super().run() + self.addons.add( + readstdin.ReadStdin(), + keepserving.KeepServing(), + self.errorcheck + ) diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 35567b62..b321e8f8 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -91,9 +91,7 @@ def run(MasterKlass, args): # pragma: no cover sys.exit(1) except (KeyboardInterrupt, RuntimeError): pass - if master is None or getattr(master, "has_errored", None): - print("%s: errors occurred during run" % sys.argv[0], file=sys.stderr) - sys.exit(1) + return master def mitmproxy(args=None): # pragma: no cover @@ -109,7 +107,9 @@ def mitmproxy(args=None): # pragma: no cover def mitmdump(args=None): # pragma: no cover from mitmproxy.tools import dump - run(dump.DumpMaster, args) + m = run(dump.DumpMaster, args) + if m and m.errorcheck.has_errored: + sys.exit(1) def mitmweb(args=None): # pragma: no cover diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index eddaa3e1..002513b9 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -230,7 +230,8 @@ class DumpFlows(RequestHandler): def post(self): self.view.clear() bio = BytesIO(self.filecontents) - self.master.load_flows(io.FlowReader(bio)) + for i in io.FlowReader(bio).stream(): + self.master.load_flow(i) bio.close() diff --git a/mitmproxy/tools/web/master.py b/mitmproxy/tools/web/master.py index 8c7f579d..e28bd002 100644 --- a/mitmproxy/tools/web/master.py +++ b/mitmproxy/tools/web/master.py @@ -3,7 +3,6 @@ import webbrowser import tornado.httpserver import tornado.ioloop from mitmproxy import addons -from mitmproxy import exceptions from mitmproxy import log from mitmproxy import master from mitmproxy.addons import eventstore @@ -42,14 +41,6 @@ class WebMaster(master.Master): ) # This line is just for type hinting self.options = self.options # type: Options - if options.rfile: - try: - self.load_flows_file(options.rfile) - except exceptions.FlowReadException as v: - self.add_log( - "Could not read flow file: %s" % v, - "error" - ) def _sig_view_add(self, view, flow): app.ClientConnection.broadcast( diff --git a/release/README.md b/release/README.md index 02fabfe9..a30221c8 100644 --- a/release/README.md +++ b/release/README.md @@ -1,14 +1,25 @@ # Release Checklist +Make sure run all these steps on the correct branch you want to create a new release for! +- Verify `mitmproxy/version.py` - Update CHANGELOG -- Verify that all CI tests pass for current master -- Tag the release, and push to Github +- Verify that all CI tests pass +- Tag the release and push to Github - Wait for tag CI to complete -- Download assets from snapshots.mitmproxy.org -- Create release notice on Github -- Upload wheel to pypi (`twine upload wheelname`) + +## GitHub Release +- Create release notice on Github [https://github.com/mitmproxy/mitmproxy/releases/new](here) +- Attach all files from the new release folder on https://snapshots.mitmproxy.org + +## PyPi +- Upload wheel to pypi: `twine upload <mitmproxy-...-.whl` + +## Docker - Update docker-releases repo - Create a new branch based of master for major versions. - - Add a commit that pins dependencies like so: https://github.com/mitmproxy/docker-releases/commit/3d6a9989fde068ad0aea257823ac3d7986ff1613. The requirements can be obtained by creating a fresh venv, pip-installing the new wheel in there, and then running `pip freeze`. -- Update `latest` tag on https://hub.docker.com/r/mitmproxy/mitmproxy/~/settings/automated-builds/ -- Bump the version in https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py and update https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/io_compat.py in the next commit + - Update the dependencies in [alpine/requirements.txt](https://github.com/mitmproxy/docker-releases/commit/3d6a9989fde068ad0aea257823ac3d7986ff1613#diff-9b7e0eea8ae74688b1ac13ea080549ba) + * Creating a fresh venv, pip-installing the new wheel in there, and then export all packages: + * `virtualenv -ppython3.5 venv && source venv/bin/activate && pip install mitmproxy && pip freeze` +- Update `latest` tag [https://hub.docker.com/r/mitmproxy/mitmproxy/~/settings/automated-builds/](here) + +After everything is done, you might want to bump the version on master in [https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/version.py](mitmproxy/version.py) if you just created a major release. @@ -64,17 +64,17 @@ setup( "click>=6.2, <7", "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! "construct>=2.8, <2.9", - "cryptography>=1.3, <1.8", + "cryptography>=1.3, <1.9", "cssutils>=1.0.1, <1.1", - "h2>=2.5.1, <3", + "h2>=2.6.1, <3", "html2text>=2016.1.8, <=2016.9.19", - "hyperframe>=4.0.1, <5", + "hyperframe>=4.0.2, <6", "jsbeautifier>=1.6.3, <1.7", "kaitaistruct>=0.6, <0.7", "passlib>=1.6.5, <1.8", "pyasn1>=0.1.9, <0.3", "pyOpenSSL>=16.0, <17.0", - "pyparsing>=2.1.3, <2.2", + "pyparsing>=2.1.3, <2.3", "pyperclip>=1.5.22, <1.6", "requests>=2.9.1, <3", "ruamel.yaml>=0.13.2, <0.14", @@ -96,7 +96,7 @@ setup( 'dev': [ "Flask>=0.10.1, <0.13", "flake8>=3.2.1, <3.4", - "mypy>=0.471, <0.480", + "mypy>=0.471, <0.502", "rstcheck>=2.2, <4.0", "tox>=2.3, <3", "pytest>=3, <3.1", @@ -106,8 +106,8 @@ setup( "pytest-faulthandler>=1.3.0, <2", "sphinx>=1.3.5, <1.6", "sphinx-autobuild>=0.5.2, <0.7", - "sphinxcontrib-documentedlist>=0.5.0, <0.6", - "sphinx_rtd_theme>=0.1.9, <0.2", + "sphinxcontrib-documentedlist>=0.5.0, <0.7", + "sphinx_rtd_theme>=0.1.9, <0.3", ], 'contentviews': [ ], diff --git a/test/mitmproxy/addons/test_check_alpn.py b/test/mitmproxy/addons/test_check_alpn.py index 2dc0c835..2b1d6058 100644 --- a/test/mitmproxy/addons/test_check_alpn.py +++ b/test/mitmproxy/addons/test_check_alpn.py @@ -12,7 +12,7 @@ class TestCheckALPN: with taddons.context() as tctx: a = check_alpn.CheckALPN() tctx.configure(a) - assert not any(msg in m for l, m in tctx.master.event_log) + assert not tctx.master.has_log(msg) def test_check_no_alpn(self, disable_alpn): msg = 'ALPN support missing' @@ -20,4 +20,4 @@ class TestCheckALPN: with taddons.context() as tctx: a = check_alpn.CheckALPN() tctx.configure(a) - assert any(msg in m for l, m in tctx.master.event_log) + assert tctx.master.has_log(msg) diff --git a/test/mitmproxy/addons/test_check_ca.py b/test/mitmproxy/addons/test_check_ca.py index fc64621c..cd34a9be 100644 --- a/test/mitmproxy/addons/test_check_ca.py +++ b/test/mitmproxy/addons/test_check_ca.py @@ -16,4 +16,4 @@ class TestCheckCA: tctx.master.server.config.certstore.default_ca.has_expired = mock.MagicMock(return_value=expired) a = check_ca.CheckCA() tctx.configure(a) - assert any(msg in m for l, m in tctx.master.event_log) is expired + assert tctx.master.has_log(msg) is expired diff --git a/test/mitmproxy/addons/test_clientplayback.py b/test/mitmproxy/addons/test_clientplayback.py index c22b3589..f71662f0 100644 --- a/test/mitmproxy/addons/test_clientplayback.py +++ b/test/mitmproxy/addons/test_clientplayback.py @@ -23,7 +23,7 @@ class MockThread(): class TestClientPlayback: def test_playback(self): cp = clientplayback.ClientPlayback() - with taddons.context(): + with taddons.context() as tctx: assert cp.count() == 0 f = tflow.tflow(resp=True) cp.load([f]) @@ -35,17 +35,14 @@ class TestClientPlayback: assert rp.called assert cp.current_thread - cp.keepserving = False cp.flows = None cp.current_thread = None - with mock.patch("mitmproxy.master.Master.shutdown") as sd: - cp.tick() - assert sd.called + cp.tick() + assert tctx.master.has_event("processing_complete") cp.current_thread = MockThread() - with mock.patch("mitmproxy.master.Master.shutdown") as sd: - cp.tick() - assert cp.current_thread is None + cp.tick() + assert cp.current_thread is None def test_configure(self, tmpdir): cp = clientplayback.ClientPlayback() diff --git a/test/mitmproxy/addons/test_disable_h2c.py b/test/mitmproxy/addons/test_disable_h2c.py index d4df8390..cf20a368 100644 --- a/test/mitmproxy/addons/test_disable_h2c.py +++ b/test/mitmproxy/addons/test_disable_h2c.py @@ -31,7 +31,6 @@ class TestDisableH2CleartextUpgrade: b = io.BytesIO(b"PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n") f = tflow.tflow() f.request = http.HTTPRequest.wrap(http1.read_request(b)) - f.reply.handle() f.intercept() a.request(f) diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index 47374617..fbcc4d16 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -151,7 +151,7 @@ class TestContentView: with taddons.context(options=options.Options()) as ctx: ctx.configure(d, flow_detail=4, verbosity=3) d.response(tflow.tflow()) - assert "Content viewer failed" in ctx.master.event_log[0][1] + assert ctx.master.has_log("content viewer failed") def test_tcp(): diff --git a/test/mitmproxy/addons/test_intercept.py b/test/mitmproxy/addons/test_intercept.py index 465e6433..f436a817 100644 --- a/test/mitmproxy/addons/test_intercept.py +++ b/test/mitmproxy/addons/test_intercept.py @@ -29,6 +29,5 @@ def test_simple(): assert not f.intercepted f = tflow.tflow(resp=True) - f.reply._state = "handled" r.response(f) assert f.intercepted diff --git a/test/mitmproxy/addons/test_keepserving.py b/test/mitmproxy/addons/test_keepserving.py new file mode 100644 index 00000000..70f7e1e6 --- /dev/null +++ b/test/mitmproxy/addons/test_keepserving.py @@ -0,0 +1,10 @@ +from mitmproxy.addons import keepserving +from mitmproxy.test import taddons + + +def test_keepserving(): + ks = keepserving.KeepServing() + + with taddons.context() as tctx: + ks.event_processing_complete() + assert tctx.master.should_exit.is_set() diff --git a/test/mitmproxy/addons/test_readfile.py b/test/mitmproxy/addons/test_readfile.py new file mode 100644 index 00000000..b30c147b --- /dev/null +++ b/test/mitmproxy/addons/test_readfile.py @@ -0,0 +1,62 @@ +from mitmproxy.addons import readfile +from mitmproxy.test import taddons +from mitmproxy.test import tflow +from mitmproxy import io +from mitmproxy import exceptions +from unittest import mock + +import pytest + + +def write_data(path, corrupt=False): + with open(path, "wb") as tf: + w = io.FlowWriter(tf) + for i in range(3): + f = tflow.tflow(resp=True) + w.add(f) + for i in range(3): + f = tflow.tflow(err=True) + w.add(f) + f = tflow.ttcpflow() + w.add(f) + f = tflow.ttcpflow(err=True) + w.add(f) + if corrupt: + tf.write(b"flibble") + + +@mock.patch('mitmproxy.master.Master.load_flow') +def test_configure(mck, tmpdir): + + rf = readfile.ReadFile() + with taddons.context() as tctx: + tf = str(tmpdir.join("tfile")) + write_data(tf) + tctx.configure(rf, rfile=str(tf)) + assert not mck.called + rf.running() + assert mck.called + + write_data(tf, corrupt=True) + tctx.configure(rf, rfile=str(tf)) + with pytest.raises(exceptions.OptionsError): + rf.running() + + +@mock.patch('mitmproxy.master.Master.load_flow') +def test_corruption(mck, tmpdir): + + rf = readfile.ReadFile() + with taddons.context() as tctx: + with pytest.raises(exceptions.FlowReadException): + rf.load_flows_file("nonexistent") + assert not mck.called + assert len(tctx.master.logs) == 1 + + tfc = str(tmpdir.join("tfile")) + write_data(tfc, corrupt=True) + + with pytest.raises(exceptions.FlowReadException): + rf.load_flows_file(tfc) + assert mck.called + assert len(tctx.master.logs) == 2 diff --git a/test/mitmproxy/addons/test_readstdin.py b/test/mitmproxy/addons/test_readstdin.py new file mode 100644 index 00000000..76b01f4f --- /dev/null +++ b/test/mitmproxy/addons/test_readstdin.py @@ -0,0 +1,53 @@ + +import io +from mitmproxy.addons import readstdin +from mitmproxy.test import taddons +from mitmproxy.test import tflow +import mitmproxy.io +from unittest import mock + + +def gen_data(corrupt=False): + tf = io.BytesIO() + w = mitmproxy.io.FlowWriter(tf) + for i in range(3): + f = tflow.tflow(resp=True) + w.add(f) + for i in range(3): + f = tflow.tflow(err=True) + w.add(f) + f = tflow.ttcpflow() + w.add(f) + f = tflow.ttcpflow(err=True) + w.add(f) + if corrupt: + tf.write(b"flibble") + tf.seek(0) + return tf + + +class mStdin: + def __init__(self, d): + self.buffer = d + + def isatty(self): + return False + + +@mock.patch('mitmproxy.master.Master.load_flow') +def test_read(m, tmpdir): + rf = readstdin.ReadStdin() + with taddons.context() as tctx: + assert not m.called + rf.running(stdin=mStdin(gen_data())) + assert m.called + + rf.running(stdin=mStdin(None)) + assert tctx.master.logs + tctx.master.clear() + + m.reset_mock() + assert not m.called + rf.running(stdin=mStdin(gen_data(corrupt=True))) + assert m.called + assert tctx.master.logs diff --git a/test/mitmproxy/addons/test_replace.py b/test/mitmproxy/addons/test_replace.py index 7d590b35..9002afb5 100644 --- a/test/mitmproxy/addons/test_replace.py +++ b/test/mitmproxy/addons/test_replace.py @@ -97,6 +97,6 @@ class TestReplaceFile: tmpfile.remove() f = tflow.tflow() f.request.content = b"foo" - assert not tctx.master.event_log + assert not tctx.master.logs r.request(f) - assert tctx.master.event_log + assert tctx.master.logs diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index 4c1b2e43..16827488 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -22,14 +22,12 @@ def test_scriptenv(): with taddons.context() as tctx: with script.scriptenv("path", []): raise SystemExit - assert tctx.master.event_log[0][0] == "error" - assert "exited" in tctx.master.event_log[0][1] + assert tctx.master.has_log("exited", "error") tctx.master.clear() with script.scriptenv("path", []): raise ValueError("fooo") - assert tctx.master.event_log[0][0] == "error" - assert "foo" in tctx.master.event_log[0][1] + assert tctx.master.has_log("fooo", "error") class Called: @@ -135,7 +133,7 @@ class TestScript: f.write(".") sc.tick() time.sleep(0.1) - if tctx.master.event_log: + if tctx.master.logs: return raise AssertionError("Change event not detected.") @@ -147,11 +145,11 @@ class TestScript: sc.start(tctx.options) f = tflow.tflow(resp=True) sc.request(f) - assert tctx.master.event_log[0][0] == "error" - assert len(tctx.master.event_log[0][1].splitlines()) == 6 - assert re.search(r'addonscripts[\\/]error.py", line \d+, in request', tctx.master.event_log[0][1]) - assert re.search(r'addonscripts[\\/]error.py", line \d+, in mkerr', tctx.master.event_log[0][1]) - assert tctx.master.event_log[0][1].endswith("ValueError: Error!\n") + assert tctx.master.logs[0].level == "error" + assert len(tctx.master.logs[0].msg.splitlines()) == 6 + assert re.search(r'addonscripts[\\/]error.py", line \d+, in request', tctx.master.logs[0].msg) + assert re.search(r'addonscripts[\\/]error.py", line \d+, in mkerr', tctx.master.logs[0].msg) + assert tctx.master.logs[0].msg.endswith("ValueError: Error!\n") def test_addon(self): with taddons.context() as tctx: @@ -256,21 +254,21 @@ class TestScriptLoader: "%s %s" % (rec, "c"), ] ) - debug = [(i[0], i[1]) for i in tctx.master.event_log if i[0] == "debug"] + debug = [i.msg for i in tctx.master.logs if i.level == "debug"] assert debug == [ - ('debug', 'a start'), - ('debug', 'a configure'), - ('debug', 'a running'), + 'a start', + 'a configure', + 'a running', - ('debug', 'b start'), - ('debug', 'b configure'), - ('debug', 'b running'), + 'b start', + 'b configure', + 'b running', - ('debug', 'c start'), - ('debug', 'c configure'), - ('debug', 'c running'), + 'c start', + 'c configure', + 'c running', ] - tctx.master.event_log = [] + tctx.master.logs = [] tctx.configure( sc, scripts = [ @@ -279,11 +277,10 @@ class TestScriptLoader: "%s %s" % (rec, "b"), ] ) - debug = [(i[0], i[1]) for i in tctx.master.event_log if i[0] == "debug"] - # No events, only order has changed + debug = [i.msg for i in tctx.master.logs if i.level == "debug"] assert debug == [] - tctx.master.event_log = [] + tctx.master.logs = [] tctx.configure( sc, scripts = [ @@ -291,11 +288,11 @@ class TestScriptLoader: "%s %s" % (rec, "a"), ] ) - debug = [(i[0], i[1]) for i in tctx.master.event_log if i[0] == "debug"] + debug = [i.msg for i in tctx.master.logs if i.level == "debug"] assert debug == [ - ('debug', 'c done'), - ('debug', 'b done'), - ('debug', 'x start'), - ('debug', 'x configure'), - ('debug', 'x running'), + 'c done', + 'b done', + 'x start', + 'x configure', + 'x running', ] diff --git a/test/mitmproxy/addons/test_serverplayback.py b/test/mitmproxy/addons/test_serverplayback.py index 54e4d281..02642c35 100644 --- a/test/mitmproxy/addons/test_serverplayback.py +++ b/test/mitmproxy/addons/test_serverplayback.py @@ -34,7 +34,7 @@ def test_tick(): s.final_flow = tflow.tflow() s.final_flow.live = False s.tick() - assert tctx.master.should_exit.is_set() + assert tctx.master.has_event("processing_complete") def test_server_playback(): @@ -315,7 +315,6 @@ def test_server_playback_full(): tctx.configure( s, refresh_server_playback = True, - keepserving=False ) f = tflow.tflow() diff --git a/test/mitmproxy/addons/test_termstatus.py b/test/mitmproxy/addons/test_termstatus.py index 01c14814..7becc857 100644 --- a/test/mitmproxy/addons/test_termstatus.py +++ b/test/mitmproxy/addons/test_termstatus.py @@ -6,7 +6,7 @@ def test_configure(): ts = termstatus.TermStatus() with taddons.context() as ctx: ts.running() - assert not ctx.master.event_log + assert not ctx.master.logs ctx.configure(ts, server=True) ts.running() - assert ctx.master.event_log + assert ctx.master.logs diff --git a/test/mitmproxy/contentviews/test_xml_html.py b/test/mitmproxy/contentviews/test_xml_html.py index 899ecfde..2b0aee4d 100644 --- a/test/mitmproxy/contentviews/test_xml_html.py +++ b/test/mitmproxy/contentviews/test_xml_html.py @@ -23,7 +23,7 @@ def test_format_xml(filename): path = data.path(filename) with open(path) as f: input = f.read() - with open(path.replace(".", "-formatted.")) as f: + with open("-formatted.".join(path.rsplit(".", 1))) as f: expected = f.read() tokens = xml_html.tokenize(input) assert xml_html.format_xml(tokens) == expected diff --git a/test/mitmproxy/net/http/test_request.py b/test/mitmproxy/net/http/test_request.py index 90ec31fe..ce49002c 100644 --- a/test/mitmproxy/net/http/test_request.py +++ b/test/mitmproxy/net/http/test_request.py @@ -1,7 +1,7 @@ from unittest import mock import pytest -from mitmproxy.net.http import Headers +from mitmproxy.net.http import Headers, Request from mitmproxy.test.tutils import treq from .test_message import _test_decoded_attr, _test_passthrough_attr @@ -35,6 +35,32 @@ class TestRequestCore: request.host = None assert repr(request) == "Request(GET /path)" + def test_make(self): + r = Request.make("GET", "https://example.com/") + assert r.method == "GET" + assert r.scheme == "https" + assert r.host == "example.com" + assert r.port == 443 + assert r.path == "/" + + r = Request.make("GET", "https://example.com/", "content", {"Foo": "bar"}) + assert r.content == b"content" + assert r.headers["content-length"] == "7" + assert r.headers["Foo"] == "bar" + + Request.make("GET", "https://example.com/", content=b"content") + with pytest.raises(TypeError): + Request.make("GET", "https://example.com/", content=42) + + r = Request.make("GET", "https://example.com/", headers=[(b"foo", b"bar")]) + assert r.headers["foo"] == "bar" + + r = Request.make("GET", "https://example.com/", headers=({"foo": "baz"})) + assert r.headers["foo"] == "baz" + + with pytest.raises(TypeError): + Request.make("GET", "https://example.com/", headers=42) + def test_replace(self): r = treq() r.path = b"foobarfoo" diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 16efe415..447b15a5 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -5,7 +5,6 @@ import pytest from unittest import mock from mitmproxy.test import tutils -from mitmproxy import controller from mitmproxy import options from mitmproxy.addons import script from mitmproxy.addons import proxyauth @@ -250,17 +249,12 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin): assert p.request(req) def test_get_connection_switching(self): - def switched(l): - for i in l: - if "serverdisconnect" in i: - return True - req = "get:'%s/p/200:b@1'" p = self.pathoc() with p.connect(): assert p.request(req % self.server.urlbase) assert p.request(req % self.server2.urlbase) - assert switched(self.proxy.tlog) + assert self.proxy.tmaster.has_log("serverdisconnect") def test_blank_leading_line(self): p = self.pathoc() @@ -602,7 +596,7 @@ class TestHttps2Http(tservers.ReverseProxyTest): p = self.pathoc(ssl=True, sni="example.com") with p.connect(): assert p.request("get:'/p/200'").status_code == 200 - assert all("Error in handle_sni" not in msg for msg in self.proxy.tlog) + assert not self.proxy.tmaster.has_log("error in handle_sni") def test_http(self): p = self.pathoc(ssl=False) @@ -731,13 +725,12 @@ class TestProxySSL(tservers.HTTPProxyTest): assert not first_flow.server_conn.via -class MasterRedirectRequest(tservers.TestMaster): - redirect_port = None # Set by TestRedirectRequest +class ARedirectRequest: + def __init__(self, redirect_port): + self.redirect_port = redirect_port - @controller.handler def request(self, f): if f.request.path == "/p/201": - # This part should have no impact, but it should also not cause any exceptions. addr = f.live.server_conn.address addr2 = ("127.0.0.1", self.redirect_port) @@ -746,17 +739,13 @@ class MasterRedirectRequest(tservers.TestMaster): # This is the actual redirection. f.request.port = self.redirect_port - super().request(f) - @controller.handler def response(self, f): f.response.content = bytes(f.client_conn.address[1]) f.response.headers["server-conn-id"] = str(f.server_conn.source_address[1]) - super().response(f) class TestRedirectRequest(tservers.HTTPProxyTest): - masterclass = MasterRedirectRequest ssl = True def test_redirect(self): @@ -769,7 +758,7 @@ class TestRedirectRequest(tservers.HTTPProxyTest): This test verifies that the original destination is restored for the third request. """ - self.master.redirect_port = self.server2.port + self.proxy.tmaster.addons.add(ARedirectRequest(self.server2.port)) p = self.pathoc() with p.connect(): @@ -778,13 +767,13 @@ class TestRedirectRequest(tservers.HTTPProxyTest): r1 = p.request("get:'/p/200'") assert r1.status_code == 200 assert self.server.last_log() - assert not self.server2.last_log() + assert not self.server2.expect_log(1, 0.5) self.server.clear_log() self.server2.clear_log() r2 = p.request("get:'/p/201'") assert r2.status_code == 201 - assert not self.server.last_log() + assert not self.server.expect_log(1, 0.5) assert self.server2.last_log() self.server.clear_log() @@ -792,25 +781,23 @@ class TestRedirectRequest(tservers.HTTPProxyTest): r3 = p.request("get:'/p/202'") assert r3.status_code == 202 assert self.server.last_log() - assert not self.server2.last_log() + assert not self.server2.expect_log(1, 0.5) assert r1.content == r2.content == r3.content -class MasterStreamRequest(tservers.TestMaster): +class AStreamRequest: """ Enables the stream flag on the flow for all requests """ - @controller.handler def responseheaders(self, f): f.response.stream = True class TestStreamRequest(tservers.HTTPProxyTest): - masterclass = MasterStreamRequest - def test_stream_simple(self): + self.proxy.tmaster.addons.add(AStreamRequest()) p = self.pathoc() with p.connect(): # a request with 100k of data but without content-length @@ -819,6 +806,7 @@ class TestStreamRequest(tservers.HTTPProxyTest): assert len(r1.content) > 100000 def test_stream_multiple(self): + self.proxy.tmaster.addons.add(AStreamRequest()) p = self.pathoc() with p.connect(): # simple request with streaming turned on @@ -830,6 +818,7 @@ class TestStreamRequest(tservers.HTTPProxyTest): assert r1.status_code == 201 def test_stream_chunked(self): + self.proxy.tmaster.addons.add(AStreamRequest()) connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) connection.connect(("127.0.0.1", self.proxy.port)) fconn = connection.makefile("rb") @@ -850,22 +839,20 @@ class TestStreamRequest(tservers.HTTPProxyTest): connection.close() -class MasterFakeResponse(tservers.TestMaster): - @controller.handler +class AFakeResponse: def request(self, f): f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) class TestFakeResponse(tservers.HTTPProxyTest): - masterclass = MasterFakeResponse def test_fake(self): + self.proxy.tmaster.addons.add(AFakeResponse()) f = self.pathod("200") assert "header-response" in f.headers class TestServerConnect(tservers.HTTPProxyTest): - masterclass = MasterFakeResponse ssl = True @classmethod @@ -876,39 +863,34 @@ class TestServerConnect(tservers.HTTPProxyTest): def test_unnecessary_serverconnect(self): """A replayed/fake response with no upstream_cert should not connect to an upstream server""" + self.proxy.tmaster.addons.add(AFakeResponse()) assert self.pathod("200").status_code == 200 - for msg in self.proxy.tmaster.tlog: - assert "serverconnect" not in msg + assert not self.proxy.tmaster.has_log("serverconnect") -class MasterKillRequest(tservers.TestMaster): +class AKillRequest: - @controller.handler def request(self, f): f.reply.kill() class TestKillRequest(tservers.HTTPProxyTest): - masterclass = MasterKillRequest - def test_kill(self): + self.proxy.tmaster.addons.add(AKillRequest()) with pytest.raises(exceptions.HttpReadDisconnect): self.pathod("200") # Nothing should have hit the server - assert not self.server.last_log() + assert not self.server.expect_log(1, 0.5) -class MasterKillResponse(tservers.TestMaster): - - @controller.handler +class AKillResponse: def response(self, f): f.reply.kill() class TestKillResponse(tservers.HTTPProxyTest): - masterclass = MasterKillResponse - def test_kill(self): + self.proxy.tmaster.addons.add(AKillResponse()) with pytest.raises(exceptions.HttpReadDisconnect): self.pathod("200") # The server should have seen a request @@ -922,9 +904,7 @@ class TestTransparentResolveError(tservers.TransparentProxyTest): assert self.pathod("304").status_code == 502 -class MasterIncomplete(tservers.TestMaster): - - @controller.handler +class AIncomplete: def request(self, f): resp = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) resp.content = None @@ -932,9 +912,8 @@ class MasterIncomplete(tservers.TestMaster): class TestIncompleteResponse(tservers.HTTPProxyTest): - masterclass = MasterIncomplete - def test_incomplete(self): + self.proxy.tmaster.addons.add(AIncomplete()) assert self.pathod("200").status_code == 502 diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index a9b6f0c4..0e397b8f 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -43,7 +43,7 @@ class TestConcurrent(tservers.MasterTest): ) ) sc.start(tctx.options) - assert "decorator not supported" in tctx.master.event_log[0][1] + assert tctx.master.has_log("decorator not supported") def test_concurrent_class(self): with taddons.context() as tctx: diff --git a/test/mitmproxy/test_addonmanager.py b/test/mitmproxy/test_addonmanager.py index 3e5f71c6..e7be25b8 100644 --- a/test/mitmproxy/test_addonmanager.py +++ b/test/mitmproxy/test_addonmanager.py @@ -11,6 +11,7 @@ class TAddon: def __init__(self, name): self.name = name self.tick = True + self.custom_called = False def __repr__(self): return "Addon(%s)" % self.name @@ -18,11 +19,17 @@ class TAddon: def done(self): pass + def event_custom(self): + self.custom_called = True + def test_simple(): o = options.Options() m = master.Master(o, proxy.DummyServer(o)) a = addonmanager.AddonManager(m) + with pytest.raises(exceptions.AddonError): + a.invoke_addon(TAddon("one"), "done") + a.add(TAddon("one")) assert a.get("one") assert not a.get("two") @@ -30,6 +37,11 @@ def test_simple(): assert not a.chain a.add(TAddon("one")) - a("done") + a.trigger("done") with pytest.raises(exceptions.AddonError): - a("tick") + a.trigger("tick") + + ta = TAddon("one") + a.add(ta) + a.trigger("custom") + assert ta.custom_called diff --git a/test/mitmproxy/test_controller.py b/test/mitmproxy/test_controller.py index 6acd3e54..ccc8bf35 100644 --- a/test/mitmproxy/test_controller.py +++ b/test/mitmproxy/test_controller.py @@ -7,6 +7,7 @@ from mitmproxy.exceptions import Kill, ControlException from mitmproxy import controller from mitmproxy import master from mitmproxy import proxy +from mitmproxy.test import taddons class TMsg: @@ -15,22 +16,18 @@ class TMsg: class TestMaster: def test_simple(self): - class DummyMaster(master.Master): - @controller.handler + class tAddon: def log(self, _): - m.should_exit.set() + ctx.master.should_exit.set() - def tick(self, timeout): - # Speed up test - super().tick(0) - - m = DummyMaster(None, proxy.DummyServer(None)) - assert not m.should_exit.is_set() - msg = TMsg() - msg.reply = controller.DummyReply() - m.event_queue.put(("log", msg)) - m.run() - assert m.should_exit.is_set() + with taddons.context() as ctx: + ctx.master.addons.add(tAddon()) + assert not ctx.master.should_exit.is_set() + msg = TMsg() + msg.reply = controller.DummyReply() + ctx.master.event_queue.put(("log", msg)) + ctx.master.run() + assert ctx.master.should_exit.is_set() def test_server_simple(self): m = master.Master(None, proxy.DummyServer(None)) @@ -63,7 +60,6 @@ class TestChannel: def reply(): m, obj = q.get() assert m == "test" - obj.reply.handle() obj.reply.send(42) obj.reply.take() obj.reply.commit() @@ -85,10 +81,7 @@ class TestChannel: class TestReply: def test_simple(self): reply = controller.Reply(42) - assert reply.state == "unhandled" - - reply.handle() - assert reply.state == "handled" + assert reply.state == "start" reply.send("foo") assert reply.value == "foo" @@ -104,7 +97,6 @@ class TestReply: def test_kill(self): reply = controller.Reply(43) - reply.handle() reply.kill() reply.take() reply.commit() @@ -112,7 +104,6 @@ class TestReply: def test_ack(self): reply = controller.Reply(44) - reply.handle() reply.ack() reply.take() reply.commit() @@ -120,7 +111,6 @@ class TestReply: def test_reply_none(self): reply = controller.Reply(45) - reply.handle() reply.send(None) reply.take() reply.commit() @@ -128,7 +118,6 @@ class TestReply: def test_commit_no_reply(self): reply = controller.Reply(46) - reply.handle() reply.take() with pytest.raises(ControlException): reply.commit() @@ -137,7 +126,6 @@ class TestReply: def test_double_send(self): reply = controller.Reply(47) - reply.handle() reply.send(1) with pytest.raises(ControlException): reply.send(2) @@ -145,12 +133,11 @@ class TestReply: reply.commit() def test_state_transitions(self): - states = {"unhandled", "handled", "taken", "committed"} + states = {"start", "taken", "committed"} accept = { - "handle": {"unhandled"}, - "take": {"handled"}, + "take": {"start"}, "commit": {"taken"}, - "ack": {"handled", "taken"}, + "ack": {"start", "taken"}, } for fn, ok in accept.items(): for state in states: @@ -169,7 +156,6 @@ class TestReply: reply = controller.Reply(47) with pytest.raises(ControlException): reply.__del__() - reply.handle() reply.ack() reply.take() reply.commit() @@ -179,24 +165,22 @@ class TestDummyReply: def test_simple(self): reply = controller.DummyReply() for _ in range(2): - reply.handle() reply.ack() reply.take() reply.commit() reply.mark_reset() reply.reset() - assert reply.state == "unhandled" + assert reply.state == "start" def test_reset(self): reply = controller.DummyReply() - reply.handle() reply.ack() reply.take() reply.commit() reply.mark_reset() assert reply.state == "committed" reply.reset() - assert reply.state == "unhandled" + assert reply.state == "start" def test_del(self): reply = controller.DummyReply() diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index f20e0c8c..030f2c4e 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -41,7 +41,7 @@ class TestScripts(tservers.MasterTest): def test_add_header(self): m, _ = tscript("simple/add_header.py") f = tflow.tflow(resp=tutils.tresp()) - m.response(f) + m.addons.handle_lifecycle("response", f) assert f.response.headers["newheader"] == "foo" def test_custom_contentviews(self): @@ -56,7 +56,7 @@ class TestScripts(tservers.MasterTest): m, sc = tscript("simple/modify_body_inject_iframe.py", "http://example.org/evil_iframe") f = tflow.tflow(resp=tutils.tresp(content=b"<html><body>mitmproxy</body></html>")) - m.response(f) + m.addons.handle_lifecycle("response", f) content = f.response.content assert b'iframe' in content and b'evil_iframe' in content @@ -65,41 +65,41 @@ class TestScripts(tservers.MasterTest): form_header = Headers(content_type="application/x-www-form-urlencoded") f = tflow.tflow(req=tutils.treq(headers=form_header)) - m.request(f) + m.addons.handle_lifecycle("request", f) assert f.request.urlencoded_form["mitmproxy"] == "rocks" f.request.headers["content-type"] = "" - m.request(f) + m.addons.handle_lifecycle("request", f) assert list(f.request.urlencoded_form.items()) == [("foo", "bar")] def test_modify_querystring(self): m, sc = tscript("simple/modify_querystring.py") f = tflow.tflow(req=tutils.treq(path="/search?q=term")) - m.request(f) + m.addons.handle_lifecycle("request", f) assert f.request.query["mitmproxy"] == "rocks" f.request.path = "/" - m.request(f) + m.addons.handle_lifecycle("request", f) assert f.request.query["mitmproxy"] == "rocks" def test_arguments(self): m, sc = tscript("simple/script_arguments.py", "mitmproxy rocks") f = tflow.tflow(resp=tutils.tresp(content=b"I <3 mitmproxy")) - m.response(f) + m.addons.handle_lifecycle("response", f) assert f.response.content == b"I <3 rocks" def test_redirect_requests(self): m, sc = tscript("simple/redirect_requests.py") f = tflow.tflow(req=tutils.treq(host="example.org")) - m.request(f) + m.addons.handle_lifecycle("request", f) assert f.request.host == "mitmproxy.org" def test_send_reply_from_proxy(self): m, sc = tscript("simple/send_reply_from_proxy.py") f = tflow.tflow(req=tutils.treq(host="example.com", port=80)) - m.request(f) + m.addons.handle_lifecycle("request", f) assert f.response.content == b"Hello World" def test_dns_spoofing(self): @@ -109,13 +109,13 @@ class TestScripts(tservers.MasterTest): host_header = Headers(host=original_host) f = tflow.tflow(req=tutils.treq(headers=host_header, port=80)) - m.requestheaders(f) + m.addons.handle_lifecycle("requestheaders", f) # Rewrite by reverse proxy mode f.request.scheme = "https" f.request.port = 443 - m.request(f) + m.addons.handle_lifecycle("request", f) assert f.request.scheme == "http" assert f.request.port == 80 @@ -145,7 +145,7 @@ class TestHARDump: path = str(tmpdir.join("somefile")) m, sc = tscript("complex/har_dump.py", shlex.quote(path)) - m.addons.invoke(m, "response", self.flow()) + m.addons.trigger("response", self.flow()) m.addons.remove(sc) with open(path, "r") as inp: @@ -156,7 +156,9 @@ class TestHARDump: path = str(tmpdir.join("somefile")) m, sc = tscript("complex/har_dump.py", shlex.quote(path)) - m.addons.invoke(m, "response", self.flow(resp_content=b"foo" + b"\xFF" * 10)) + m.addons.trigger( + "response", self.flow(resp_content=b"foo" + b"\xFF" * 10) + ) m.addons.remove(sc) with open(path, "r") as inp: @@ -194,7 +196,7 @@ class TestHARDump: path = str(tmpdir.join("somefile")) m, sc = tscript("complex/har_dump.py", shlex.quote(path)) - m.addons.invoke(m, "response", f) + m.addons.trigger("response", f) m.addons.remove(sc) with open(path, "r") as inp: diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py index f4d32cbb..630fc7e4 100644 --- a/test/mitmproxy/test_flow.py +++ b/test/mitmproxy/test_flow.py @@ -3,12 +3,13 @@ import pytest from mitmproxy.test import tflow import mitmproxy.io -from mitmproxy import flowfilter, options +from mitmproxy import flowfilter +from mitmproxy import options +from mitmproxy.proxy import config from mitmproxy.contrib import tnetstring from mitmproxy.exceptions import FlowReadException from mitmproxy import flow from mitmproxy import http -from mitmproxy.proxy import ProxyConfig from mitmproxy.proxy.server import DummyServer from mitmproxy import master from . import tservers @@ -16,23 +17,6 @@ from . import tservers class TestSerialize: - def _treader(self): - sio = io.BytesIO() - w = mitmproxy.io.FlowWriter(sio) - for i in range(3): - f = tflow.tflow(resp=True) - w.add(f) - for i in range(3): - f = tflow.tflow(err=True) - w.add(f) - f = tflow.ttcpflow() - w.add(f) - f = tflow.ttcpflow(err=True) - w.add(f) - - sio.seek(0) - return mitmproxy.io.FlowReader(sio) - def test_roundtrip(self): sio = io.BytesIO() f = tflow.tflow() @@ -51,26 +35,6 @@ class TestSerialize: assert f2.request == f.request assert f2.marked - def test_load_flows(self): - r = self._treader() - s = tservers.TestState() - fm = master.Master(None, DummyServer()) - fm.addons.add(s) - fm.load_flows(r) - assert len(s.flows) == 6 - - def test_load_flows_reverse(self): - r = self._treader() - s = tservers.TestState() - opts = options.Options( - mode="reverse:https://use-this-domain" - ) - conf = ProxyConfig(opts) - fm = master.Master(opts, DummyServer(conf)) - fm.addons.add(s) - fm.load_flows(r) - assert s.flows[0].request.host == "use-this-domain" - def test_filter(self): sio = io.BytesIO() flt = flowfilter.parse("~c 200") @@ -122,6 +86,17 @@ class TestSerialize: class TestFlowMaster: + def test_load_flow_reverse(self): + s = tservers.TestState() + opts = options.Options( + mode="reverse:https://use-this-domain" + ) + conf = config.ProxyConfig(opts) + fm = master.Master(opts, DummyServer(conf)) + fm.addons.add(s) + f = tflow.tflow(resp=True) + fm.load_flow(f) + assert s.flows[0].request.host == "use-this-domain" def test_replay(self): fm = master.Master(None, DummyServer()) @@ -140,26 +115,26 @@ class TestFlowMaster: def test_create_flow(self): fm = master.Master(None, DummyServer()) - assert fm.create_request("GET", "http", "example.com", 80, "/") + assert fm.create_request("GET", "http://example.com/") def test_all(self): s = tservers.TestState() fm = master.Master(None, DummyServer()) fm.addons.add(s) f = tflow.tflow(req=None) - fm.clientconnect(f.client_conn) + fm.addons.handle_lifecycle("clientconnect", f.client_conn) f.request = http.HTTPRequest.wrap(mitmproxy.test.tutils.treq()) - fm.request(f) + fm.addons.handle_lifecycle("request", f) assert len(s.flows) == 1 f.response = http.HTTPResponse.wrap(mitmproxy.test.tutils.tresp()) - fm.response(f) + fm.addons.handle_lifecycle("response", f) assert len(s.flows) == 1 - fm.clientdisconnect(f.client_conn) + fm.addons.handle_lifecycle("clientdisconnect", f.client_conn) f.error = flow.Error("msg") - fm.error(f) + fm.addons.handle_lifecycle("error", f) fm.shutdown() diff --git a/test/mitmproxy/test_http.py b/test/mitmproxy/test_http.py index 889eb0a7..aa283530 100644 --- a/test/mitmproxy/test_http.py +++ b/test/mitmproxy/test_http.py @@ -175,7 +175,6 @@ class TestHTTPFlow: def test_kill(self): f = tflow.tflow() - f.reply.handle() f.intercept() assert f.killable f.kill() @@ -184,7 +183,6 @@ class TestHTTPFlow: def test_resume(self): f = tflow.tflow() - f.reply.handle() f.intercept() assert f.reply.state == "taken" f.resume() diff --git a/test/mitmproxy/test_taddons.py b/test/mitmproxy/test_taddons.py new file mode 100644 index 00000000..1e42141c --- /dev/null +++ b/test/mitmproxy/test_taddons.py @@ -0,0 +1,12 @@ + +from mitmproxy.test import taddons +from mitmproxy import ctx + + +def test_recordingmaster(): + with taddons.context() as tctx: + assert not tctx.master.has_log("nonexistent") + assert not tctx.master.has_event("nonexistent") + ctx.log.error("foo") + assert not tctx.master.has_log("foo", level="debug") + assert tctx.master.has_log("foo", level="error") diff --git a/test/mitmproxy/tools/console/test_master.py b/test/mitmproxy/tools/console/test_master.py index 6c716ad1..44b9ff3f 100644 --- a/test/mitmproxy/tools/console/test_master.py +++ b/test/mitmproxy/tools/console/test_master.py @@ -5,6 +5,7 @@ from mitmproxy import proxy from mitmproxy import options from mitmproxy.tools.console import common from ... import tservers +import urwid def test_format_keyvals(): @@ -35,7 +36,10 @@ class TestMaster(tservers.MasterTest): def test_basic(self): m = self.mkmaster() for i in (1, 2, 3): - self.dummy_cycle(m, 1, b"") + try: + self.dummy_cycle(m, 1, b"") + except urwid.ExitMainLoop: + pass assert len(m.view) == i def test_run_script_once(self): @@ -48,11 +52,11 @@ class TestMaster(tservers.MasterTest): """regression test for https://github.com/mitmproxy/mitmproxy/issues/1605""" m = self.mkmaster(intercept="~b bar") f = tflow.tflow(req=tutils.treq(content=b"foo")) - m.request(f) + m.addons.handle_lifecycle("request", f) assert not m.view[0].intercepted f = tflow.tflow(req=tutils.treq(content=b"bar")) - m.request(f) + m.addons.handle_lifecycle("request", f) assert m.view[1].intercepted f = tflow.tflow(resp=tutils.tresp(content=b"bar")) - m.request(f) + m.addons.handle_lifecycle("request", f) assert m.view[2].intercepted diff --git a/test/mitmproxy/tools/test_dump.py b/test/mitmproxy/tools/test_dump.py index a15bf583..8e2fa5b2 100644 --- a/test/mitmproxy/tools/test_dump.py +++ b/test/mitmproxy/tools/test_dump.py @@ -2,7 +2,6 @@ import pytest from unittest import mock from mitmproxy import proxy -from mitmproxy import exceptions from mitmproxy import log from mitmproxy import controller from mitmproxy import options @@ -17,24 +16,12 @@ class TestDumpMaster(tservers.MasterTest): m = dump.DumpMaster(o, proxy.DummyServer(), with_termlog=False, with_dumper=False) return m - def test_read(self, tmpdir): - p = str(tmpdir.join("read")) - self.flowfile(p) - self.dummy_cycle( - self.mkmaster(None, rfile=p), - 1, b"", - ) - with pytest.raises(exceptions.OptionsError): - self.mkmaster(None, rfile="/nonexistent") - with pytest.raises(exceptions.OptionsError): - self.mkmaster(None, rfile="test_dump.py") - def test_has_error(self): m = self.mkmaster(None) ent = log.LogEntry("foo", "error") ent.reply = controller.DummyReply() - m.log(ent) - assert m.has_errored + m.addons.trigger("log", ent) + assert m.errorcheck.has_errored @pytest.mark.parametrize("termlog", [False, True]) def test_addons_termlog(self, termlog): diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py index 00dc2c7c..e3d5dc44 100644 --- a/test/mitmproxy/tools/web/test_app.py +++ b/test/mitmproxy/tools/web/test_app.py @@ -83,7 +83,6 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): def test_resume(self): for f in self.view: - f.reply.handle() f.intercept() assert self.fetch( @@ -95,7 +94,6 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): def test_kill(self): for f in self.view: f.backup() - f.reply.handle() f.intercept() assert self.fetch("/flows/42/kill", method="POST").code == 200 @@ -109,7 +107,6 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): f = self.view.get_by_id("42") assert f - f.reply.handle() assert self.fetch("/flows/42", method="DELETE").code == 200 assert not self.view.get_by_id("42") diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index c47411ee..b737b82a 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -6,32 +6,27 @@ import sys import mitmproxy.platform from mitmproxy.proxy.config import ProxyConfig from mitmproxy.proxy.server import ProxyServer -from mitmproxy import master from mitmproxy import controller from mitmproxy import options from mitmproxy import exceptions from mitmproxy import io -from mitmproxy import http import pathod.test import pathod.pathoc +from mitmproxy import eventsequence from mitmproxy.test import tflow from mitmproxy.test import tutils +from mitmproxy.test import taddons class MasterTest: def cycle(self, master, content): f = tflow.tflow(req=tutils.treq(content=content)) - master.clientconnect(f.client_conn) - master.serverconnect(f.server_conn) - master.request(f) - if not f.error: - f.response = http.HTTPResponse.wrap( - tutils.tresp(content=content) - ) - master.response(f) - master.clientdisconnect(f) + master.addons.handle_lifecycle("clientconnect", f.client_conn) + for i in eventsequence.iterate(f): + master.addons.handle_lifecycle(*i) + master.addons.handle_lifecycle("clientdisconnect", f.client_conn) return f def dummy_cycle(self, master, n, content): @@ -68,11 +63,11 @@ class TestState: # self.flows.append(f) -class TestMaster(master.Master): +class TestMaster(taddons.RecordingMaster): def __init__(self, opts, config): s = ProxyServer(config) - master.Master.__init__(self, opts, s) + super().__init__(opts, s) def clear_addons(self, addons): self.addons.clear() @@ -80,18 +75,11 @@ class TestMaster(master.Master): self.addons.add(self.state) self.addons.add(*addons) self.addons.configure_all(self.options, self.options.keys()) - self.addons.invoke_all_with_context("running") - - def clear_log(self): - self.tlog = [] + self.addons.trigger("running") def reset(self, addons): self.clear_addons(addons) - self.clear_log() - - @controller.handler - def log(self, e): - self.tlog.append(e.msg) + self.clear() class ProxyThread(threading.Thread): @@ -111,7 +99,7 @@ class ProxyThread(threading.Thread): @property def tlog(self): - return self.tmaster.tlog + return self.tmaster.logs def run(self): self.tmaster.run() diff --git a/web/src/js/__tests__/ducks/ui/flowSpec.js b/web/src/js/__tests__/ducks/ui/flowSpec.js index 91b56746..e994624d 100644 --- a/web/src/js/__tests__/ducks/ui/flowSpec.js +++ b/web/src/js/__tests__/ducks/ui/flowSpec.js @@ -9,12 +9,28 @@ import reducer, { setShowFullContent, setContent, updateEdit, - stopEdit + stopEdit, + setContentView, + selectTab, + displayLarge } from '../../../ducks/ui/flow' import { select, updateFlow } from '../../../ducks/flows' describe('flow reducer', () => { + it('should return initial state', () => { + expect(reducer(undefined, {})).toEqual({ + displayLarge: false, + viewDescription: '', + showFullContent: false, + modifiedFlow: false, + contentView: 'Auto', + tab: 'request', + content: [], + maxContentLines: 80, + }) + }) + it('should change to edit mode', () => { let testFlow = {flow : 'foo'} const newState = reducer(undefined, startEdit({ flow: 'foo' })) @@ -74,4 +90,21 @@ describe('flow reducer', () => { let updatedFlow = {id: 1} expect(reducer({modifiedFlow}, stopEdit(updatedFlow, modifiedFlow)).modifiedFlow).toBeFalsy() }) + + it('should set content view', () => { + let state = reducer(undefined, setContentView('Edit')) + expect(state.contentView).toEqual('Edit') + expect(state.showFullContent).toBeTruthy() + }) + + it('should select different tabs', () => { + let state = reducer(undefined, selectTab('response')) + expect(state.tab).toEqual('response') + expect(state.displayLarge).toBeFalsy() + expect(state.showFullContent).toBeFalsy() + }) + + it('should display large', () => { + expect(reducer(undefined, displayLarge()).displayLarge).toBeTruthy() + }) }) |