diff options
-rw-r--r-- | mitmproxy/addons/clientplayback.py | 47 | ||||
-rw-r--r-- | mitmproxy/addons/core.py | 63 | ||||
-rw-r--r-- | mitmproxy/addons/script.py | 33 | ||||
-rw-r--r-- | mitmproxy/addons/serverplayback.py | 46 | ||||
-rw-r--r-- | mitmproxy/addons/view.py | 175 | ||||
-rw-r--r-- | mitmproxy/command.py | 18 | ||||
-rw-r--r-- | mitmproxy/eventsequence.py | 1 | ||||
-rw-r--r-- | mitmproxy/tools/console/flowlist.py | 108 | ||||
-rw-r--r-- | mitmproxy/tools/console/master.py | 75 | ||||
-rw-r--r-- | mitmproxy/tools/web/app.py | 20 | ||||
-rw-r--r-- | test/mitmproxy/addons/test_clientplayback.py | 19 | ||||
-rw-r--r-- | test/mitmproxy/addons/test_core.py | 46 | ||||
-rw-r--r-- | test/mitmproxy/addons/test_script.py | 70 | ||||
-rw-r--r-- | test/mitmproxy/addons/test_serverplayback.py | 12 | ||||
-rw-r--r-- | test/mitmproxy/addons/test_view.py | 113 | ||||
-rw-r--r-- | test/mitmproxy/test_command.py | 12 | ||||
-rw-r--r-- | test/mitmproxy/tools/web/test_app.py | 10 |
17 files changed, 534 insertions, 334 deletions
diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index acb77bb2..0db6d336 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -2,6 +2,7 @@ from mitmproxy import exceptions from mitmproxy import ctx from mitmproxy import io from mitmproxy import flow +from mitmproxy import command import typing @@ -11,32 +12,54 @@ class ClientPlayback: self.flows = None self.current_thread = None self.has_replayed = False + self.configured = False def count(self) -> int: if self.flows: return len(self.flows) return 0 - def load(self, flows: typing.Sequence[flow.Flow]): + @command.command("replay.client.stop") + def stop_replay(self) -> None: + """ + Stop client replay. + """ + self.flows = [] + ctx.master.addons.trigger("update", []) + + @command.command("replay.client") + def start_replay(self, flows: typing.Sequence[flow.Flow]) -> None: + """ + Replay requests from flows. + """ + self.flows = flows + ctx.master.addons.trigger("update", []) + + @command.command("replay.client.file") + def load_file(self, path: str) -> None: + try: + flows = io.read_flows_from_paths([path]) + except exceptions.FlowReadException as e: + raise exceptions.CommandError(str(e)) self.flows = flows def configure(self, updated): - if "client_replay" in updated: - if ctx.options.client_replay: - ctx.log.info("Client Replay: {}".format(ctx.options.client_replay)) - try: - flows = io.read_flows_from_paths(ctx.options.client_replay) - except exceptions.FlowReadException as e: - raise exceptions.OptionsError(str(e)) - self.load(flows) - else: - self.flows = None + if not self.configured and ctx.options.client_replay: + self.configured = True + ctx.log.info("Client Replay: {}".format(ctx.options.client_replay)) + try: + flows = io.read_flows_from_paths(ctx.options.client_replay) + except exceptions.FlowReadException as e: + raise exceptions.OptionsError(str(e)) + self.start_replay(flows) def tick(self): if self.current_thread and not self.current_thread.is_alive(): self.current_thread = None if self.flows and not self.current_thread: - self.current_thread = ctx.master.replay_request(self.flows.pop(0)) + f = self.flows.pop(0) + self.current_thread = ctx.master.replay_request(f) + ctx.master.addons.trigger("update", [f]) self.has_replayed = True if self.has_replayed: if not self.flows and not self.current_thread: diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index 3f9cb15e..b482edbb 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -1,6 +1,9 @@ +import typing + from mitmproxy import ctx from mitmproxy import exceptions from mitmproxy import command +from mitmproxy import flow class Core: @@ -16,3 +19,63 @@ class Core: ctx.options.set(spec) except exceptions.OptionsError as e: raise exceptions.CommandError(e) from e + + @command.command("flow.resume") + def resume(self, flows: typing.Sequence[flow.Flow]) -> None: + """ + Resume flows if they are intercepted. + """ + intercepted = [i for i in flows if i.intercepted] + for f in intercepted: + f.resume() + ctx.master.addons.trigger("update", intercepted) + + # FIXME: this will become view.mark later + @command.command("flow.mark") + def mark(self, flows: typing.Sequence[flow.Flow], val: bool) -> None: + """ + Mark flows. + """ + updated = [] + for i in flows: + if i.marked != val: + i.marked = val + updated.append(i) + ctx.master.addons.trigger("update", updated) + + # FIXME: this will become view.mark.toggle later + @command.command("flow.mark.toggle") + def mark_toggle(self, flows: typing.Sequence[flow.Flow]) -> None: + """ + Toggle mark for flows. + """ + for i in flows: + i.marked = not i.marked + ctx.master.addons.trigger("update", flows) + + @command.command("flow.kill") + def kill(self, flows: typing.Sequence[flow.Flow]) -> None: + """ + Kill running flows. + """ + updated = [] + for f in flows: + if f.killable: + f.kill() + updated.append(f) + ctx.log.alert("Killed %s flows." % len(updated)) + ctx.master.addons.trigger("update", updated) + + # FIXME: this will become view.revert later + @command.command("flow.revert") + def revert(self, flows: typing.Sequence[flow.Flow]) -> None: + """ + Revert flow changes. + """ + updated = [] + for f in flows: + if f.modified(): + f.revert() + updated.append(f) + ctx.log.alert("Reverted %s flows." % len(updated)) + ctx.master.addons.trigger("update", updated) diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index 99a8f6a4..e90dd885 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -2,9 +2,13 @@ import os import importlib import time import sys +import typing from mitmproxy import addonmanager from mitmproxy import exceptions +from mitmproxy import flow +from mitmproxy import command +from mitmproxy import eventsequence from mitmproxy import ctx @@ -34,10 +38,13 @@ class Script: def __init__(self, path): self.name = "scriptmanager:" + path self.path = path + self.fullpath = os.path.expanduser(path) self.ns = None self.last_load = 0 self.last_mtime = 0 + if not os.path.isfile(self.fullpath): + raise exceptions.OptionsError("No such script: %s" % path) @property def addons(self): @@ -45,12 +52,12 @@ class Script: def tick(self): if time.time() - self.last_load > self.ReloadInterval: - mtime = os.stat(self.path).st_mtime + mtime = os.stat(self.fullpath).st_mtime if mtime > self.last_mtime: ctx.log.info("Loading script: %s" % self.path) if self.ns: ctx.master.addons.remove(self.ns) - self.ns = load_script(ctx, self.path) + self.ns = load_script(ctx, self.fullpath) if self.ns: # We're already running, so we have to explicitly register and # configure the addon @@ -76,9 +83,25 @@ class ScriptLoader: def running(self): self.is_running = True - def run_once(self, command, flows): - # Returning once we have proper commands - raise NotImplementedError + @command.command("script.run") + def script_run(self, flows: typing.Sequence[flow.Flow], path: str) -> None: + """ + Run a script on the specified flows. The script is loaded with + default options, and all lifecycle events for each flow are + simulated. + """ + try: + s = Script(path) + l = addonmanager.Loader(ctx.master) + ctx.master.addons.invoke_addon(s, "load", l) + ctx.master.addons.invoke_addon(s, "configure", ctx.options.keys()) + # Script is loaded on the first tick + ctx.master.addons.invoke_addon(s, "tick") + for f in flows: + for evt, arg in eventsequence.iterate(f): + ctx.master.addons.invoke_addon(s, evt, arg) + except exceptions.OptionsError as e: + raise exceptions.CommandError("Error running script: %s" % e) from e def configure(self, updated): if "scripts" in updated: diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index 2255aaf2..927f6e15 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -1,11 +1,14 @@ import hashlib import urllib +import typing from typing import Any # noqa from typing import List # noqa from mitmproxy import ctx +from mitmproxy import flow from mitmproxy import exceptions from mitmproxy import io +from mitmproxy import command class ServerPlayback: @@ -13,15 +16,35 @@ class ServerPlayback: self.flowmap = {} self.stop = False self.final_flow = None + self.configured = False - def load_flows(self, flows): + @command.command("replay.server") + def load_flows(self, flows: typing.Sequence[flow.Flow]) -> None: + """ + Replay server responses from flows. + """ + self.flowmap = {} for i in flows: - if i.response: + if i.response: # type: ignore l = self.flowmap.setdefault(self._hash(i), []) l.append(i) - - def clear(self): + ctx.master.addons.trigger("update", []) + + @command.command("replay.server.file") + def load_file(self, path: str) -> None: + try: + flows = io.read_flows_from_paths([path]) + except exceptions.FlowReadException as e: + raise exceptions.CommandError(str(e)) + self.load_flows(flows) + + @command.command("replay.server.stop") + def clear(self) -> None: + """ + Stop server replay. + """ self.flowmap = {} + ctx.master.addons.trigger("update", []) def count(self): return sum([len(i) for i in self.flowmap.values()]) @@ -90,14 +113,13 @@ class ServerPlayback: return ret def configure(self, updated): - if "server_replay" in updated: - self.clear() - if ctx.options.server_replay: - try: - flows = io.read_flows_from_paths(ctx.options.server_replay) - except exceptions.FlowReadException as e: - raise exceptions.OptionsError(str(e)) - self.load_flows(flows) + if not self.configured and ctx.options.server_replay: + self.configured = True + try: + flows = io.read_flows_from_paths(ctx.options.server_replay) + except exceptions.FlowReadException as e: + raise exceptions.OptionsError(str(e)) + self.load_flows(flows) def tick(self): if self.stop and not self.final_flow.live: diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index f4082abe..794e7617 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -18,6 +18,7 @@ import sortedcontainers import mitmproxy.flow from mitmproxy import flowfilter from mitmproxy import exceptions +from mitmproxy import command from mitmproxy import ctx from mitmproxy import http # noqa @@ -223,7 +224,7 @@ class View(collections.Sequence): self.filter = flt or matchall self._refilter() - def clear(self): + def clear(self) -> None: """ Clears both the store and view. """ @@ -243,55 +244,19 @@ class View(collections.Sequence): self._refilter() self.sig_store_refresh.send(self) - def add(self, f: mitmproxy.flow.Flow) -> None: + def add(self, flows: typing.Sequence[mitmproxy.flow.Flow]) -> None: """ Adds a flow to the state. If the flow already exists, it is ignored. """ - if f.id not in self._store: - self._store[f.id] = f - if self.filter(f): - self._base_add(f) - if self.focus_follow: - self.focus.flow = f - self.sig_view_add.send(self, flow=f) - - def remove(self, f: mitmproxy.flow.Flow): - """ - Removes the flow from the underlying store and the view. - """ - if f.id in self._store: - if f in self._view: - self._view.remove(f) - self.sig_view_remove.send(self, flow=f) - del self._store[f.id] - self.sig_store_remove.send(self, flow=f) - - def update(self, f: mitmproxy.flow.Flow): - """ - Updates a flow. If the flow is not in the state, it's ignored. - """ - if f.id in self._store: - if self.filter(f): - if f not in self._view: + for f in flows: + if f.id not in self._store: + self._store[f.id] = f + if self.filter(f): self._base_add(f) if self.focus_follow: self.focus.flow = f self.sig_view_add.send(self, flow=f) - else: - # This is a tad complicated. The sortedcontainers - # implementation assumes that the order key is stable. If - # it changes mid-way Very Bad Things happen. We detect when - # this happens, and re-fresh the item. - self.order_key.refresh(f) - self.sig_view_update.send(self, flow=f) - else: - try: - self._view.remove(f) - self.sig_view_remove.send(self, flow=f) - except ValueError: - # The value was not in the view - pass def get_by_id(self, flow_id: str) -> typing.Optional[mitmproxy.flow.Flow]: """ @@ -300,32 +265,54 @@ class View(collections.Sequence): """ return self._store.get(flow_id) - # Event handlers - def configure(self, updated): - if "view_filter" in updated: - filt = None - if ctx.options.view_filter: - filt = flowfilter.parse(ctx.options.view_filter) - if not filt: - raise exceptions.OptionsError( - "Invalid interception filter: %s" % ctx.options.view_filter - ) - self.set_filter(filt) - if "console_order" in updated: - if ctx.options.console_order not in self.orders: - raise exceptions.OptionsError( - "Unknown flow order: %s" % ctx.options.console_order - ) - self.set_order(self.orders[ctx.options.console_order]) - if "console_order_reversed" in updated: - self.set_reversed(ctx.options.console_order_reversed) - if "console_focus_follow" in updated: - self.focus_follow = ctx.options.console_focus_follow + @command.command("view.go") + def go(self, dst: int) -> None: + """ + Go to a specified offset. Positive offests are from the beginning of + the view, negative from the end of the view, so that 0 is the first + flow, -1 is the last flow. + """ + if dst < 0: + dst = len(self) + dst + if dst < 0: + dst = 0 + if dst > len(self) - 1: + dst = len(self) - 1 + self.focus.flow = self[dst] + + @command.command("view.duplicate") + def duplicate(self, flows: typing.Sequence[mitmproxy.flow.Flow]) -> None: + """ + Duplicates the specified flows, and sets the focus to the first + duplicate. + """ + dups = [f.copy() for f in flows] + if dups: + self.add(dups) + self.focus.flow = dups[0] + + @command.command("view.remove") + def remove(self, flows: typing.Sequence[mitmproxy.flow.Flow]) -> None: + """ + Removes the flow from the underlying store and the view. + """ + for f in flows: + if f.id in self._store: + if f.killable: + f.kill() + if f in self._view: + self._view.remove(f) + self.sig_view_remove.send(self, flow=f) + del self._store[f.id] + self.sig_store_remove.send(self, flow=f) + @command.command("view.resolve") def resolve(self, spec: str) -> typing.Sequence[mitmproxy.flow.Flow]: """ Resolve a flow list specification to an actual list of flows. """ + if spec == "@all": + return [i for i in self._store.values()] if spec == "@focus": return [self.focus.flow] if self.focus.flow else [] elif spec == "@shown": @@ -342,26 +329,72 @@ class View(collections.Sequence): raise exceptions.CommandError("Invalid flow filter: %s" % spec) return [i for i in self._store.values() if filt(i)] - def load(self, l): - l.add_command("console.resolve", self.resolve) + # Event handlers + def configure(self, updated): + if "view_filter" in updated: + filt = None + if ctx.options.view_filter: + filt = flowfilter.parse(ctx.options.view_filter) + if not filt: + raise exceptions.OptionsError( + "Invalid interception filter: %s" % ctx.options.view_filter + ) + self.set_filter(filt) + if "console_order" in updated: + if ctx.options.console_order not in self.orders: + raise exceptions.OptionsError( + "Unknown flow order: %s" % ctx.options.console_order + ) + self.set_order(self.orders[ctx.options.console_order]) + if "console_order_reversed" in updated: + self.set_reversed(ctx.options.console_order_reversed) + if "console_focus_follow" in updated: + self.focus_follow = ctx.options.console_focus_follow def request(self, f): - self.add(f) + self.add([f]) def error(self, f): - self.update(f) + self.update([f]) def response(self, f): - self.update(f) + self.update([f]) def intercept(self, f): - self.update(f) + self.update([f]) def resume(self, f): - self.update(f) + self.update([f]) def kill(self, f): - self.update(f) + self.update([f]) + + def update(self, flows: typing.Sequence[mitmproxy.flow.Flow]) -> None: + """ + Updates a list of flows. If flow is not in the state, it's ignored. + """ + for f in flows: + if f.id in self._store: + if self.filter(f): + if f not in self._view: + self._base_add(f) + if self.focus_follow: + self.focus.flow = f + self.sig_view_add.send(self, flow=f) + else: + # This is a tad complicated. The sortedcontainers + # implementation assumes that the order key is stable. If + # it changes mid-way Very Bad Things happen. We detect when + # this happens, and re-fresh the item. + self.order_key.refresh(f) + self.sig_view_update.send(self, flow=f) + else: + try: + self._view.remove(f) + self.sig_view_remove.send(self, flow=f) + except ValueError: + # The value was not in the view + pass class Focus: diff --git a/mitmproxy/command.py b/mitmproxy/command.py index fa6e23ea..3b58cc25 100644 --- a/mitmproxy/command.py +++ b/mitmproxy/command.py @@ -109,10 +109,24 @@ def parsearg(manager: CommandManager, spec: str, argtype: type) -> typing.Any: """ if argtype == str: return spec + elif argtype == bool: + if spec == "true": + return True + elif spec == "false": + return False + else: + raise exceptions.CommandError( + "Booleans are 'true' or 'false', got %s" % spec + ) + elif argtype == int: + try: + return int(spec) + except ValueError as e: + raise exceptions.CommandError("Expected an integer, got %s." % spec) elif argtype == typing.Sequence[flow.Flow]: - return manager.call_args("console.resolve", [spec]) + return manager.call_args("view.resolve", [spec]) elif argtype == flow.Flow: - flows = manager.call_args("console.resolve", [spec]) + flows = manager.call_args("view.resolve", [spec]) if len(flows) != 1: raise exceptions.CommandError( "Command requires one flow, specification matched %s." % len(flows) diff --git a/mitmproxy/eventsequence.py b/mitmproxy/eventsequence.py index 30c037f1..4e199972 100644 --- a/mitmproxy/eventsequence.py +++ b/mitmproxy/eventsequence.py @@ -35,6 +35,7 @@ Events = frozenset([ "load", "running", "tick", + "update", ]) diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index bb59a9b7..8f8f5493 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -1,6 +1,5 @@ import urwid -from mitmproxy import exceptions from mitmproxy.tools.console import common from mitmproxy.tools.console import signals from mitmproxy.addons import view @@ -133,14 +132,6 @@ class FlowItem(urwid.WidgetWrap): def selectable(self): return True - def server_replay_prompt(self, k): - a = self.master.addons.get("serverplayback") - if k == "a": - a.load([i.copy() for i in self.master.view]) - elif k == "t": - a.load([self.flow.copy()]) - signals.update_settings.send(self) - def mouse_event(self, size, event, button, col, row, focus): if event == "mouse press" and button == 1: if self.flow.request: @@ -150,71 +141,7 @@ class FlowItem(urwid.WidgetWrap): def keypress(self, xxx_todo_changeme, key): (maxcol,) = xxx_todo_changeme key = common.shortcuts(key) - if key == "a": - self.flow.resume() - self.master.view.update(self.flow) - elif key == "d": - if self.flow.killable: - self.flow.kill() - self.master.view.remove(self.flow) - elif key == "D": - cp = self.flow.copy() - self.master.view.add(cp) - self.master.view.focus.flow = cp - elif key == "m": - self.flow.marked = not self.flow.marked - signals.flowlist_change.send(self) - elif key == "r": - try: - self.master.replay_request(self.flow) - except exceptions.ReplayException as e: - signals.add_log("Replay error: %s" % e, "warn") - signals.flowlist_change.send(self) - elif key == "S": - def stop_server_playback(response): - if response == "y": - self.master.options.server_replay = [] - a = self.master.addons.get("serverplayback") - if a.count(): - signals.status_prompt_onekey.send( - prompt = "Stop current server replay?", - keys = ( - ("yes", "y"), - ("no", "n"), - ), - callback = stop_server_playback, - ) - else: - signals.status_prompt_onekey.send( - prompt = "Server Replay", - keys = ( - ("all flows", "a"), - ("this flow", "t"), - ), - callback = self.server_replay_prompt, - ) - elif key == "U": - for f in self.master.view: - f.marked = False - signals.flowlist_change.send(self) - elif key == "V": - if not self.flow.modified(): - signals.status_message.send(message="Flow not modified.") - return - self.flow.revert() - signals.flowlist_change.send(self) - signals.status_message.send(message="Reverted.") - elif key == "X": - if self.flow.killable: - self.flow.kill() - self.master.view.update(self.flow) - elif key == "|": - signals.status_prompt_path.send( - prompt = "Send flow to script", - callback = self.master.run_script_once, - args = (self.flow,) - ) - elif key == "E": + if key == "E": signals.status_prompt_onekey.send( self, prompt = "Export to file", @@ -222,14 +149,14 @@ class FlowItem(urwid.WidgetWrap): callback = common.export_to_clip_or_file, args = (None, self.flow, common.ask_save_path) ) - elif key == "C": - signals.status_prompt_onekey.send( - self, - prompt = "Export to clipboard", - keys = [(e[0], e[1]) for e in export.EXPORTERS], - callback = common.export_to_clip_or_file, - args = (None, self.flow, common.copy_to_clipboard_or_prompt) - ) + # elif key == "C": + # signals.status_prompt_onekey.send( + # self, + # prompt = "Export to clipboard", + # keys = [(e[0], e[1]) for e in export.EXPORTERS], + # callback = common.export_to_clip_or_file, + # args = (None, self.flow, common.copy_to_clipboard_or_prompt) + # ) elif key == "b": common.ask_save_body(None, self.flow) else: @@ -321,22 +248,7 @@ class FlowListBox(urwid.ListBox): def keypress(self, size, key): key = common.shortcuts(key) - if key == "A": - for f in self.master.view: - if f.intercepted: - f.resume() - self.master.view.update(f) - elif key == "z": - self.master.view.clear() - elif key == "Z": - self.master.view.clear_not_marked() - elif key == "g": - if len(self.master.view): - self.master.view.focus.index = 0 - elif key == "G": - if len(self.master.view): - self.master.view.focus.index = len(self.master.view) - 1 - elif key == "L": + if key == "L": signals.status_prompt_path.send( self, prompt = "Load flows", diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 7787ba11..29a61bb3 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -13,10 +13,8 @@ import traceback import urwid from mitmproxy import addons -from mitmproxy import exceptions from mitmproxy import command from mitmproxy import master -from mitmproxy import io from mitmproxy import log from mitmproxy import flow from mitmproxy.addons import intercept @@ -78,7 +76,7 @@ class UnsupportedLog: signals.add_log(strutils.bytes_to_escaped_str(message.content), "debug") -class ConsoleCommands: +class ConsoleAddon: """ An addon that exposes console-specific commands. """ @@ -131,6 +129,10 @@ class ConsoleCommands: def running(self): self.started = True + def update(self, flows): + if not flows: + signals.update_settings.send(self) + def configure(self, updated): if self.started: if "console_eventlog" in updated: @@ -147,11 +149,26 @@ def default_keymap(km): km.add("i", "console.command 'set intercept='") km.add("W", "console.command 'set save_stream_file='") + km.add("A", "flow.resume @all", context="flowlist") + km.add("a", "flow.resume @focus", context="flowlist") + km.add("d", "view.remove @focus", context="flowlist") + km.add("D", "view.duplicate @focus", context="flowlist") + km.add("e", "set console_eventlog=toggle", context="flowlist") + km.add("f", "console.command 'set view_filter='", context="flowlist") km.add("F", "set console_focus_follow=toggle", context="flowlist") + km.add("g", "view.go 0", context="flowlist") + km.add("G", "view.go -1", context="flowlist") + km.add("m", "flow.mark.toggle @focus", context="flowlist") + km.add("r", "replay.client @focus", context="flowlist") + km.add("S", "console.command 'replay.server '") km.add("v", "set console_order_reversed=toggle", context="flowlist") - km.add("f", "console.command 'set view_filter='", context="flowlist") - km.add("e", "set console_eventlog=toggle", context="flowlist") + km.add("U", "flow.mark @all false", context="flowlist") km.add("w", "console.command 'save.file @shown '", context="flowlist") + km.add("V", "flow.revert @focus", context="flowlist") + km.add("X", "flow.kill @focus", context="flowlist") + km.add("z", "view.remove @all", context="flowlist") + km.add("Z", "view.remove @hidden", context="flowlist") + km.add("|", "console.command 'script.run @focus '", context="flowlist") km.add("enter", "console.view.flow @focus", context="flowlist") @@ -184,7 +201,7 @@ class ConsoleMaster(master.Master): self.view, UnsupportedLog(), readfile.ReadFile(), - ConsoleCommands(self), + ConsoleAddon(self), ) def sigint_handler(*args, **kwargs): @@ -262,31 +279,10 @@ class ConsoleMaster(master.Master): self.loop.widget = window self.loop.draw_screen() - def run_script_once(self, command, f): - sc = self.addons.get("scriptloader") - try: - with self.handlecontext(): - sc.run_once(command, [f]) - except ValueError as e: - signals.add_log("Input error: %s" % e, "warn") - def refresh_view(self): self.view_flowlist() signals.replace_view_state.send(self) - def _readflows(self, path): - """ - Utitility function that reads a list of flows - or prints an error to the UI if that fails. - Returns - - None, if there was an error. - - a list of flows, otherwise. - """ - try: - return io.read_flows_from_paths(path) - except exceptions.FlowReadException as e: - signals.status_message.send(message=str(e)) - def spawn_editor(self, data): text = not isinstance(data, bytes) fd, name = tempfile.mkstemp('', "mproxy", text=text) @@ -507,31 +503,6 @@ class ConsoleMaster(master.Master): ) ) - def _write_flows(self, path, flows): - with open(path, "wb") as f: - fw = io.FlowWriter(f) - for i in flows: - fw.add(i) - - def save_one_flow(self, path, flow): - return self._write_flows(path, [flow]) - - def save_flows(self, path): - return self._write_flows(path, self.view) - - def load_flows_callback(self, path): - ret = self.load_flows_path(path) - return ret or "Flows loaded from %s" % path - - def load_flows_path(self, path): - reterr = None - try: - master.Master.load_flows_file(self, path) - except exceptions.FlowReadException as e: - reterr = str(e) - signals.flowlist_change.send(self) - return reterr - def quit(self, a): if a != "n": self.shutdown() diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 1e01c57c..c55c0cb5 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -246,7 +246,7 @@ class ResumeFlows(RequestHandler): def post(self): for f in self.view: f.resume() - self.view.update(f) + self.view.update([f]) class KillFlows(RequestHandler): @@ -254,27 +254,27 @@ class KillFlows(RequestHandler): for f in self.view: if f.killable: f.kill() - self.view.update(f) + self.view.update([f]) class ResumeFlow(RequestHandler): def post(self, flow_id): self.flow.resume() - self.view.update(self.flow) + self.view.update([self.flow]) class KillFlow(RequestHandler): def post(self, flow_id): if self.flow.killable: self.flow.kill() - self.view.update(self.flow) + self.view.update([self.flow]) class FlowHandler(RequestHandler): def delete(self, flow_id): if self.flow.killable: self.flow.kill() - self.view.remove(self.flow) + self.view.remove([self.flow]) def put(self, flow_id): flow = self.flow @@ -317,13 +317,13 @@ class FlowHandler(RequestHandler): except APIError: flow.revert() raise - self.view.update(flow) + self.view.update([flow]) class DuplicateFlow(RequestHandler): def post(self, flow_id): f = self.flow.copy() - self.view.add(f) + self.view.add([f]) self.write(f.id) @@ -331,14 +331,14 @@ class RevertFlow(RequestHandler): def post(self, flow_id): if self.flow.modified(): self.flow.revert() - self.view.update(self.flow) + self.view.update([self.flow]) class ReplayFlow(RequestHandler): def post(self, flow_id): self.flow.backup() self.flow.response = None - self.view.update(self.flow) + self.view.update([self.flow]) try: self.master.replay_request(self.flow) @@ -351,7 +351,7 @@ class FlowContent(RequestHandler): self.flow.backup() message = getattr(self.flow, message) message.content = self.filecontents - self.view.update(self.flow) + self.view.update([self.flow]) def get(self, flow_id, message): message = getattr(self.flow, message) diff --git a/test/mitmproxy/addons/test_clientplayback.py b/test/mitmproxy/addons/test_clientplayback.py index f71662f0..7ffda317 100644 --- a/test/mitmproxy/addons/test_clientplayback.py +++ b/test/mitmproxy/addons/test_clientplayback.py @@ -26,7 +26,7 @@ class TestClientPlayback: with taddons.context() as tctx: assert cp.count() == 0 f = tflow.tflow(resp=True) - cp.load([f]) + cp.start_replay([f]) assert cp.count() == 1 RP = "mitmproxy.proxy.protocol.http_replay.RequestReplayThread" with mock.patch(RP) as rp: @@ -44,13 +44,30 @@ class TestClientPlayback: cp.tick() assert cp.current_thread is None + cp.start_replay([f]) + cp.stop_replay() + assert not cp.flows + + def test_load_file(self, tmpdir): + cp = clientplayback.ClientPlayback() + with taddons.context(): + fpath = str(tmpdir.join("flows")) + tdump(fpath, [tflow.tflow(resp=True)]) + cp.load_file(fpath) + assert cp.flows + with pytest.raises(exceptions.CommandError): + cp.load_file("/nonexistent") + def test_configure(self, tmpdir): cp = clientplayback.ClientPlayback() with taddons.context() as tctx: path = str(tmpdir.join("flows")) tdump(path, [tflow.tflow()]) tctx.configure(cp, client_replay=[path]) + cp.configured = False tctx.configure(cp, client_replay=[]) + cp.configured = False tctx.configure(cp) + cp.configured = False with pytest.raises(exceptions.OptionsError): tctx.configure(cp, client_replay=["nonexistent"]) diff --git a/test/mitmproxy/addons/test_core.py b/test/mitmproxy/addons/test_core.py index 6ebf4ba9..64d0fa19 100644 --- a/test/mitmproxy/addons/test_core.py +++ b/test/mitmproxy/addons/test_core.py @@ -1,5 +1,6 @@ from mitmproxy.addons import core from mitmproxy.test import taddons +from mitmproxy.test import tflow from mitmproxy import exceptions import pytest @@ -15,3 +16,48 @@ def test_set(): with pytest.raises(exceptions.CommandError): tctx.command(sa.set, "nonexistent") + + +def test_resume(): + sa = core.Core() + with taddons.context(): + f = tflow.tflow() + assert not sa.resume([f]) + f.intercept() + sa.resume([f]) + assert not f.reply.state == "taken" + + +def test_mark(): + sa = core.Core() + with taddons.context(): + f = tflow.tflow() + assert not f.marked + sa.mark([f], True) + assert f.marked + + sa.mark_toggle([f]) + assert not f.marked + sa.mark_toggle([f]) + assert f.marked + + +def test_kill(): + sa = core.Core() + with taddons.context(): + f = tflow.tflow() + f.intercept() + assert f.killable + sa.kill([f]) + assert not f.killable + + +def test_revert(): + sa = core.Core() + with taddons.context(): + f = tflow.tflow() + f.backup() + f.request.content = b"bar" + assert f.modified() + sa.revert([f]) + assert not f.modified() diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index 859d99f9..a3df1fcf 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -9,9 +9,6 @@ from mitmproxy.test import tutils from mitmproxy.test import taddons from mitmproxy import addonmanager from mitmproxy import exceptions -from mitmproxy import options -from mitmproxy import proxy -from mitmproxy import master from mitmproxy.addons import script @@ -48,9 +45,9 @@ def test_script_print_stdout(): class TestScript: def test_notfound(self): - with taddons.context() as tctx: - sc = script.Script("nonexistent") - tctx.master.addons.add(sc) + with taddons.context(): + with pytest.raises(exceptions.OptionsError): + script.Script("nonexistent") def test_simple(self): with taddons.context() as tctx: @@ -136,25 +133,45 @@ class TestCutTraceback: class TestScriptLoader: - def test_simple(self): - o = options.Options(scripts=[]) - m = master.Master(o, proxy.DummyServer()) + def test_script_run(self): + rp = tutils.test_data.path( + "mitmproxy/data/addonscripts/recorder/recorder.py" + ) sc = script.ScriptLoader() - sc.running() - m.addons.add(sc) - assert len(m.addons) == 1 - o.update( - scripts = [ - tutils.test_data.path( - "mitmproxy/data/addonscripts/recorder/recorder.py" - ) + with taddons.context() as tctx: + sc.script_run([tflow.tflow(resp=True)], rp) + debug = [i.msg for i in tctx.master.logs if i.level == "debug"] + assert debug == [ + 'recorder load', 'recorder running', 'recorder configure', + 'recorder tick', + 'recorder requestheaders', 'recorder request', + 'recorder responseheaders', 'recorder response' ] - ) - assert len(m.addons) == 1 - assert len(sc.addons) == 1 - o.update(scripts = []) - assert len(m.addons) == 1 - assert len(sc.addons) == 0 + + def test_script_run_nonexistent(self): + sc = script.ScriptLoader() + with taddons.context(): + with pytest.raises(exceptions.CommandError): + sc.script_run([tflow.tflow(resp=True)], "/nonexistent") + + def test_simple(self): + sc = script.ScriptLoader() + with taddons.context() as tctx: + tctx.master.addons.add(sc) + sc.running() + assert len(tctx.master.addons) == 1 + tctx.master.options.update( + scripts = [ + tutils.test_data.path( + "mitmproxy/data/addonscripts/recorder/recorder.py" + ) + ] + ) + assert len(tctx.master.addons) == 1 + assert len(sc.addons) == 1 + tctx.master.options.update(scripts = []) + assert len(tctx.master.addons) == 1 + assert len(sc.addons) == 0 def test_dupes(self): sc = script.ScriptLoader() @@ -166,13 +183,6 @@ class TestScriptLoader: scripts = ["one", "one"] ) - def test_nonexistent(self): - sc = script.ScriptLoader() - with taddons.context() as tctx: - tctx.master.addons.add(sc) - tctx.configure(sc, scripts = ["nonexistent"]) - tctx.master.has_log("nonexistent: file not found") - def test_order(self): rec = tutils.test_data.path("mitmproxy/data/addonscripts/recorder") sc = script.ScriptLoader() diff --git a/test/mitmproxy/addons/test_serverplayback.py b/test/mitmproxy/addons/test_serverplayback.py index 29de48a0..3ceab3fa 100644 --- a/test/mitmproxy/addons/test_serverplayback.py +++ b/test/mitmproxy/addons/test_serverplayback.py @@ -16,12 +16,24 @@ def tdump(path, flows): w.add(i) +def test_load_file(tmpdir): + s = serverplayback.ServerPlayback() + with taddons.context(): + fpath = str(tmpdir.join("flows")) + tdump(fpath, [tflow.tflow(resp=True)]) + s.load_file(fpath) + assert s.flowmap + with pytest.raises(exceptions.CommandError): + s.load_file("/nonexistent") + + def test_config(tmpdir): s = serverplayback.ServerPlayback() with taddons.context() as tctx: fpath = str(tmpdir.join("flows")) tdump(fpath, [tflow.tflow(resp=True)]) tctx.configure(s, server_replay=[fpath]) + s.configured = False with pytest.raises(exceptions.OptionsError): tctx.configure(s, server_replay=[str(tmpdir)]) diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index 05d4af30..979f0aa1 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -4,7 +4,6 @@ from mitmproxy.test import tflow from mitmproxy.addons import view from mitmproxy import flowfilter -from mitmproxy import options from mitmproxy import exceptions from mitmproxy.test import taddons @@ -26,12 +25,12 @@ def test_order_refresh(): v.sig_view_refresh.connect(save) tf = tflow.tflow(resp=True) - with taddons.context(options=options.Options()) as tctx: + with taddons.context() as tctx: tctx.configure(v, console_order="time") - v.add(tf) + v.add([tf]) tf.request.timestamp_start = 1 assert not sargs - v.update(tf) + v.update([tf]) assert sargs @@ -133,13 +132,14 @@ def test_filter(): def test_load(): v = view.View() - with taddons.context(options=options.Options()) as tctx: + with taddons.context() as tctx: tctx.master.addons.add(v) def test_resolve(): v = view.View() - with taddons.context(options=options.Options()) as tctx: + with taddons.context() as tctx: + assert tctx.command(v.resolve, "@all") == [] assert tctx.command(v.resolve, "@focus") == [] assert tctx.command(v.resolve, "@shown") == [] assert tctx.command(v.resolve, "@hidden") == [] @@ -149,6 +149,7 @@ def test_resolve(): v.request(tft(method="get")) assert len(tctx.command(v.resolve, "~m get")) == 1 assert len(tctx.command(v.resolve, "@focus")) == 1 + assert len(tctx.command(v.resolve, "@all")) == 1 assert len(tctx.command(v.resolve, "@shown")) == 1 assert len(tctx.command(v.resolve, "@unmarked")) == 1 assert tctx.command(v.resolve, "@hidden") == [] @@ -156,6 +157,7 @@ def test_resolve(): v.request(tft(method="put")) assert len(tctx.command(v.resolve, "@focus")) == 1 assert len(tctx.command(v.resolve, "@shown")) == 2 + assert len(tctx.command(v.resolve, "@all")) == 2 assert tctx.command(v.resolve, "@hidden") == [] assert tctx.command(v.resolve, "@marked") == [] @@ -175,14 +177,52 @@ def test_resolve(): assert m(tctx.command(v.resolve, "@hidden")) == ["PUT", "PUT"] assert m(tctx.command(v.resolve, "@marked")) == ["GET"] assert m(tctx.command(v.resolve, "@unmarked")) == ["PUT", "GET", "PUT"] + assert m(tctx.command(v.resolve, "@all")) == ["GET", "PUT", "GET", "PUT"] with pytest.raises(exceptions.CommandError, match="Invalid flow filter"): tctx.command(v.resolve, "~") +def test_go(): + v = view.View() + with taddons.context(): + v.add([ + tflow.tflow(), + tflow.tflow(), + tflow.tflow(), + tflow.tflow(), + tflow.tflow(), + ]) + assert v.focus.index == 0 + v.go(-1) + assert v.focus.index == 4 + v.go(0) + assert v.focus.index == 0 + v.go(1) + assert v.focus.index == 1 + v.go(999) + assert v.focus.index == 4 + v.go(-999) + assert v.focus.index == 0 + + +def test_duplicate(): + v = view.View() + with taddons.context(): + f = [ + tflow.tflow(), + tflow.tflow(), + ] + v.add(f) + assert len(v) == 2 + v.duplicate(f) + assert len(v) == 4 + assert v.focus.index == 2 + + def test_order(): v = view.View() - with taddons.context(options=options.Options()) as tctx: + with taddons.context() as tctx: v.request(tft(method="get", start=1)) v.request(tft(method="put", start=2)) v.request(tft(method="get", start=3)) @@ -230,14 +270,14 @@ def test_update(): assert f in v f.request.method = "put" - v.update(f) + v.update([f]) assert f not in v f.request.method = "get" - v.update(f) + v.update([f]) assert f in v - v.update(f) + v.update([f]) assert f in v @@ -276,7 +316,7 @@ def test_signals(): assert not any([rec_add, rec_update, rec_remove, rec_refresh]) # Simple add - v.add(tft()) + v.add([tft()]) assert rec_add assert not any([rec_update, rec_remove, rec_refresh]) @@ -291,14 +331,14 @@ def test_signals(): # An update that results in a flow being added to the view clearrec() v[0].request.method = "PUT" - v.update(v[0]) + v.update([v[0]]) assert rec_remove assert not any([rec_update, rec_refresh, rec_add]) # An update that does not affect the view just sends update v.set_filter(flowfilter.parse("~m put")) clearrec() - v.update(v[0]) + v.update([v[0]]) assert rec_update assert not any([rec_remove, rec_refresh, rec_add]) @@ -307,33 +347,33 @@ def test_signals(): v.set_filter(flowfilter.parse("~m get")) assert not len(v) clearrec() - v.update(f) + v.update([f]) assert not any([rec_add, rec_update, rec_remove, rec_refresh]) def test_focus_follow(): v = view.View() - with taddons.context(options=options.Options()) as tctx: + with taddons.context() as tctx: tctx.configure(v, console_focus_follow=True, view_filter="~m get") - v.add(tft(start=5)) + v.add([tft(start=5)]) assert v.focus.index == 0 - v.add(tft(start=4)) + v.add([tft(start=4)]) assert v.focus.index == 0 assert v.focus.flow.request.timestamp_start == 4 - v.add(tft(start=7)) + v.add([tft(start=7)]) assert v.focus.index == 2 assert v.focus.flow.request.timestamp_start == 7 mod = tft(method="put", start=6) - v.add(mod) + v.add([mod]) assert v.focus.index == 2 assert v.focus.flow.request.timestamp_start == 7 mod.request.method = "GET" - v.update(mod) + v.update([mod]) assert v.focus.index == 2 assert v.focus.flow.request.timestamp_start == 6 @@ -341,7 +381,7 @@ def test_focus_follow(): def test_focus(): # Special case - initialising with a view that already contains data v = view.View() - v.add(tft()) + v.add([tft()]) f = view.Focus(v) assert f.index is 0 assert f.flow is v[0] @@ -352,7 +392,7 @@ def test_focus(): assert f.index is None assert f.flow is None - v.add(tft(start=1)) + v.add([tft(start=1)]) assert f.index == 0 assert f.flow is v[0] @@ -362,11 +402,11 @@ def test_focus(): with pytest.raises(ValueError): f.__setattr__("index", 99) - v.add(tft(start=0)) + v.add([tft(start=0)]) assert f.index == 1 assert f.flow is v[1] - v.add(tft(start=2)) + v.add([tft(start=2)]) assert f.index == 1 assert f.flow is v[1] @@ -374,22 +414,25 @@ def test_focus(): assert f.index == 0 f.index = 1 - v.remove(v[1]) + v.remove([v[1]]) + v[1].intercept() assert f.index == 1 assert f.flow is v[1] - v.remove(v[1]) + v.remove([v[1]]) assert f.index == 0 assert f.flow is v[0] - v.remove(v[0]) + v.remove([v[0]]) assert f.index is None assert f.flow is None - v.add(tft(method="get", start=0)) - v.add(tft(method="get", start=1)) - v.add(tft(method="put", start=2)) - v.add(tft(method="get", start=3)) + v.add([ + tft(method="get", start=0), + tft(method="get", start=1), + tft(method="put", start=2), + tft(method="get", start=3), + ]) f.flow = v[2] assert f.flow.request.method == "PUT" @@ -409,16 +452,16 @@ def test_settings(): with pytest.raises(KeyError): v.settings[f] - v.add(f) + v.add([f]) v.settings[f]["foo"] = "bar" assert v.settings[f]["foo"] == "bar" assert len(list(v.settings)) == 1 - v.remove(f) + v.remove([f]) with pytest.raises(KeyError): v.settings[f] assert not v.settings.keys() - v.add(f) + v.add([f]) v.settings[f]["foo"] = "bar" assert v.settings.keys() v.clear() @@ -427,7 +470,7 @@ def test_settings(): def test_configure(): v = view.View() - with taddons.context(options=options.Options()) as tctx: + with taddons.context() as tctx: tctx.configure(v, view_filter="~q") with pytest.raises(Exception, match="Invalid interception filter"): tctx.configure(v, view_filter="~~") diff --git a/test/mitmproxy/test_command.py b/test/mitmproxy/test_command.py index 64928dbf..96d79dba 100644 --- a/test/mitmproxy/test_command.py +++ b/test/mitmproxy/test_command.py @@ -65,7 +65,7 @@ def test_typename(): class DummyConsole: def load(self, l): - l.add_command("console.resolve", self.resolve) + l.add_command("view.resolve", self.resolve) def resolve(self, spec: str) -> typing.Sequence[flow.Flow]: n = int(spec) @@ -76,6 +76,16 @@ def test_parsearg(): with taddons.context() as tctx: tctx.master.addons.add(DummyConsole()) assert command.parsearg(tctx.master.commands, "foo", str) == "foo" + + assert command.parsearg(tctx.master.commands, "1", int) == 1 + with pytest.raises(exceptions.CommandError): + command.parsearg(tctx.master.commands, "foo", int) + + assert command.parsearg(tctx.master.commands, "true", bool) is True + assert command.parsearg(tctx.master.commands, "false", bool) is False + with pytest.raises(exceptions.CommandError): + command.parsearg(tctx.master.commands, "flobble", bool) + assert len(command.parsearg( tctx.master.commands, "2", typing.Sequence[flow.Flow] )) == 2 diff --git a/test/mitmproxy/tools/web/test_app.py b/test/mitmproxy/tools/web/test_app.py index e3d5dc44..2b6181d3 100644 --- a/test/mitmproxy/tools/web/test_app.py +++ b/test/mitmproxy/tools/web/test_app.py @@ -23,8 +23,8 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): m = webmaster.WebMaster(o, proxy.DummyServer(), with_termlog=False) f = tflow.tflow(resp=True) f.id = "42" - m.view.add(f) - m.view.add(tflow.tflow(err=True)) + m.view.add([f]) + m.view.add([tflow.tflow(err=True)]) m.add_log("test log", "info") self.master = m self.view = m.view @@ -78,7 +78,7 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): # restore for f in flows: - self.view.add(f) + self.view.add([f]) self.events.data = events def test_resume(self): @@ -110,7 +110,7 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): assert self.fetch("/flows/42", method="DELETE").code == 200 assert not self.view.get_by_id("42") - self.view.add(f) + self.view.add([f]) assert self.fetch("/flows/1234", method="DELETE").code == 404 @@ -162,7 +162,7 @@ class TestApp(tornado.testing.AsyncHTTPTestCase): f = self.view.get_by_id(resp.body.decode()) assert f assert f.id != "42" - self.view.remove(f) + self.view.remove([f]) def test_flow_revert(self): f = self.view.get_by_id("42") |