diff options
| -rw-r--r-- | mitmproxy/addons/readfile.py | 55 | ||||
| -rw-r--r-- | mitmproxy/addons/script.py | 9 | ||||
| -rw-r--r-- | mitmproxy/tools/main.py | 3 | ||||
| -rw-r--r-- | setup.py | 5 | ||||
| -rw-r--r-- | test/mitmproxy/addons/test_readfile.py | 94 | ||||
| -rw-r--r-- | test/mitmproxy/addons/test_script.py | 23 | ||||
| -rw-r--r-- | test/mitmproxy/data/addonscripts/recorder/error.py | 7 |
7 files changed, 130 insertions, 66 deletions
diff --git a/mitmproxy/addons/readfile.py b/mitmproxy/addons/readfile.py index aaf02d01..2ae81fae 100644 --- a/mitmproxy/addons/readfile.py +++ b/mitmproxy/addons/readfile.py @@ -1,9 +1,11 @@ +import asyncio import os.path import sys import typing from mitmproxy import ctx from mitmproxy import exceptions +from mitmproxy import flowfilter from mitmproxy import io @@ -11,18 +13,38 @@ class ReadFile: """ An addon that handles reading from file on startup. """ + def __init__(self): + self.filter = None + def load(self, loader): loader.add_option( "rfile", typing.Optional[str], None, "Read flows from file." ) + loader.add_option( + "readfile_filter", typing.Optional[str], None, + "Read only matching flows." + ) - def load_flows(self, fo: typing.IO[bytes]) -> int: + def configure(self, updated): + if "readfile_filter" in updated: + filt = None + if ctx.options.readfile_filter: + filt = flowfilter.parse(ctx.options.readfile_filter) + if not filt: + raise exceptions.OptionsError( + "Invalid readfile filter: %s" % ctx.options.readfile_filter + ) + self.filter = filt + + async def load_flows(self, fo: typing.IO[bytes]) -> int: cnt = 0 freader = io.FlowReader(fo) try: for flow in freader.stream(): - ctx.master.load_flow(flow) + if self.filter and not self.filter(flow): + continue + await ctx.master.load_flow(flow) cnt += 1 except (IOError, exceptions.FlowReadException) as e: if cnt: @@ -33,29 +55,34 @@ class ReadFile: else: return cnt - def load_flows_from_path(self, path: str) -> int: + async def load_flows_from_path(self, path: str) -> int: path = os.path.expanduser(path) try: with open(path, "rb") as f: - return self.load_flows(f) + return await self.load_flows(f) except IOError as e: ctx.log.error("Cannot load flows: {}".format(e)) raise exceptions.FlowReadException(str(e)) from e + async def doread(self, rfile): + try: + await self.load_flows_from_path(ctx.options.rfile) + except exceptions.FlowReadException as e: + raise exceptions.OptionsError(e) from e + finally: + ctx.master.addons.trigger("processing_complete") + def running(self): if ctx.options.rfile: - try: - self.load_flows_from_path(ctx.options.rfile) - except exceptions.FlowReadException as e: - raise exceptions.OptionsError(e) from e - finally: - ctx.master.addons.trigger("processing_complete") + asyncio.get_event_loop().create_task(self.doread(ctx.options.rfile)) class ReadFileStdin(ReadFile): """Support the special case of "-" for reading from stdin""" - def load_flows_from_path(self, path: str) -> int: - if path == "-": - return self.load_flows(sys.stdin.buffer) + async def load_flows_from_path(self, path: str) -> int: + if path == "-": # pragma: no cover + # Need to think about how to test this. This function is scheduled + # onto the event loop, where a sys.stdin mock has no effect. + return await self.load_flows(sys.stdin.buffer) else: - return super().load_flows_from_path(path) + return await super().load_flows_from_path(path) diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index f5c4d599..51341f2f 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -25,6 +25,7 @@ def load_script(path: str) -> types.ModuleType: sys.modules.pop(fullname, None) oldpath = sys.path sys.path.insert(0, os.path.dirname(path)) + m = None try: loader = importlib.machinery.SourceFileLoader(fullname, path) spec = importlib.util.spec_from_loader(fullname, loader=loader) @@ -32,9 +33,11 @@ def load_script(path: str) -> types.ModuleType: loader.exec_module(m) if not getattr(m, "name", None): m.name = path # type: ignore - return m + except Exception as e: + script_error_handler(path, e, msg=str(e)) finally: sys.path[:] = oldpath + return m def script_error_handler(path, exc, msg="", tb=False): @@ -48,11 +51,11 @@ def script_error_handler(path, exc, msg="", tb=False): lineno = "" if hasattr(exc, "lineno"): lineno = str(exc.lineno) - log_msg = "in Script {}:{} {}".format(path, lineno, exception) + log_msg = "in script {}:{} {}".format(path, lineno, exception) if tb: etype, value, tback = sys.exc_info() tback = addonmanager.cut_traceback(tback, "invoke_addon") - log_msg = log_msg.join(["\n"] + traceback.format_exception(etype, value, tback)) + log_msg = log_msg + "\n" + "".join(traceback.format_exception(etype, value, tback)) ctx.log.error(log_msg) diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 8328cf80..e8ec993d 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -150,8 +150,9 @@ def mitmdump(args=None): # pragma: no cover if args.filter_args: v = " ".join(args.filter_args) return dict( - view_filter=v, save_stream_filter=v, + readfile_filter=v, + dumper_filter=v, ) return {} @@ -68,7 +68,7 @@ setup( "h2>=3.0.1,<4", "hyperframe>=5.1.0,<6", "kaitaistruct>=0.7,<0.9", - "ldap3>=2.4,<2.5", + "ldap3>=2.5,<2.6", "passlib>=1.6.5, <1.8", "pyasn1>=0.3.1,<0.5", "pyOpenSSL>=17.5,<17.6", @@ -85,9 +85,10 @@ setup( "pydivert>=2.0.3,<2.2", ], 'dev': [ + "asynctest>=0.12.0", "flake8>=3.5, <3.6", "Flask>=0.10.1, <0.13", - "mypy>=0.580,<0.581", + "mypy>=0.590,<0.591", "pytest-asyncio>=0.8", "pytest-cov>=2.5.1,<3", "pytest-faulthandler>=1.3.1,<2", diff --git a/test/mitmproxy/addons/test_readfile.py b/test/mitmproxy/addons/test_readfile.py index a0826a26..f7e0c5c5 100644 --- a/test/mitmproxy/addons/test_readfile.py +++ b/test/mitmproxy/addons/test_readfile.py @@ -1,7 +1,9 @@ +import asyncio import io from unittest import mock import pytest +import asynctest import mitmproxy.io from mitmproxy import exceptions @@ -38,69 +40,83 @@ def corrupt_data(): class TestReadFile: - @mock.patch('mitmproxy.master.Master.load_flow') - def test_configure(self, mck, tmpdir, data, corrupt_data): + def test_configure(self): + rf = readfile.ReadFile() + with taddons.context() as tctx: + tctx.configure(rf, readfile_filter="~q") + with pytest.raises(Exception, match="Invalid readfile filter"): + tctx.configure(rf, readfile_filter="~~") + + @pytest.mark.asyncio + async def test_read(self, tmpdir, data, corrupt_data): rf = readfile.ReadFile() with taddons.context(rf) as tctx: tf = tmpdir.join("tfile") - tf.write(data.getvalue()) - tctx.configure(rf, rfile=str(tf)) - assert not mck.called - rf.running() - assert mck.called + with asynctest.patch('mitmproxy.master.Master.load_flow') as mck: + tf.write(data.getvalue()) + tctx.configure( + rf, + rfile = str(tf), + readfile_filter = ".*" + ) + assert not mck.awaited + rf.running() + await asyncio.sleep(0) + assert mck.awaited tf.write(corrupt_data.getvalue()) tctx.configure(rf, rfile=str(tf)) - with pytest.raises(exceptions.OptionsError): - rf.running() + rf.running() + assert await tctx.master.await_log("corrupted") @pytest.mark.asyncio async def test_corrupt(self, corrupt_data): rf = readfile.ReadFile() with taddons.context(rf) as tctx: - with mock.patch('mitmproxy.master.Master.load_flow') as mck: - with pytest.raises(exceptions.FlowReadException): - rf.load_flows(io.BytesIO(b"qibble")) - assert not mck.called + with pytest.raises(exceptions.FlowReadException): + await rf.load_flows(io.BytesIO(b"qibble")) - tctx.master.clear() - with pytest.raises(exceptions.FlowReadException): - rf.load_flows(corrupt_data) - assert await tctx.master.await_log("file corrupted") - assert mck.called + tctx.master.clear() + with pytest.raises(exceptions.FlowReadException): + await rf.load_flows(corrupt_data) + assert await tctx.master.await_log("file corrupted") @pytest.mark.asyncio - async def test_nonexisting_file(self): + async def test_nonexistent_file(self): rf = readfile.ReadFile() with taddons.context(rf) as tctx: with pytest.raises(exceptions.FlowReadException): - rf.load_flows_from_path("nonexistent") + await rf.load_flows_from_path("nonexistent") assert await tctx.master.await_log("nonexistent") class TestReadFileStdin: - @mock.patch('mitmproxy.master.Master.load_flow') - @mock.patch('sys.stdin') - def test_stdin(self, stdin, load_flow, data, corrupt_data): + @asynctest.patch('sys.stdin') + @pytest.mark.asyncio + async def test_stdin(self, stdin, data, corrupt_data): rf = readfile.ReadFileStdin() - with taddons.context(rf) as tctx: - stdin.buffer = data - tctx.configure(rf, rfile="-") - assert not load_flow.called - rf.running() - assert load_flow.called + with taddons.context(rf): + with asynctest.patch('mitmproxy.master.Master.load_flow') as mck: + stdin.buffer = data + assert not mck.awaited + await rf.load_flows(stdin.buffer) + assert mck.awaited - stdin.buffer = corrupt_data - tctx.configure(rf, rfile="-") - with pytest.raises(exceptions.OptionsError): - rf.running() + stdin.buffer = corrupt_data + with pytest.raises(exceptions.FlowReadException): + await rf.load_flows(stdin.buffer) + @pytest.mark.asyncio @mock.patch('mitmproxy.master.Master.load_flow') - def test_normal(self, load_flow, tmpdir, data): + async def test_normal(self, load_flow, tmpdir, data): rf = readfile.ReadFileStdin() - with taddons.context(rf): - tfile = tmpdir.join("tfile") - tfile.write(data.getvalue()) - rf.load_flows_from_path(str(tfile)) - assert load_flow.called + with taddons.context(rf) as tctx: + tf = tmpdir.join("tfile") + with asynctest.patch('mitmproxy.master.Master.load_flow') as mck: + tf.write(data.getvalue()) + tctx.configure(rf, rfile=str(tf)) + assert not mck.awaited + rf.running() + await asyncio.sleep(0) + assert mck.awaited diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index 3f0ce68c..96e19841 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -12,18 +12,27 @@ from mitmproxy.test import tflow from mitmproxy.test import tutils -def test_load_script(): - ns = script.load_script( - tutils.test_data.path( - "mitmproxy/data/addonscripts/recorder/recorder.py" +@pytest.mark.asyncio +async def test_load_script(): + with taddons.context() as tctx: + ns = script.load_script( + tutils.test_data.path( + "mitmproxy/data/addonscripts/recorder/recorder.py" + ) ) - ) - assert ns.addons + assert ns.addons - with pytest.raises(FileNotFoundError): script.load_script( "nonexistent" ) + assert await tctx.master.await_log("No such file or directory") + + script.load_script( + tutils.test_data.path( + "mitmproxy/data/addonscripts/recorder/error.py" + ) + ) + assert await tctx.master.await_log("invalid syntax") def test_load_fullname(): diff --git a/test/mitmproxy/data/addonscripts/recorder/error.py b/test/mitmproxy/data/addonscripts/recorder/error.py new file mode 100644 index 00000000..2e7e648a --- /dev/null +++ b/test/mitmproxy/data/addonscripts/recorder/error.py @@ -0,0 +1,7 @@ +""" +This file is intended to have syntax errors for test purposes +""" + +impotr recorder # Intended Syntax Error + +addons = [recorder.Recorder("e")] |