diff options
| -rw-r--r-- | mitmproxy/builtins/dumper.py | 252 | ||||
| -rw-r--r-- | mitmproxy/controller.py | 6 | ||||
| -rw-r--r-- | mitmproxy/dump.py | 227 | ||||
| -rw-r--r-- | test/mitmproxy/builtins/test_dumper.py | 86 | ||||
| -rw-r--r-- | test/mitmproxy/mastertest.py | 7 | ||||
| -rw-r--r-- | test/mitmproxy/test_dump.py | 83 |
6 files changed, 360 insertions, 301 deletions
diff --git a/mitmproxy/builtins/dumper.py b/mitmproxy/builtins/dumper.py new file mode 100644 index 00000000..239630fb --- /dev/null +++ b/mitmproxy/builtins/dumper.py @@ -0,0 +1,252 @@ +from __future__ import absolute_import, print_function, division + +import itertools +import traceback + +import click + +from mitmproxy import contentviews +from mitmproxy import ctx +from mitmproxy import exceptions +from mitmproxy import filt +from netlib import human +from netlib import strutils + + +def indent(n, text): + l = str(text).strip().splitlines() + pad = " " * n + return "\n".join(pad + i for i in l) + + +class Dumper(): + def __init__(self): + self.filter = None + self.flow_detail = None + self.outfp = None + self.showhost = None + + def echo(self, text, ident=None, **style): + if ident: + text = indent(ident, text) + click.secho(text, file=self.outfp, **style) + if self.outfp: + self.outfp.flush() + + def _echo_message(self, message): + if self.flow_detail >= 2 and hasattr(message, "headers"): + headers = "\r\n".join( + "{}: {}".format( + click.style( + strutils.bytes_to_escaped_str(k), fg="blue", bold=True + ), + click.style( + strutils.bytes_to_escaped_str(v), fg="blue" + ) + ) + for k, v in message.headers.fields + ) + self.echo(headers, ident=4) + if self.flow_detail >= 3: + try: + content = message.content + except ValueError: + content = message.get_content(strict=False) + + if content is None: + self.echo("(content missing)", ident=4) + elif content: + self.echo("") + + try: + type, lines = contentviews.get_content_view( + contentviews.get("Auto"), + content, + headers=getattr(message, "headers", None) + ) + except exceptions.ContentViewException: + s = "Content viewer failed: \n" + traceback.format_exc() + ctx.log.debug(s) + type, lines = contentviews.get_content_view( + contentviews.get("Raw"), + content, + headers=getattr(message, "headers", None) + ) + + styles = dict( + highlight=dict(bold=True), + offset=dict(fg="blue"), + header=dict(fg="green", bold=True), + text=dict(fg="green") + ) + + def colorful(line): + yield u" " # we can already indent here + for (style, text) in line: + yield click.style(text, **styles.get(style, {})) + + if self.flow_detail == 3: + lines_to_echo = itertools.islice(lines, 70) + else: + lines_to_echo = lines + + lines_to_echo = list(lines_to_echo) + + content = u"\r\n".join( + u"".join(colorful(line)) for line in lines_to_echo + ) + + self.echo(content) + if next(lines, None): + self.echo("(cut off)", ident=4, dim=True) + + if self.flow_detail >= 2: + self.echo("") + + def _echo_request_line(self, flow): + if flow.request.stickycookie: + stickycookie = click.style( + "[stickycookie] ", fg="yellow", bold=True + ) + else: + stickycookie = "" + + if flow.client_conn: + client = click.style( + strutils.escape_control_characters( + flow.client_conn.address.host + ), + bold=True + ) + elif flow.request.is_replay: + client = click.style("[replay]", fg="yellow", bold=True) + else: + client = "" + + method = flow.request.method + method_color = dict( + GET="green", + DELETE="red" + ).get(method.upper(), "magenta") + method = click.style( + strutils.escape_control_characters(method), + fg=method_color, + bold=True + ) + if self.showhost: + url = flow.request.pretty_url + else: + url = flow.request.url + url = click.style(strutils.escape_control_characters(url), bold=True) + + httpversion = "" + if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"): + # We hide "normal" HTTP 1. + httpversion = " " + flow.request.http_version + + line = "{stickycookie}{client} {method} {url}{httpversion}".format( + stickycookie=stickycookie, + client=client, + method=method, + url=url, + httpversion=httpversion + ) + self.echo(line) + + def _echo_response_line(self, flow): + if flow.response.is_replay: + replay = click.style("[replay] ", fg="yellow", bold=True) + else: + replay = "" + + code = flow.response.status_code + code_color = None + if 200 <= code < 300: + code_color = "green" + elif 300 <= code < 400: + code_color = "magenta" + elif 400 <= code < 600: + code_color = "red" + code = click.style( + str(code), + fg=code_color, + bold=True, + blink=(code == 418) + ) + reason = click.style( + strutils.escape_control_characters(flow.response.reason), + fg=code_color, + bold=True + ) + + if flow.response.raw_content is None: + size = "(content missing)" + else: + size = human.pretty_size(len(flow.response.raw_content)) + size = click.style(size, bold=True) + + arrows = click.style(" <<", bold=True) + + line = "{replay} {arrows} {code} {reason} {size}".format( + replay=replay, + arrows=arrows, + code=code, + reason=reason, + size=size + ) + self.echo(line) + + def echo_flow(self, f): + if f.request: + self._echo_request_line(f) + self._echo_message(f.request) + + if f.response: + self._echo_response_line(f) + self._echo_message(f.response) + + if f.error: + self.echo(" << {}".format(f.error.msg), bold=True, fg="red") + + def match(self, f): + if self.flow_detail == 0: + return False + if not self.filt: + return True + elif f.match(self.filt): + return True + return False + + def configure(self, options): + if options.filtstr: + self.filt = filt.parse(options.filtstr) + if not self.filt: + raise exceptions.OptionsError( + "Invalid filter expression: %s" % options.filtstr + ) + else: + self.filt = None + self.flow_detail = options.flow_detail + self.outfp = options.tfile + self.showhost = options.showhost + + def response(self, f): + if self.match(f): + self.echo_flow(f) + + def error(self, f): + if self.match(f): + self.echo_flow(f) + + def tcp_message(self, f): + # FIXME: Filter should be applied here + if self.options.flow_detail == 0: + return + message = f.messages[-1] + direction = "->" if message.from_client else "<-" + self.echo("{client} {direction} tcp {direction} {server}".format( + client=repr(f.client_conn.address), + server=repr(f.server_conn.address), + direction=direction, + )) + self._echo_message(message) diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index 54d75e6b..070ec862 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -220,6 +220,12 @@ def handler(f): if handling and not message.reply.acked and not message.reply.taken: message.reply.ack() + + # Reset the handled flag - it's common for us to feed the same object + # through handlers repeatedly, so we don't want this to persist across + # calls. + if message.reply.handled: + message.reply.handled = False return ret # Mark this function as a handler wrapper wrapper.__dict__["__handler"] = True diff --git a/mitmproxy/dump.py b/mitmproxy/dump.py index e7cebf99..65eb515b 100644 --- a/mitmproxy/dump.py +++ b/mitmproxy/dump.py @@ -1,24 +1,19 @@ from __future__ import absolute_import, print_function, division -import itertools import sys -import traceback - -import click from typing import Optional # noqa import typing # noqa -from mitmproxy import contentviews +import click + from mitmproxy import controller from mitmproxy import exceptions -from mitmproxy import filt from mitmproxy import flow from mitmproxy import builtins from mitmproxy import utils -from netlib import human +from mitmproxy.builtins import dumper from netlib import tcp -from netlib import strutils class DumpError(Exception): @@ -28,9 +23,9 @@ class DumpError(Exception): class Options(flow.options.Options): def __init__( self, + keepserving=False, # type: bool filtstr=None, # type: Optional[str] flow_detail=1, # type: int - keepserving=False, # type: bool tfile=None, # type: Optional[typing.io.TextIO] **kwargs ): @@ -47,10 +42,9 @@ class DumpMaster(flow.FlowMaster): flow.FlowMaster.__init__(self, options, server, flow.State()) self.has_errored = False self.addons.add(*builtins.default_addons()) + self.addons.add(dumper.Dumper()) # This line is just for type hinting self.options = self.options # type: Options - self.o = options - self.showhost = options.showhost self.replay_ignore_params = options.replay_ignore_params self.replay_ignore_content = options.replay_ignore_content self.replay_ignore_host = options.replay_ignore_host @@ -64,11 +58,6 @@ class DumpMaster(flow.FlowMaster): "HTTP/2 is disabled. Use --no-http2 to silence this warning.", file=sys.stderr) - if options.filtstr: - self.filt = filt.parse(options.filtstr) - else: - self.filt = None - if options.setheaders: for i in options.setheaders: self.setheaders.add(*i) @@ -115,221 +104,21 @@ class DumpMaster(flow.FlowMaster): if level == "error": self.has_errored = True if self.options.verbosity >= utils.log_tier(level): - self.echo( + click.secho( e, + file=self.options.tfile, fg="red" if level == "error" else None, dim=(level == "debug"), err=(level == "error") ) - @staticmethod - def indent(n, text): - l = str(text).strip().splitlines() - pad = " " * n - return "\n".join(pad + i for i in l) - - def echo(self, text, indent=None, **style): - if indent: - text = self.indent(indent, text) - click.secho(text, file=self.options.tfile, **style) - - def _echo_message(self, message): - if self.options.flow_detail >= 2 and hasattr(message, "headers"): - headers = "\r\n".join( - "{}: {}".format( - click.style(strutils.bytes_to_escaped_str(k), fg="blue", bold=True), - click.style(strutils.bytes_to_escaped_str(v), fg="blue")) - for k, v in message.headers.fields - ) - self.echo(headers, indent=4) - if self.options.flow_detail >= 3: - try: - content = message.content - except ValueError: - content = message.get_content(strict=False) - - if content is None: - self.echo("(content missing)", indent=4) - elif content: - self.echo("") - - try: - type, lines = contentviews.get_content_view( - contentviews.get("Auto"), - content, - headers=getattr(message, "headers", None) - ) - except exceptions.ContentViewException: - s = "Content viewer failed: \n" + traceback.format_exc() - self.add_log(s, "debug") - type, lines = contentviews.get_content_view( - contentviews.get("Raw"), - content, - headers=getattr(message, "headers", None) - ) - - styles = dict( - highlight=dict(bold=True), - offset=dict(fg="blue"), - header=dict(fg="green", bold=True), - text=dict(fg="green") - ) - - def colorful(line): - yield u" " # we can already indent here - for (style, text) in line: - yield click.style(text, **styles.get(style, {})) - - if self.options.flow_detail == 3: - lines_to_echo = itertools.islice(lines, 70) - else: - lines_to_echo = lines - - lines_to_echo = list(lines_to_echo) - - content = u"\r\n".join( - u"".join(colorful(line)) for line in lines_to_echo - ) - - self.echo(content) - if next(lines, None): - self.echo("(cut off)", indent=4, dim=True) - - if self.options.flow_detail >= 2: - self.echo("") - - def _echo_request_line(self, flow): - if flow.request.stickycookie: - stickycookie = click.style( - "[stickycookie] ", fg="yellow", bold=True - ) - else: - stickycookie = "" - - if flow.client_conn: - client = click.style(strutils.escape_control_characters(flow.client_conn.address.host), bold=True) - else: - client = click.style("[replay]", fg="yellow", bold=True) - - method = flow.request.method - method_color = dict( - GET="green", - DELETE="red" - ).get(method.upper(), "magenta") - method = click.style(strutils.escape_control_characters(method), fg=method_color, bold=True) - if self.showhost: - url = flow.request.pretty_url - else: - url = flow.request.url - url = click.style(strutils.escape_control_characters(url), bold=True) - - httpversion = "" - if flow.request.http_version not in ("HTTP/1.1", "HTTP/1.0"): - httpversion = " " + flow.request.http_version # We hide "normal" HTTP 1. - - line = "{stickycookie}{client} {method} {url}{httpversion}".format( - stickycookie=stickycookie, - client=client, - method=method, - url=url, - httpversion=httpversion - ) - self.echo(line) - - def _echo_response_line(self, flow): - if flow.response.is_replay: - replay = click.style("[replay] ", fg="yellow", bold=True) - else: - replay = "" - - code = flow.response.status_code - code_color = None - if 200 <= code < 300: - code_color = "green" - elif 300 <= code < 400: - code_color = "magenta" - elif 400 <= code < 600: - code_color = "red" - code = click.style(str(code), fg=code_color, bold=True, blink=(code == 418)) - reason = click.style(strutils.escape_control_characters(flow.response.reason), fg=code_color, bold=True) - - if flow.response.raw_content is None: - size = "(content missing)" - else: - size = human.pretty_size(len(flow.response.raw_content)) - size = click.style(size, bold=True) - - arrows = click.style("<<", bold=True) - - line = "{replay} {arrows} {code} {reason} {size}".format( - replay=replay, - arrows=arrows, - code=code, - reason=reason, - size=size - ) - self.echo(line) - - def echo_flow(self, f): - if self.options.flow_detail == 0: - return - - if f.request: - self._echo_request_line(f) - self._echo_message(f.request) - - if f.response: - self._echo_response_line(f) - self._echo_message(f.response) - - if f.error: - self.echo(" << {}".format(f.error.msg), bold=True, fg="red") - - if self.options.tfile: - self.options.tfile.flush() - - def _process_flow(self, f): - if self.filt and not f.match(self.filt): - return - - self.echo_flow(f) - @controller.handler def request(self, f): - f = flow.FlowMaster.request(self, f) + f = super(DumpMaster, self).request(f) if f: self.state.delete_flow(f) return f - @controller.handler - def response(self, f): - f = flow.FlowMaster.response(self, f) - if f: - self._process_flow(f) - return f - - @controller.handler - def error(self, f): - flow.FlowMaster.error(self, f) - if f: - self._process_flow(f) - return f - - @controller.handler - def tcp_message(self, f): - super(DumpMaster, self).tcp_message(f) - - if self.options.flow_detail == 0: - return - message = f.messages[-1] - direction = "->" if message.from_client else "<-" - self.echo("{client} {direction} tcp {direction} {server}".format( - client=repr(f.client_conn.address), - server=repr(f.server_conn.address), - direction=direction, - )) - self._echo_message(message) - def run(self): # pragma: no cover if self.options.rfile and not self.options.keepserving: return diff --git a/test/mitmproxy/builtins/test_dumper.py b/test/mitmproxy/builtins/test_dumper.py new file mode 100644 index 00000000..57e3d036 --- /dev/null +++ b/test/mitmproxy/builtins/test_dumper.py @@ -0,0 +1,86 @@ +from .. import tutils, mastertest +from six.moves import cStringIO as StringIO + +from mitmproxy.builtins import dumper +from mitmproxy.flow import state +from mitmproxy import exceptions +from mitmproxy import dump +from mitmproxy import models +import netlib.tutils +import mock + + +class TestDumper(mastertest.MasterTest): + def test_simple(self): + d = dumper.Dumper() + sio = StringIO() + + d.configure(dump.Options(tfile = sio, flow_detail = 0)) + d.response(tutils.tflow()) + assert not sio.getvalue() + + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + d.response(tutils.tflow()) + assert sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + d.response(tutils.tflow(resp=True)) + assert "<<" in sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + d.response(tutils.tflow(err=True)) + assert "<<" in sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + flow = tutils.tflow() + flow.request = netlib.tutils.treq() + flow.request.stickycookie = True + flow.client_conn = mock.MagicMock() + flow.client_conn.address.host = "foo" + flow.response = netlib.tutils.tresp(content=None) + flow.response.is_replay = True + flow.response.status_code = 300 + d.response(flow) + assert sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio, flow_detail = 4)) + flow = tutils.tflow(resp=netlib.tutils.tresp(content=b"{")) + flow.response.headers["content-type"] = "application/json" + flow.response.status_code = 400 + d.response(flow) + assert sio.getvalue() + + sio = StringIO() + d.configure(dump.Options(tfile = sio)) + flow = tutils.tflow() + flow.request.content = None + flow.response = models.HTTPResponse.wrap(netlib.tutils.tresp()) + flow.response.content = None + d.response(flow) + assert "content missing" in sio.getvalue() + + +class TestContentView(mastertest.MasterTest): + @mock.patch("mitmproxy.contentviews.get_content_view") + def test_contentview(self, get_content_view): + se = exceptions.ContentViewException(""), ("x", iter([])) + get_content_view.side_effect = se + + s = state.State() + sio = StringIO() + m = mastertest.RecordingMaster( + dump.Options( + flow_detail=4, + verbosity=3, + tfile=sio, + ), + None, s + ) + d = dumper.Dumper() + m.addons.add(d) + self.invoke(m, "response", tutils.tflow()) + assert "Content viewer failed" in m.event_log[0][1] diff --git a/test/mitmproxy/mastertest.py b/test/mitmproxy/mastertest.py index d1fe8cb4..dcc0dc48 100644 --- a/test/mitmproxy/mastertest.py +++ b/test/mitmproxy/mastertest.py @@ -8,11 +8,12 @@ from mitmproxy import flow, proxy, models, controller class MasterTest: - def invoke(self, master, handler, message): + def invoke(self, master, handler, *message): with master.handlecontext(): func = getattr(master, handler) - func(message) - message.reply = controller.DummyReply() + func(*message) + if message: + message[0].reply = controller.DummyReply() def cycle(self, master, content): f = tutils.tflow(req=netlib.tutils.treq(content=content)) diff --git a/test/mitmproxy/test_dump.py b/test/mitmproxy/test_dump.py index c94630a9..90f33264 100644 --- a/test/mitmproxy/test_dump.py +++ b/test/mitmproxy/test_dump.py @@ -1,67 +1,11 @@ import os from six.moves import cStringIO as StringIO -from mitmproxy.exceptions import ContentViewException -import netlib.tutils - -from mitmproxy import dump, flow, models, exceptions +from mitmproxy import dump, flow, exceptions from . import tutils, mastertest import mock -def test_strfuncs(): - o = dump.Options( - tfile = StringIO(), - flow_detail = 0, - ) - m = dump.DumpMaster(None, o) - - m.o.flow_detail = 0 - m.echo_flow(tutils.tflow()) - assert not o.tfile.getvalue() - - m.o.flow_detail = 4 - m.echo_flow(tutils.tflow()) - assert o.tfile.getvalue() - - o.tfile = StringIO() - m.echo_flow(tutils.tflow(resp=True)) - assert "<<" in o.tfile.getvalue() - - o.tfile = StringIO() - m.echo_flow(tutils.tflow(err=True)) - assert "<<" in o.tfile.getvalue() - - flow = tutils.tflow() - flow.request = netlib.tutils.treq() - flow.request.stickycookie = True - flow.client_conn = mock.MagicMock() - flow.client_conn.address.host = "foo" - flow.response = netlib.tutils.tresp(content=None) - flow.response.is_replay = True - flow.response.status_code = 300 - m.echo_flow(flow) - - flow = tutils.tflow(resp=netlib.tutils.tresp(content=b"{")) - flow.response.headers["content-type"] = "application/json" - flow.response.status_code = 400 - m.echo_flow(flow) - - -@mock.patch("mitmproxy.contentviews.get_content_view") -def test_contentview(get_content_view): - get_content_view.side_effect = ContentViewException(""), ("x", iter([])) - - o = dump.Options( - flow_detail=4, - verbosity=3, - tfile=StringIO(), - ) - m = dump.DumpMaster(None, o) - m.echo_flow(tutils.tflow()) - assert "Content viewer failed" in m.options.tfile.getvalue() - - class TestDumpMaster(mastertest.MasterTest): def dummy_cycle(self, master, n, content): mastertest.MasterTest.dummy_cycle(self, master, n, content) @@ -72,11 +16,7 @@ class TestDumpMaster(mastertest.MasterTest): options["verbosity"] = 0 if "flow_detail" not in options: options["flow_detail"] = 0 - o = dump.Options( - filtstr=filt, - tfile=StringIO(), - **options - ) + o = dump.Options(filtstr=filt, tfile=StringIO(), **options) return dump.DumpMaster(None, o) def test_basic(self): @@ -104,24 +44,10 @@ class TestDumpMaster(mastertest.MasterTest): ) m = dump.DumpMaster(None, o) f = tutils.tflow(err=True) - m.request(f) + m.error(f) assert m.error(f) assert "error" in o.tfile.getvalue() - def test_missing_content(self): - o = dump.Options( - flow_detail=3, - tfile=StringIO(), - ) - m = dump.DumpMaster(None, o) - f = tutils.tflow() - f.request.content = None - m.request(f) - f.response = models.HTTPResponse.wrap(netlib.tutils.tresp()) - f.response.content = None - m.response(f) - assert "content missing" in o.tfile.getvalue() - def test_replay(self): o = dump.Options(server_replay=["nonexistent"], kill=True) tutils.raises(dump.DumpError, dump.DumpMaster, None, o) @@ -155,9 +81,8 @@ class TestDumpMaster(mastertest.MasterTest): self.flowfile(p) assert "GET" in self.dummy_cycle( self.mkmaster(None, flow_detail=1, rfile=p), - 0, b"", + 1, b"", ) - tutils.raises( dump.DumpError, self.mkmaster, None, verbosity=1, rfile="/nonexistent" |