diff options
30 files changed, 113 insertions, 85 deletions
@@ -26,7 +26,7 @@ and pathod websites. |mitmproxy_site| -The latest documentation for mitmproxy is also available on ReadTheDocs. +The latest documentation for mitmproxy is available on our website. |mitmproxy_docs| @@ -151,7 +151,7 @@ with the following command: :alt: mitmproxy.org .. |mitmproxy_docs| image:: https://shields.mitmproxy.org/api/docs-latest-brightgreen.svg - :target: http://docs.mitmproxy.org/en/latest/ + :target: https://mitmproxy.org/docs/latest/ :alt: mitmproxy documentation .. |mitmproxy_discourse| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-discourse.mitmproxy.org-orange.svg diff --git a/docs/raw/proxy-modes.vsdx b/docs/raw/proxy-modes.vsdx Binary files differindex 0128a142..d115ea9e 100644..100755 --- a/docs/raw/proxy-modes.vsdx +++ b/docs/raw/proxy-modes.vsdx diff --git a/docs/src/content/howto-transparent-vms.md b/docs/src/content/howto-transparent-vms.md index b186fd39..1446ede7 100644 --- a/docs/src/content/howto-transparent-vms.md +++ b/docs/src/content/howto-transparent-vms.md @@ -106,7 +106,7 @@ sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-p Finally, we can run mitmproxy in transparent mode with {{< highlight bash >}} -mitmproxy -T +mitmproxy --mode transparent {{< / highlight >}} The proxied machine cannot to leak any data outside of HTTP or DNS requests. If diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index 224cb5ee..3d99e9dc 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -58,7 +58,7 @@ dropped privileges. It can be used as follows: gcc examples/complex/full_transparency_shim.c -o mitmproxy_shim -lcap sudo chown root:root mitmproxy_shim sudo chmod u+s mitmproxy_shim -./mitmproxy_shim $(which mitmproxy) -T --spoof-source-address +./mitmproxy_shim $(which mitmproxy) --mode transparent --set spoof-source-address {{< / highlight >}} @@ -112,10 +112,10 @@ something like this: You probably want a command like this: {{< highlight bash >}} -mitmproxy -T --host +mitmproxy --mode transparent --showhost {{< / highlight >}} -The `-T` flag turns on transparent mode, and the `--host` argument tells +The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells mitmproxy to use the value of the Host header for URL display. ### 6. Finally, configure your test device @@ -163,10 +163,10 @@ doas pfctl -e You probably want a command like this: {{< highlight bash >}} -mitmproxy -T --host +mitmproxy --mode transparent --showhost {{< / highlight >}} -The `-T` flag turns on transparent mode, and the `--host` argument tells +The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells mitmproxy to use the value of the Host header for URL display. ### 6. Finally, configure your test device @@ -245,10 +245,10 @@ tighten the restriction up to the user running mitmproxy. You probably want a command like this: {{< highlight bash >}} -mitmproxy -T --host +mitmproxy --mode transparent --showhost {{< / highlight >}} -The `-T` flag turns on transparent mode, and the `--host` argument tells +The `--mode transparent` flag turns on transparent mode, and the `--showhost` argument tells mitmproxy to use the value of the Host header for URL display. ### 6. Finally, configure your test device diff --git a/docs/src/static/schematics/proxy-modes-reverse.png b/docs/src/static/schematics/proxy-modes-reverse.png Binary files differindex 071d3fc8..87372688 100644..100755 --- a/docs/src/static/schematics/proxy-modes-reverse.png +++ b/docs/src/static/schematics/proxy-modes-reverse.png diff --git a/docs/src/static/schematics/proxy-modes-transparent-1.png b/docs/src/static/schematics/proxy-modes-transparent-1.png Binary files differindex 002e0e76..557406f8 100644..100755 --- a/docs/src/static/schematics/proxy-modes-transparent-1.png +++ b/docs/src/static/schematics/proxy-modes-transparent-1.png diff --git a/docs/src/static/schematics/proxy-modes-transparent-2.png b/docs/src/static/schematics/proxy-modes-transparent-2.png Binary files differindex 41997b05..ecc09f5a 100644..100755 --- a/docs/src/static/schematics/proxy-modes-transparent-2.png +++ b/docs/src/static/schematics/proxy-modes-transparent-2.png diff --git a/docs/src/static/schematics/proxy-modes-transparent-3.png b/docs/src/static/schematics/proxy-modes-transparent-3.png Binary files differindex ee26cb4f..177aaaef 100644..100755 --- a/docs/src/static/schematics/proxy-modes-transparent-3.png +++ b/docs/src/static/schematics/proxy-modes-transparent-3.png diff --git a/docs/src/static/schematics/proxy-modes-transparent-wrong.png b/docs/src/static/schematics/proxy-modes-transparent-wrong.png Binary files differindex ca501e93..c69c6b70 100644..100755 --- a/docs/src/static/schematics/proxy-modes-transparent-wrong.png +++ b/docs/src/static/schematics/proxy-modes-transparent-wrong.png diff --git a/docs/src/static/schematics/proxy-modes-upstream.png b/docs/src/static/schematics/proxy-modes-upstream.png Binary files differindex d40a6494..daa2b9af 100644..100755 --- a/docs/src/static/schematics/proxy-modes-upstream.png +++ b/docs/src/static/schematics/proxy-modes-upstream.png diff --git a/examples/simple/internet_in_mirror.py b/examples/simple/internet_in_mirror.py new file mode 100644 index 00000000..5d3e555d --- /dev/null +++ b/examples/simple/internet_in_mirror.py @@ -0,0 +1,9 @@ +""" +This script reflects all content passing through the proxy. +""" +from mitmproxy import http + + +def response(flow: http.HTTPFlow) -> None: + reflector = b"<style>body {transform: scaleX(-1);}</style></head>" + flow.response.content = flow.response.content.replace(b"</head>", reflector) diff --git a/examples/simple/upsidedownternet.py b/examples/simple/upsidedownternet.py deleted file mode 100644 index f150a5c3..00000000 --- a/examples/simple/upsidedownternet.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -This script rotates all images passing through the proxy by 180 degrees. -""" -import io -from PIL import Image -from mitmproxy import http - - -def response(flow: http.HTTPFlow) -> None: - if flow.response.headers.get("content-type", "").startswith("image"): - s = io.BytesIO(flow.response.content) - img = Image.open(s).rotate(180) - s2 = io.BytesIO() - img.save(s2, "png") - flow.response.content = s2.getvalue() - flow.response.headers["content-type"] = "image/png" diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py index 37c501ee..bfaacf6d 100644 --- a/mitmproxy/addonmanager.py +++ b/mitmproxy/addonmanager.py @@ -92,6 +92,13 @@ class Loader: help: str, choices: typing.Optional[typing.Sequence[str]] = None ) -> None: + """ + Add an option to mitmproxy. + + Help should be a single paragraph with no linebreaks - it will be + reflowed by tools. Information on the data type should be omitted - + it will be generated and added by tools as needed. + """ if name in self.master.options: existing = self.master.options._options[name] same_signature = ( diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 0df2d339..aaad8aa2 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -47,14 +47,18 @@ class Dumper: "The default content view mode.", choices = [i.name.lower() for i in contentviews.views] ) + loader.add_option( + "dumper_filter", typing.Optional[str], None, + "Limit which flows are dumped." + ) def configure(self, updated): - if "view_filter" in updated: - if ctx.options.view_filter: - self.filter = flowfilter.parse(ctx.options.view_filter) + if "dumper_filter" in updated: + if ctx.options.dumper_filter: + self.filter = flowfilter.parse(ctx.options.dumper_filter) if not self.filter: raise exceptions.OptionsError( - "Invalid filter expression: %s" % ctx.options.view_filter + "Invalid filter expression: %s" % ctx.options.dumper_filter ) else: self.filter = None diff --git a/mitmproxy/addons/termlog.py b/mitmproxy/addons/termlog.py index 2a7e2d09..f09aa4b4 100644 --- a/mitmproxy/addons/termlog.py +++ b/mitmproxy/addons/termlog.py @@ -14,13 +14,20 @@ class TermLog: def __init__(self, outfile=None): self.outfile = outfile + def load(self, loader): + loader.add_option( + "termlog_verbosity", str, 'info', + "Log verbosity.", + choices=log.LogTierOrder + ) + def log(self, e): if log.log_tier(e.level) == log.log_tier("error"): outfile = self.outfile or realstderr else: outfile = self.outfile or realstdout - if log.log_tier(ctx.options.verbosity) >= log.log_tier(e.level): + if log.log_tier(ctx.options.termlog_verbosity) >= log.log_tier(e.level): click.secho( e.msg, file=outfile, diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index c0f7e7b4..598cb66d 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -147,6 +147,10 @@ class View(collections.Sequence): def load(self, loader): loader.add_option( + "view_filter", typing.Optional[str], None, + "Limit the view to matching flows." + ) + loader.add_option( "view_order", str, "time", "Flow sort order.", choices=list(map(lambda c: c[1], orders)), diff --git a/mitmproxy/log.py b/mitmproxy/log.py index 3083a000..d2988011 100644 --- a/mitmproxy/log.py +++ b/mitmproxy/log.py @@ -57,5 +57,14 @@ class Log: self.master.add_log(text, level) +LogTierOrder = [ + "error", + "warn", + "info", + "alert", + "debug", +] + + def log_tier(level): return dict(error=0, warn=1, info=2, alert=2, debug=3).get(level) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 7a00bb87..b6f3398b 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -3,23 +3,10 @@ from typing import Optional, Sequence from mitmproxy import optmanager from mitmproxy.net import tls -log_verbosity = [ - "error", - "warn", - "info", - "alert", - "debug", -] CA_DIR = "~/.mitmproxy" LISTEN_PORT = 8080 -# Some help text style guidelines: -# -# - Should be a single paragraph with no linebreaks. Help will be reflowed by -# tools. -# - Avoid adding information about the data type - we can generate that. - class Options(optmanager.OptManager): @@ -42,6 +29,7 @@ class Options(optmanager.OptManager): mode = None # type: str rawtcp = None # type: bool server = None # type: bool + showhost = None # type: bool spoof_source_address = None # type: bool ssl_insecure = None # type: bool ssl_verify_upstream_trusted_ca = None # type: Optional[str] @@ -53,13 +41,6 @@ class Options(optmanager.OptManager): upstream_cert = None # type: bool websocket = None # type: bool - # FIXME: Options that must be migrated to addons, but are complicated - # because they're used by more than one addon, or because they're - # embedded in the core code somehow. - showhost = None # type: bool - verbosity = None # type: str - view_filter = None # type: Optional[str] - def __init__(self, **kwargs) -> None: super().__init__() self.add_option( @@ -70,11 +51,6 @@ class Options(optmanager.OptManager): "showhost", bool, False, "Use the Host header to construct URLs for display." ) - self.add_option( - "verbosity", str, 'info', - "Log verbosity.", - choices=log_verbosity - ) # Proxy options self.add_option( @@ -231,9 +207,4 @@ class Options(optmanager.OptManager): """ ) - self.add_option( - "view_filter", Optional[str], None, - "Limit which flows are displayed." - ) - self.update(**kwargs) diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index 3d76d20c..2b9ff334 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -7,6 +7,7 @@ from mitmproxy import command from mitmproxy import exceptions from mitmproxy import flow from mitmproxy import http +from mitmproxy import log from mitmproxy import contentviews from mitmproxy.utils import strutils import mitmproxy.types @@ -82,6 +83,11 @@ class ConsoleAddon: choices = [i.name.lower() for i in contentviews.views] ) loader.add_option( + "console_eventlog_verbosity", str, 'info', + "EventLog verbosity.", + choices=log.LogTierOrder + ) + loader.add_option( "console_layout", str, "single", "Console layout.", choices=sorted(console_layouts), diff --git a/mitmproxy/tools/console/eventlog.py b/mitmproxy/tools/console/eventlog.py index 8083180d..f3305469 100644 --- a/mitmproxy/tools/console/eventlog.py +++ b/mitmproxy/tools/console/eventlog.py @@ -21,7 +21,7 @@ class EventLog(urwid.ListBox, layoutwidget.LayoutWidget): master.events.sig_add.connect(self.add_event) master.events.sig_refresh.connect(self.refresh_events) - self.master.options.subscribe(self.refresh_events, ["verbosity"]) + self.master.options.subscribe(self.refresh_events, ["console_eventlog_verbosity"]) self.refresh_events() super().__init__(self.walker) @@ -44,7 +44,7 @@ class EventLog(urwid.ListBox, layoutwidget.LayoutWidget): return super().keypress(size, key) def add_event(self, event_store, entry: log.LogEntry): - if log.log_tier(self.master.options.verbosity) < log.log_tier(entry.level): + if log.log_tier(self.master.options.console_eventlog_verbosity) < log.log_tier(entry.level): return txt = "%s: %s" % (entry.level, str(entry.msg)) if entry.level in ("error", "warn", "alert"): diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 76d8724a..5da6ef0b 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -87,7 +87,7 @@ class ConsoleMaster(master.Master): ) def sig_add_log(self, event_store, entry: log.LogEntry): - if log.log_tier(self.options.verbosity) < log.log_tier(entry.level): + if log.log_tier(self.options.console_eventlog_verbosity) < log.log_tier(entry.level): return if entry.level in ("error", "warn", "alert"): if self.first_tick: diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 5c2d9f2f..330060f7 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -46,10 +46,10 @@ def process_options(parser, opts, args): if args.quiet or args.options or args.commands: # also reduce log verbosity if --options or --commands is passed, # we don't want log messages from regular startup then. - args.verbosity = 'error' + args.termlog_verbosity = 'error' args.flow_detail = 0 if args.verbose: - args.verbosity = 'debug' + args.termlog_verbosity = 'debug' args.flow_detail = 2 adict = {} @@ -104,9 +104,7 @@ def run( master.server = server master.addons.trigger("configure", opts.keys()) master.addons.trigger("tick") - remaining = opts.update_known(**unknown) - if remaining and log.log_tier(opts.verbosity) > 1: - print("Ignored options: %s" % remaining) + opts.update_known(**unknown) if args.options: print(optmanager.dump_defaults(opts)) sys.exit(0) diff --git a/release/README.md b/release/README.md index 2b709318..b2f97aab 100644 --- a/release/README.md +++ b/release/README.md @@ -16,7 +16,11 @@ Make sure run all these steps on the correct branch you want to create a new rel - Attach all files from the new release folder on https://snapshots.mitmproxy.org ## PyPi -- Upload wheel to pypi: `twine upload <mitmproxy-...-.whl` +- `tox -e rtool -- upload-release` + +## Homebrew +- `tox -e rtool -- homebrew-pr` +- The Homebrew maintainers are typically very fast and detect our new relese within a day, but we can be a nice citizen and create the PR ourself. ## Docker - Update docker-releases repo @@ -28,6 +32,13 @@ Make sure run all these steps on the correct branch you want to create a new rel * `2.0.0` for new major versions * `2.0.2` for new patch versions - Update `latest` tag [here](https://hub.docker.com/r/mitmproxy/mitmproxy/~/settings/automated-builds/) +- Check that the build for this tag succeeds [https://hub.docker.com/r/mitmproxy/mitmproxy/builds/](here) +- If build failed: + - Fix it and commit + - `git tag 3.0.2` the new commit + - `git push origin :refs/tags/3.0.2` to delete the old remote tag + - `git push --tags` to push the new tag + - Check the build details page again ## Prepare for next release diff --git a/release/rtool.py b/release/rtool.py index 9050107e..25f37e61 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -3,6 +3,7 @@ import contextlib import fnmatch import os +import sys import platform import re import runpy @@ -286,6 +287,24 @@ def upload_release(username, password, repository): ]) +@cli.command("homebrew-pr") +def homebrew_pr(): + """ + Create a new Homebrew PR + """ + if platform.system() != "Darwin": + print("You need to run this on macOS to create a new Homebrew PR. Sorry.") + sys.exit(1) + + print("Creating a new PR with Homebrew...") + subprocess.check_call([ + "brew", + "bump-formula-pr", + "--url", "https://github.com/mitmproxy/mitmproxy/archive/v{}".format(get_version()), + "mitmproxy", + ]) + + @cli.command("upload-snapshot") @click.option("--host", envvar="SNAPSHOT_HOST", prompt=True) @click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22) @@ -66,7 +66,6 @@ setup( "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! "click>=6.2, <7", "cryptography>=2.1.4,<2.2", - 'h11>=0.7.0,<0.8', "h2>=3.0.1,<4", "hyperframe>=5.1.0,<6", "kaitaistruct>=0.7,<0.9", @@ -76,7 +75,6 @@ setup( "pyOpenSSL>=17.5,<17.6", "pyparsing>=2.1.3, <2.3", "pyperclip>=1.6.0, <1.7", - "requests>=2.9.1, <3", "ruamel.yaml>=0.13.2, <0.16", "sortedcontainers>=1.5.4, <1.6", "tornado>=4.3, <4.6", @@ -96,12 +94,12 @@ setup( "pytest-timeout>=1.2.1,<2", "pytest-xdist>=1.22,<2", "pytest>=3.3,<4", + "requests>=2.9.1, <3", "tox>=2.3, <3", "rstcheck>=2.2, <4.0", ], 'examples': [ - "beautifulsoup4>=4.4.1, <4.7", - "Pillow>=4.3,<5.1", + "beautifulsoup4>=4.4.1, <4.7" ] } ) diff --git a/test/examples/test_xss_scanner.py b/test/examples/test_xss_scanner.py index 610bdd72..1d723d53 100644 --- a/test/examples/test_xss_scanner.py +++ b/test/examples/test_xss_scanner.py @@ -310,6 +310,9 @@ class TestXSSScanner(): def __init__(self): self.args = [] + def info(self, str): + self.args.append(str) + def error(self, str): self.args.append(str) return Logger() diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index ead6b7e7..228bacf8 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -15,7 +15,7 @@ from mitmproxy import http def test_configure(): d = dumper.Dumper() with taddons.context(d) as ctx: - ctx.configure(d, view_filter="~b foo") + ctx.configure(d, dumper_filter="~b foo") assert d.filter f = tflow.tflow(resp=True) @@ -23,10 +23,10 @@ def test_configure(): f.response.content = b"foo" assert d.match(f) - ctx.configure(d, view_filter=None) + ctx.configure(d, dumper_filter=None) assert not d.filter with pytest.raises(exceptions.OptionsError): - ctx.configure(d, view_filter="~~") + ctx.configure(d, dumper_filter="~~") assert not d.filter @@ -147,7 +147,7 @@ class TestContentView: sio = io.StringIO() d = dumper.Dumper(sio) with taddons.context(d) as ctx: - ctx.configure(d, flow_detail=4, verbosity='debug') + ctx.configure(d, flow_detail=4) d.response(tflow.tflow()) assert ctx.master.has_log("content viewer failed") diff --git a/test/mitmproxy/addons/test_termlog.py b/test/mitmproxy/addons/test_termlog.py index 027bdfeb..6c95df0c 100644 --- a/test/mitmproxy/addons/test_termlog.py +++ b/test/mitmproxy/addons/test_termlog.py @@ -3,7 +3,6 @@ import pytest from mitmproxy.addons import termlog from mitmproxy import log -from mitmproxy.options import Options from mitmproxy.test import taddons @@ -16,7 +15,8 @@ class TestTermLog: ]) def test_output(self, outfile, expected_out, expected_err, capfd): t = termlog.TermLog(outfile=outfile) - with taddons.context(options=Options(verbosity='info')) as tctx: + with taddons.context(t) as tctx: + tctx.options.termlog_verbosity = "info" tctx.configure(t) t.log(log.LogEntry("one", "info")) t.log(log.LogEntry("two", "debug")) diff --git a/test/mitmproxy/tools/console/test_master.py b/test/mitmproxy/tools/console/test_master.py index 5be035e8..2879170d 100644 --- a/test/mitmproxy/tools/console/test_master.py +++ b/test/mitmproxy/tools/console/test_master.py @@ -7,8 +7,6 @@ from ... import tservers class TestMaster(tservers.MasterTest): def mkmaster(self, **opts): - if "verbosity" not in opts: - opts["verbosity"] = 'warn' o = options.Options(**opts) m = console.master.ConsoleMaster(o) m.addons.trigger("configure", o.keys()) diff --git a/test/mitmproxy/tools/test_dump.py b/test/mitmproxy/tools/test_dump.py index f303c808..f950d719 100644 --- a/test/mitmproxy/tools/test_dump.py +++ b/test/mitmproxy/tools/test_dump.py @@ -10,13 +10,13 @@ from .. import tservers class TestDumpMaster(tservers.MasterTest): - def mkmaster(self, flt, **opts): - o = options.Options(view_filter=flt, verbosity='error', **opts) + def mkmaster(self, **opts): + o = options.Options(**opts) m = dump.DumpMaster(o, with_termlog=False, with_dumper=False) return m def test_has_error(self): - m = self.mkmaster(None) + m = self.mkmaster() ent = log.LogEntry("foo", "error") ent.reply = controller.DummyReply() m.addons.trigger("log", ent) |
