diff options
109 files changed, 715 insertions, 724 deletions
@@ -13,6 +13,7 @@ MANIFEST build/ dist/ mitmproxy/contrib/kaitaistruct/*.ksy +.pytest_cache # UI @@ -221,11 +221,11 @@ * All mitmproxy tools are now Python 3 only! We plan to support Python 3.5 and higher. - * Web-Based User Interface: Mitmproxy now offically has a web-based user interface + * Web-Based User Interface: Mitmproxy now officially has a web-based user interface called mitmweb. We consider it stable for all features currently exposed in the UI, but it still misses a lot of mitmproxy’s options. - * Windows Compatibility: With mitmweb, mitmproxy is now useable on Windows. + * Windows Compatibility: With mitmweb, mitmproxy is now usable on Windows. We are also introducing an installer (kindly sponsored by BitRock) that simplifies setup. @@ -387,7 +387,7 @@ * libmproxy: Avoid double-connect in case of TLS Server Name Indication. This yields a massive speedup for TLS handshakes. - * libmproxy: Prevent unneccessary upstream connections (macmantrl) + * libmproxy: Prevent unnecessary upstream connections (macmantrl) * Inline Scripts: New API for HTTP Headers: http://docs.mitmproxy.org/en/latest/dev/models.html#netlib.http.Headers @@ -650,7 +650,7 @@ JSON, Javascript, images, XML, URL-encoded forms, as well as hexadecimal and raw views. - * Add Set Headers, analagous to replacement hooks. Defines headers that are set + * Add Set Headers, analogous to replacement hooks. Defines headers that are set on flows, based on a matching pattern. * A graphical editor for path components in mitmproxy. @@ -708,7 +708,7 @@ expanding random and generated portions, and logging a reproducible specification. - * Streamline the specification langauge. HTTP response message is now + * Streamline the specification language. HTTP response message is now specified using the "r" mnemonic. * Add a "u" mnemonic for specifying User-Agent strings. Add a set of @@ -26,7 +26,7 @@ and pathod websites. |mitmproxy_site| -The latest documentation for mitmproxy is also available on ReadTheDocs. +The latest documentation for mitmproxy is available on our website. |mitmproxy_docs| @@ -95,6 +95,12 @@ requirements installed, and you can run the full test suite (including tests for tox +To run complete tests with a full coverage report, you can use the following command: + +.. code-block:: bash + + tox -- --verbose --cov-report=term + For speedier testing, we recommend you run `pytest`_ directly on individual test files or folders: .. code-block:: bash @@ -145,7 +151,7 @@ with the following command: :alt: mitmproxy.org .. |mitmproxy_docs| image:: https://shields.mitmproxy.org/api/docs-latest-brightgreen.svg - :target: http://docs.mitmproxy.org/en/latest/ + :target: https://mitmproxy.org/docs/latest/ :alt: mitmproxy documentation .. |mitmproxy_discourse| image:: https://shields.mitmproxy.org/api/https%3A%2F%2F-discourse.mitmproxy.org-orange.svg diff --git a/docs/raw/proxy-modes.vsdx b/docs/raw/proxy-modes.vsdx Binary files differindex 0128a142..d115ea9e 100644..100755 --- a/docs/raw/proxy-modes.vsdx +++ b/docs/raw/proxy-modes.vsdx diff --git a/docs/src/content/concepts-modes.md b/docs/src/content/concepts-modes.md index 86bb7b0f..7a0b835a 100644 --- a/docs/src/content/concepts-modes.md +++ b/docs/src/content/concepts-modes.md @@ -157,20 +157,20 @@ There are various use-cases: example.com domain and get all requests recorded in mitmproxy. - Say you have some toy project that should get SSL support. Simply set up mitmproxy as a reverse proxy on port 443 and you're done (`mitmdump -p 443 - -R http://localhost:80/`). Mitmproxy auto-detects TLS traffic and intercepts + --mode reverse:http://localhost:80/`). Mitmproxy auto-detects TLS traffic and intercepts it dynamically. There are better tools for this specific task, but mitmproxy is very quick and simple way to set up an SSL-speaking server. - Want to add a non-SSL-capable compression proxy in front of your server? You - could even spawn a mitmproxy instance that terminates SSL (`-R http://...`), + could even spawn a mitmproxy instance that terminates SSL (`--mode reverse:http://...`), point it to the compression proxy and let the compression proxy point to a - SSL-initiating mitmproxy (`-R https://...`), which then points to the real + SSL-initiating mitmproxy (`--mode reverse:https://...`), which then points to the real server. As you see, it's a fairly flexible thing. ### Host Header In reverse proxy mode, mitmproxy automatically rewrites the Host header to match the upstream server. This allows mitmproxy to easily connect to existing -endpoints on the open web (e.g. `mitmproxy -R https://example.com`). You can +endpoints on the open web (e.g. `mitmproxy --mode reverse:https://example.com`). You can disable this behaviour with the `keep_host_header` option. However, keep in mind that absolute URLs within the returned document or HTTP diff --git a/docs/src/content/howto-transparent-vms.md b/docs/src/content/howto-transparent-vms.md index b186fd39..1446ede7 100644 --- a/docs/src/content/howto-transparent-vms.md +++ b/docs/src/content/howto-transparent-vms.md @@ -106,7 +106,7 @@ sudo iptables -t nat -A PREROUTING -i eth1 -p tcp --dport 443 -j REDIRECT --to-p Finally, we can run mitmproxy in transparent mode with {{< highlight bash >}} -mitmproxy -T +mitmproxy --mode transparent {{< / highlight >}} The proxied machine cannot to leak any data outside of HTTP or DNS requests. If diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index 224cb5ee..3d99e9dc 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -58,7 +58,7 @@ dropped privileges. It can be used as follows: gcc examples/complex/full_transparency_shim.c -o mitmproxy_shim -lcap sudo chown root:root mitmproxy_shim sudo chmod u+s mitmproxy_shim -./mitmproxy_shim $(which mitmproxy) -T --spoof-source-address +./mitmproxy_shim $(which mitmproxy) --mode transparent --set spoof-source-address {{< / highlight >}} @@ -112,10 +112,10 @@ something like this: You probably want a command like this: {{< highlight bash >}} -mitmproxy -T --host +mitmproxy --mode transparent --showhost {{< / highlight >}} -The `-T` flag turns on transparent mode, and the `--host` argument tells +The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells mitmproxy to use the value of the Host header for URL display. ### 6. Finally, configure your test device @@ -163,10 +163,10 @@ doas pfctl -e You probably want a command like this: {{< highlight bash >}} -mitmproxy -T --host +mitmproxy --mode transparent --showhost {{< / highlight >}} -The `-T` flag turns on transparent mode, and the `--host` argument tells +The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells mitmproxy to use the value of the Host header for URL display. ### 6. Finally, configure your test device @@ -245,10 +245,10 @@ tighten the restriction up to the user running mitmproxy. You probably want a command like this: {{< highlight bash >}} -mitmproxy -T --host +mitmproxy --mode transparent --showhost {{< / highlight >}} -The `-T` flag turns on transparent mode, and the `--host` argument tells +The `--mode transparent` flag turns on transparent mode, and the `--showhost` argument tells mitmproxy to use the value of the Host header for URL display. ### 6. Finally, configure your test device diff --git a/docs/src/content/overview-tools.md b/docs/src/content/overview-tools.md index 7612383a..0200e899 100644 --- a/docs/src/content/overview-tools.md +++ b/docs/src/content/overview-tools.md @@ -8,7 +8,7 @@ menu: # Overview -You should thin of the mitmproxy project's tools as a set of front-ends that +You should think of the mitmproxy project's tools as a set of front-ends that expose the same underlying functionality. We aim to have feature parity across all of our tooling, and all tools share a common configuration mechanism and most command-line options. diff --git a/docs/src/static/schematics/proxy-modes-reverse.png b/docs/src/static/schematics/proxy-modes-reverse.png Binary files differindex 071d3fc8..87372688 100644..100755 --- a/docs/src/static/schematics/proxy-modes-reverse.png +++ b/docs/src/static/schematics/proxy-modes-reverse.png diff --git a/docs/src/static/schematics/proxy-modes-transparent-1.png b/docs/src/static/schematics/proxy-modes-transparent-1.png Binary files differindex 002e0e76..557406f8 100644..100755 --- a/docs/src/static/schematics/proxy-modes-transparent-1.png +++ b/docs/src/static/schematics/proxy-modes-transparent-1.png diff --git a/docs/src/static/schematics/proxy-modes-transparent-2.png b/docs/src/static/schematics/proxy-modes-transparent-2.png Binary files differindex 41997b05..ecc09f5a 100644..100755 --- a/docs/src/static/schematics/proxy-modes-transparent-2.png +++ b/docs/src/static/schematics/proxy-modes-transparent-2.png diff --git a/docs/src/static/schematics/proxy-modes-transparent-3.png b/docs/src/static/schematics/proxy-modes-transparent-3.png Binary files differindex ee26cb4f..177aaaef 100644..100755 --- a/docs/src/static/schematics/proxy-modes-transparent-3.png +++ b/docs/src/static/schematics/proxy-modes-transparent-3.png diff --git a/docs/src/static/schematics/proxy-modes-transparent-wrong.png b/docs/src/static/schematics/proxy-modes-transparent-wrong.png Binary files differindex ca501e93..c69c6b70 100644..100755 --- a/docs/src/static/schematics/proxy-modes-transparent-wrong.png +++ b/docs/src/static/schematics/proxy-modes-transparent-wrong.png diff --git a/docs/src/static/schematics/proxy-modes-upstream.png b/docs/src/static/schematics/proxy-modes-upstream.png Binary files differindex d40a6494..daa2b9af 100644..100755 --- a/docs/src/static/schematics/proxy-modes-upstream.png +++ b/docs/src/static/schematics/proxy-modes-upstream.png diff --git a/examples/addons/events.py b/examples/addons/events.py index 93664954..d3c90430 100644 --- a/examples/addons/events.py +++ b/examples/addons/events.py @@ -155,7 +155,7 @@ class Events: def log(self, entry: mitmproxy.log.LogEntry): """ - Called whenver a new log entry is created through the mitmproxy + Called whenever a new log entry is created through the mitmproxy context. Be careful not to log from this event, which will cause an infinite loop! """ diff --git a/examples/complex/README.md b/examples/complex/README.md index 77dbe2f5..c53503e4 100644 --- a/examples/complex/README.md +++ b/examples/complex/README.md @@ -10,7 +10,7 @@ | mitmproxywrapper.py | Bracket mitmproxy run with proxy enable/disable on OS X | | nonblocking.py | Demonstrate parallel processing with a blocking script | | remote_debug.py | This script enables remote debugging of the mitmproxy _UI_ with PyCharm. | -| sslstrip.py | sslstrip-like funtionality implemented with mitmproxy | +| sslstrip.py | sslstrip-like functionality implemented with mitmproxy | | stream.py | Enable streaming for all responses. | | stream_modify.py | Modify a streamed response body. | | tcp_message.py | Modify a raw TCP connection | diff --git a/examples/complex/stream_modify.py b/examples/complex/stream_modify.py index 5e5da95b..46bdcb78 100644 --- a/examples/complex/stream_modify.py +++ b/examples/complex/stream_modify.py @@ -3,7 +3,7 @@ This inline script modifies a streamed response. If you do not need streaming, see the modify_response_body example. Be aware that content replacement isn't trivial: - If the transfer encoding isn't chunked, you cannot simply change the content length. - - If you want to replace all occurences of "foobar", make sure to catch the cases + - If you want to replace all occurrences of "foobar", make sure to catch the cases where one chunk ends with [...]foo" and the next starts with "bar[...]. """ diff --git a/examples/complex/xss_scanner.py b/examples/complex/xss_scanner.py index 0ee38cd4..0c0dd0f3 100755 --- a/examples/complex/xss_scanner.py +++ b/examples/complex/xss_scanner.py @@ -215,7 +215,7 @@ def get_SQLi_data(new_body: str, original_body: str, request_URL: str, injection # A qc is either ' or " def inside_quote(qc: str, substring_bytes: bytes, text_index: int, body_bytes: bytes) -> bool: - """ Whether the Numberth occurence of the first string in the second + """ Whether the Numberth occurrence of the first string in the second string is inside quotes as defined by the supplied QuoteChar """ substring = substring_bytes.decode('utf-8') body = body_bytes.decode('utf-8') @@ -246,7 +246,7 @@ def paths_to_text(html: str, string: str) -> List[str]: - Note that it does a BFS """ def remove_last_occurence_of_sub_string(string: str, substr: str) -> str: - """ Delete the last occurence of substr from str + """ Delete the last occurrence of substr from str String String -> String """ index = string.rfind(substr) @@ -274,7 +274,7 @@ def paths_to_text(html: str, string: str) -> List[str]: def get_XSS_data(body: Union[str, bytes], request_URL: str, injection_point: str) -> Optional[XSSData]: """ Return a XSSDict if there is a XSS otherwise return None """ def in_script(text, index, body) -> bool: - """ Whether the Numberth occurence of the first string in the second + """ Whether the Numberth occurrence of the first string in the second string is inside a script tag """ paths = paths_to_text(body.decode('utf-8'), text.decode("utf-8")) try: @@ -284,7 +284,7 @@ def get_XSS_data(body: Union[str, bytes], request_URL: str, injection_point: str return False def in_HTML(text: bytes, index: int, body: bytes) -> bool: - """ Whether the Numberth occurence of the first string in the second + """ Whether the Numberth occurrence of the first string in the second string is inside the HTML but not inside a script tag or part of a HTML attribute""" # if there is a < then lxml will interpret that as a tag, so only search for the stuff before it diff --git a/examples/simple/internet_in_mirror.py b/examples/simple/internet_in_mirror.py new file mode 100644 index 00000000..5d3e555d --- /dev/null +++ b/examples/simple/internet_in_mirror.py @@ -0,0 +1,9 @@ +""" +This script reflects all content passing through the proxy. +""" +from mitmproxy import http + + +def response(flow: http.HTTPFlow) -> None: + reflector = b"<style>body {transform: scaleX(-1);}</style></head>" + flow.response.content = flow.response.content.replace(b"</head>", reflector) diff --git a/examples/simple/upsidedownternet.py b/examples/simple/upsidedownternet.py deleted file mode 100644 index f150a5c3..00000000 --- a/examples/simple/upsidedownternet.py +++ /dev/null @@ -1,16 +0,0 @@ -""" -This script rotates all images passing through the proxy by 180 degrees. -""" -import io -from PIL import Image -from mitmproxy import http - - -def response(flow: http.HTTPFlow) -> None: - if flow.response.headers.get("content-type", "").startswith("image"): - s = io.BytesIO(flow.response.content) - img = Image.open(s).rotate(180) - s2 = io.BytesIO() - img.save(s2, "png") - flow.response.content = s2.getvalue() - flow.response.headers["content-type"] = "image/png" diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py index 37c501ee..bfaacf6d 100644 --- a/mitmproxy/addonmanager.py +++ b/mitmproxy/addonmanager.py @@ -92,6 +92,13 @@ class Loader: help: str, choices: typing.Optional[typing.Sequence[str]] = None ) -> None: + """ + Add an option to mitmproxy. + + Help should be a single paragraph with no linebreaks - it will be + reflowed by tools. Information on the data type should be omitted - + it will be generated and added by tools as needed. + """ if name in self.master.options: existing = self.master.options._options[name] same_signature = ( diff --git a/mitmproxy/addons/__init__.py b/mitmproxy/addons/__init__.py index 8f84c20d..988bc904 100644 --- a/mitmproxy/addons/__init__.py +++ b/mitmproxy/addons/__init__.py @@ -4,7 +4,6 @@ from mitmproxy.addons import anticomp from mitmproxy.addons import browser from mitmproxy.addons import check_ca from mitmproxy.addons import clientplayback -from mitmproxy.addons import core_option_validation from mitmproxy.addons import core from mitmproxy.addons import cut from mitmproxy.addons import disable_h2c @@ -25,7 +24,6 @@ from mitmproxy.addons import upstream_auth def default_addons(): return [ core.Core(), - core_option_validation.CoreOptionValidation(), browser.Browser(), allowremote.AllowRemote(), anticache.AntiCache(), diff --git a/mitmproxy/addons/anticache.py b/mitmproxy/addons/anticache.py index 5b34d5a5..9f5c2dc1 100644 --- a/mitmproxy/addons/anticache.py +++ b/mitmproxy/addons/anticache.py @@ -2,6 +2,15 @@ from mitmproxy import ctx class AntiCache: + def load(self, loader): + loader.add_option( + "anticache", bool, False, + """ + Strip out request headers that might cause the server to return + 304-not-modified. + """ + ) + def request(self, flow): if ctx.options.anticache: flow.request.anticache() diff --git a/mitmproxy/addons/anticomp.py b/mitmproxy/addons/anticomp.py index d7d1ca8d..3415302a 100644 --- a/mitmproxy/addons/anticomp.py +++ b/mitmproxy/addons/anticomp.py @@ -2,6 +2,12 @@ from mitmproxy import ctx class AntiComp: + def load(self, loader): + loader.add_option( + "anticomp", bool, False, + "Try to convince servers to send us un-compressed data." + ) + def request(self, flow): if ctx.options.anticomp: flow.request.anticomp() diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index 2dd488b9..a017ec0f 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -14,6 +14,12 @@ class ClientPlayback: self.current_thread = None self.configured = False + def load(self, loader): + loader.add_option( + "client_replay", typing.Sequence[str], [], + "Replay client requests from a saved file." + ) + def count(self) -> int: if self.current_thread: current = 1 diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index ca21e1dc..6edac6c3 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -1,15 +1,72 @@ import typing +from mitmproxy.utils import human from mitmproxy import ctx from mitmproxy import exceptions from mitmproxy import command from mitmproxy import flow from mitmproxy import optmanager +from mitmproxy import platform +from mitmproxy.net import server_spec from mitmproxy.net.http import status_codes import mitmproxy.types +CA_DIR = "~/.mitmproxy" +LISTEN_PORT = 8080 + + class Core: + def load(self, loader): + loader.add_option( + "body_size_limit", typing.Optional[str], None, + """ + Byte size limit of HTTP request and response bodies. Understands + k/m/g suffixes, i.e. 3m for 3 megabytes. + """ + ) + loader.add_option( + "keep_host_header", bool, False, + """ + Reverse Proxy: Keep the original host header instead of rewriting it + to the reverse proxy target. + """ + ) + + def configure(self, updated): + opts = ctx.options + if opts.add_upstream_certs_to_client_chain and not opts.upstream_cert: + raise exceptions.OptionsError( + "The no-upstream-cert and add-upstream-certs-to-client-chain " + "options are mutually exclusive. If no-upstream-cert is enabled " + "then the upstream certificate is not retrieved before generating " + "the client certificate chain." + ) + if "body_size_limit" in updated: + try: + human.parse_size(opts.body_size_limit) + except ValueError as e: + raise exceptions.OptionsError( + "Invalid body size limit specification: %s" % + opts.body_size_limit + ) + if "mode" in updated: + mode = opts.mode + if mode.startswith("reverse:") or mode.startswith("upstream:"): + try: + server_spec.parse_with_mode(mode) + except ValueError as e: + raise exceptions.OptionsError(str(e)) from e + elif mode == "transparent": + if not platform.original_addr: + raise exceptions.OptionsError( + "Transparent mode not supported on this platform." + ) + elif mode not in ["regular", "socks5"]: + raise exceptions.OptionsError( + "Invalid mode specification: %s" % mode + ) + @command.command("set") def set(self, *spec: str) -> None: """ diff --git a/mitmproxy/addons/core_option_validation.py b/mitmproxy/addons/core_option_validation.py deleted file mode 100644 index 42da0b74..00000000 --- a/mitmproxy/addons/core_option_validation.py +++ /dev/null @@ -1,45 +0,0 @@ -""" - The core addon is responsible for verifying core settings that are not - checked by other addons. -""" -from mitmproxy import exceptions -from mitmproxy import platform -from mitmproxy import ctx -from mitmproxy.net import server_spec -from mitmproxy.utils import human - - -class CoreOptionValidation: - def configure(self, updated): - opts = ctx.options - if opts.add_upstream_certs_to_client_chain and not opts.upstream_cert: - raise exceptions.OptionsError( - "The no-upstream-cert and add-upstream-certs-to-client-chain " - "options are mutually exclusive. If no-upstream-cert is enabled " - "then the upstream certificate is not retrieved before generating " - "the client certificate chain." - ) - if "body_size_limit" in updated: - try: - human.parse_size(opts.body_size_limit) - except ValueError as e: - raise exceptions.OptionsError( - "Invalid body size limit specification: %s" % - opts.body_size_limit - ) - if "mode" in updated: - mode = opts.mode - if mode.startswith("reverse:") or mode.startswith("upstream:"): - try: - server_spec.parse_with_mode(mode) - except ValueError as e: - raise exceptions.OptionsError(str(e)) from e - elif mode == "transparent": - if not platform.original_addr: - raise exceptions.OptionsError( - "Transparent mode not supported on this platform." - ) - elif mode not in ["regular", "socks5"]: - raise exceptions.OptionsError( - "Invalid mode specification: %s" % mode - ) diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py index 1c8fbc05..f9874038 100644 --- a/mitmproxy/addons/cut.py +++ b/mitmproxy/addons/cut.py @@ -61,7 +61,7 @@ class Cut: from the base of the flow object, with a few conveniences - "port" and "host" retrieve parts of an address tuple, ".header[key]" retrieves a header value. Return values converted to strings or - bytes: SSL certicates are converted to PEM format, bools are "true" + bytes: SSL certificates are converted to PEM format, bools are "true" or "false", "bytes" are preserved, and all other values are converted to strings. """ diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index 48bc8118..aaad8aa2 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -31,13 +31,34 @@ class Dumper: self.filter = None # type: flowfilter.TFilter self.outfp = outfile # type: typing.io.TextIO + def load(self, loader): + loader.add_option( + "flow_detail", int, 1, + """ + The display detail level for flows in mitmdump: 0 (almost quiet) to 3 (very verbose). + 0: shortened request URL, response status code, WebSocket and TCP message notifications. + 1: full request URL with response status code + 2: 1 + HTTP headers + 3: 2 + full response content, content of WebSocket and TCP messages. + """ + ) + loader.add_option( + "dumper_default_contentview", str, "auto", + "The default content view mode.", + choices = [i.name.lower() for i in contentviews.views] + ) + loader.add_option( + "dumper_filter", typing.Optional[str], None, + "Limit which flows are dumped." + ) + def configure(self, updated): - if "view_filter" in updated: - if ctx.options.view_filter: - self.filter = flowfilter.parse(ctx.options.view_filter) + if "dumper_filter" in updated: + if ctx.options.dumper_filter: + self.filter = flowfilter.parse(ctx.options.dumper_filter) if not self.filter: raise exceptions.OptionsError( - "Invalid filter expression: %s" % ctx.options.view_filter + "Invalid filter expression: %s" % ctx.options.dumper_filter ) else: self.filter = None @@ -61,7 +82,7 @@ class Dumper: def _echo_message(self, message): _, lines, error = contentviews.get_message_content_view( - ctx.options.default_contentview, + ctx.options.dumper_default_contentview, message ) if error: diff --git a/mitmproxy/addons/intercept.py b/mitmproxy/addons/intercept.py index 9e1a283e..d39d1962 100644 --- a/mitmproxy/addons/intercept.py +++ b/mitmproxy/addons/intercept.py @@ -1,3 +1,5 @@ +import typing + from mitmproxy import flowfilter from mitmproxy import exceptions from mitmproxy import ctx @@ -7,6 +9,17 @@ class Intercept: def __init__(self): self.filt = None + def load(self, loader): + loader.add_option( + "intercept_active", bool, False, + "Intercept toggle" + ) + + loader.add_option( + "intercept", typing.Optional[str], None, + "Intercept filter expression." + ) + def configure(self, updated): if "intercept" in updated: if not ctx.options.intercept: diff --git a/mitmproxy/addons/keepserving.py b/mitmproxy/addons/keepserving.py index 9c975a7b..6413299d 100644 --- a/mitmproxy/addons/keepserving.py +++ b/mitmproxy/addons/keepserving.py @@ -2,6 +2,16 @@ from mitmproxy import ctx class KeepServing: + def load(self, loader): + loader.add_option( + "keepserving", bool, False, + """ + Continue serving after client playback, server playback or file + read. This option is ignored by interactive tools, which always keep + serving. + """ + ) + def event_processing_complete(self): if not ctx.master.options.keepserving: ctx.master.shutdown() diff --git a/mitmproxy/addons/onboarding.py b/mitmproxy/addons/onboarding.py index 07536c34..900acb08 100644 --- a/mitmproxy/addons/onboarding.py +++ b/mitmproxy/addons/onboarding.py @@ -2,6 +2,9 @@ from mitmproxy.addons import wsgiapp from mitmproxy.addons.onboardingapp import app from mitmproxy import ctx +APP_HOST = "mitm.it" +APP_PORT = 80 + class Onboarding(wsgiapp.WSGIApp): name = "onboarding" @@ -9,6 +12,23 @@ class Onboarding(wsgiapp.WSGIApp): def __init__(self): super().__init__(app.Adapter(app.application), None, None) + def load(self, loader): + loader.add_option( + "onboarding", bool, True, + "Toggle the mitmproxy onboarding app." + ) + loader.add_option( + "onboarding_host", str, APP_HOST, + """ + Onboarding app domain. For transparent mode, use an IP when a DNS + entry for the app domain is not present. + """ + ) + loader.add_option( + "onboarding_port", int, APP_PORT, + "Port to serve the onboarding app from." + ) + def configure(self, updated): self.host = ctx.options.onboarding_host self.port = ctx.options.onboarding_port diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py index dc99d5cc..37d7d93c 100644 --- a/mitmproxy/addons/proxyauth.py +++ b/mitmproxy/addons/proxyauth.py @@ -52,6 +52,18 @@ class ProxyAuth: self.authenticated = weakref.WeakKeyDictionary() # type: MutableMapping[connections.ClientConnection, Tuple[str, str]] """Contains all connections that are permanently authenticated after an HTTP CONNECT""" + def load(self, loader): + loader.add_option( + "proxyauth", Optional[str], None, + """ + Require proxy authentication. Format: + "username:pass", + "any" to accept any user/pass combination, + "@path" to use an Apache htpasswd file, + or "ldap[s]:url_server_ldap:dn_auth:password:dn_subtree" for LDAP authentication. + """ + ) + def enabled(self) -> bool: return any([self.nonanonymous, self.htpasswd, self.singleuser, self.ldapconn, self.ldapserver]) @@ -160,7 +172,7 @@ class ProxyAuth: server = ldap3.Server(ldap_server) else: raise exceptions.OptionsError( - "Invalid ldap specfication on the first part" + "Invalid ldap specification on the first part" ) conn = ldap3.Connection( server, diff --git a/mitmproxy/addons/readfile.py b/mitmproxy/addons/readfile.py index 05b6c309..aaf02d01 100644 --- a/mitmproxy/addons/readfile.py +++ b/mitmproxy/addons/readfile.py @@ -11,6 +11,11 @@ class ReadFile: """ An addon that handles reading from file on startup. """ + def load(self, loader): + loader.add_option( + "rfile", typing.Optional[str], None, + "Read flows from file." + ) def load_flows(self, fo: typing.IO[bytes]) -> int: cnt = 0 diff --git a/mitmproxy/addons/replace.py b/mitmproxy/addons/replace.py index 054264fa..5173254a 100644 --- a/mitmproxy/addons/replace.py +++ b/mitmproxy/addons/replace.py @@ -1,5 +1,6 @@ import os import re +import typing from mitmproxy import exceptions from mitmproxy import flowfilter @@ -47,6 +48,15 @@ class Replace: def __init__(self): self.lst = [] + def load(self, loader): + loader.add_option( + "replacements", typing.Sequence[str], [], + """ + Replacement patterns of the form "/pattern/regex/replacement", where + the separator can be any character. + """ + ) + def configure(self, updated): """ .replacements is a list of tuples (fpat, rex, s): diff --git a/mitmproxy/addons/save.py b/mitmproxy/addons/save.py index 44afef68..e6e98ff8 100644 --- a/mitmproxy/addons/save.py +++ b/mitmproxy/addons/save.py @@ -16,6 +16,16 @@ class Save: self.filt = None self.active_flows = set() # type: Set[flow.Flow] + def load(self, loader): + loader.add_option( + "save_stream_file", typing.Optional[str], None, + "Stream flows to file as they arrive. Prefix path with + to append." + ) + loader.add_option( + "save_stream_filter", typing.Optional[str], None, + "Filter which flows are written to file." + ) + def open_file(self, path): if path.startswith("+"): path = path[1:] diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py index 0a524359..dcad943a 100644 --- a/mitmproxy/addons/script.py +++ b/mitmproxy/addons/script.py @@ -98,6 +98,14 @@ class ScriptLoader: self.is_running = False self.addons = [] + def load(self, loader): + loader.add_option( + "scripts", typing.Sequence[str], [], + """ + Execute a script. + """ + ) + def running(self): self.is_running = True diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py index d8b2299a..73fb1666 100644 --- a/mitmproxy/addons/serverplayback.py +++ b/mitmproxy/addons/serverplayback.py @@ -1,8 +1,6 @@ import hashlib import urllib import typing -from typing import Any # noqa -from typing import List # noqa from mitmproxy import ctx from mitmproxy import flow @@ -19,6 +17,60 @@ class ServerPlayback: self.final_flow = None self.configured = False + def load(self, loader): + loader.add_option( + "server_replay_kill_extra", bool, False, + "Kill extra requests during replay." + ) + loader.add_option( + "server_replay_nopop", bool, False, + """ + Don't remove flows from server replay state after use. This makes it + possible to replay same response multiple times. + """ + ) + loader.add_option( + "server_replay_refresh", bool, True, + """ + Refresh server replay responses by adjusting date, expires and + last-modified headers, as well as adjusting cookie expiration. + """ + ) + loader.add_option( + "server_replay_use_headers", typing.Sequence[str], [], + "Request headers to be considered during replay." + ) + loader.add_option( + "server_replay", typing.Sequence[str], [], + "Replay server responses from a saved file." + ) + loader.add_option( + "server_replay_ignore_content", bool, False, + "Ignore request's content while searching for a saved flow to replay." + ) + loader.add_option( + "server_replay_ignore_params", typing.Sequence[str], [], + """ + Request's parameters to be ignored while searching for a saved flow + to replay. + """ + ) + loader.add_option( + "server_replay_ignore_payload_params", typing.Sequence[str], [], + """ + Request's payload parameters (application/x-www-form-urlencoded or + multipart/form-data) to be ignored while searching for a saved flow + to replay. + """ + ) + loader.add_option( + "server_replay_ignore_host", bool, False, + """ + Ignore request's destination host while searching for a saved flow + to replay. + """ + ) + @command.command("replay.server") def load_flows(self, flows: typing.Sequence[flow.Flow]) -> None: """ diff --git a/mitmproxy/addons/setheaders.py b/mitmproxy/addons/setheaders.py index d4d16e40..3517f70f 100644 --- a/mitmproxy/addons/setheaders.py +++ b/mitmproxy/addons/setheaders.py @@ -1,3 +1,5 @@ +import typing + from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import ctx @@ -44,6 +46,15 @@ class SetHeaders: def __init__(self): self.lst = [] + def load(self, loader): + loader.add_option( + "setheaders", typing.Sequence[str], [], + """ + Header set pattern of the form "/pattern/header/value", where the + separator can be any character. + """ + ) + def configure(self, updated): if "setheaders" in updated: self.lst = [] diff --git a/mitmproxy/addons/stickyauth.py b/mitmproxy/addons/stickyauth.py index 24831d5b..ab0599ee 100644 --- a/mitmproxy/addons/stickyauth.py +++ b/mitmproxy/addons/stickyauth.py @@ -1,3 +1,5 @@ +import typing + from mitmproxy import exceptions from mitmproxy import flowfilter from mitmproxy import ctx @@ -8,6 +10,12 @@ class StickyAuth: self.flt = None self.hosts = {} + def load(self, loader): + loader.add_option( + "stickyauth", typing.Optional[str], None, + "Set sticky auth filter. Matched against requests." + ) + def configure(self, updated): if "stickyauth" in updated: if ctx.options.stickyauth: diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py index e58e0a58..e5f85fa1 100644 --- a/mitmproxy/addons/stickycookie.py +++ b/mitmproxy/addons/stickycookie.py @@ -34,6 +34,12 @@ class StickyCookie: self.jar = collections.defaultdict(dict) # type: Dict[TOrigin, Dict[str, str]] self.flt = None # type: Optional[flowfilter.TFilter] + def load(self, loader): + loader.add_option( + "stickycookie", Optional[str], None, + "Set sticky cookie filter. Matched against requests." + ) + def configure(self, updated): if "stickycookie" in updated: if ctx.options.stickycookie: diff --git a/mitmproxy/addons/streambodies.py b/mitmproxy/addons/streambodies.py index c841075f..6ca9918f 100644 --- a/mitmproxy/addons/streambodies.py +++ b/mitmproxy/addons/streambodies.py @@ -1,3 +1,5 @@ +import typing + from mitmproxy.net.http import http1 from mitmproxy import exceptions from mitmproxy import ctx @@ -8,6 +10,23 @@ class StreamBodies: def __init__(self): self.max_size = None + def load(self, loader): + loader.add_option( + "stream_large_bodies", typing.Optional[str], None, + """ + Stream data to the client if response body exceeds the given + threshold. If streamed, the body will not be stored in any way. + Understands k/m/g suffixes, i.e. 3m for 3 megabytes. + """ + ) + loader.add_option( + "stream_websockets", bool, False, + """ + Stream WebSocket messages between client and server. + Messages are captured and cannot be modified. + """ + ) + def configure(self, updated): if "stream_large_bodies" in updated and ctx.options.stream_large_bodies: try: diff --git a/mitmproxy/addons/termlog.py b/mitmproxy/addons/termlog.py index 2a7e2d09..f09aa4b4 100644 --- a/mitmproxy/addons/termlog.py +++ b/mitmproxy/addons/termlog.py @@ -14,13 +14,20 @@ class TermLog: def __init__(self, outfile=None): self.outfile = outfile + def load(self, loader): + loader.add_option( + "termlog_verbosity", str, 'info', + "Log verbosity.", + choices=log.LogTierOrder + ) + def log(self, e): if log.log_tier(e.level) == log.log_tier("error"): outfile = self.outfile or realstderr else: outfile = self.outfile or realstdout - if log.log_tier(ctx.options.verbosity) >= log.log_tier(e.level): + if log.log_tier(ctx.options.termlog_verbosity) >= log.log_tier(e.level): click.secho( e.msg, file=outfile, diff --git a/mitmproxy/addons/upstream_auth.py b/mitmproxy/addons/upstream_auth.py index 685494c2..c0581e27 100644 --- a/mitmproxy/addons/upstream_auth.py +++ b/mitmproxy/addons/upstream_auth.py @@ -1,4 +1,5 @@ import re +import typing import base64 from mitmproxy import exceptions @@ -28,11 +29,20 @@ class UpstreamAuth(): def __init__(self): self.auth = None + def load(self, loader): + loader.add_option( + "upstream_auth", typing.Optional[str], None, + """ + Add HTTP Basic authentication to upstream proxy and reverse proxy + requests. Format: username:password. + """ + ) + def configure(self, updated): # FIXME: We're doing this because our proxy core is terminally confused # at the moment. Ideally, we should be able to check if we're in # reverse proxy mode at the HTTP layer, so that scripts can put the - # proxy in reverse proxy mode for specific reuests. + # proxy in reverse proxy mode for specific requests. if "upstream_auth" in updated: if ctx.options.upstream_auth is None: self.auth = None @@ -47,5 +57,5 @@ class UpstreamAuth(): if self.auth: if f.mode == "upstream" and not f.server_conn.via: f.request.headers["Proxy-Authorization"] = self.auth - elif ctx.options.mode == "reverse": + elif ctx.options.mode.startswith("reverse"): f.request.headers["Proxy-Authorization"] = self.auth diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index e87daf35..598cb66d 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -147,6 +147,10 @@ class View(collections.Sequence): def load(self, loader): loader.add_option( + "view_filter", typing.Optional[str], None, + "Limit the view to matching flows." + ) + loader.add_option( "view_order", str, "time", "Flow sort order.", choices=list(map(lambda c: c[1], orders)), @@ -325,7 +329,7 @@ class View(collections.Sequence): key: str ) -> None: """ - Toggle a boolean value in the settings store, seting the value to + Toggle a boolean value in the settings store, setting the value to the string "true" or "false". """ updated = [] diff --git a/mitmproxy/addons/wsgiapp.py b/mitmproxy/addons/wsgiapp.py index 155444fc..549d8c87 100644 --- a/mitmproxy/addons/wsgiapp.py +++ b/mitmproxy/addons/wsgiapp.py @@ -7,7 +7,7 @@ from mitmproxy import version class WSGIApp: """ - An addon that hosts a WSGI app withing mitproxy, at a specified + An addon that hosts a WSGI app within mitproxy, at a specified hostname and port. """ def __init__(self, app, host, port): diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index 4e10529a..6487b750 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -160,7 +160,7 @@ class CertStore: def load_dhparam(path): # mitmproxy<=0.10 doesn't generate a dhparam file. - # Create it now if neccessary. + # Create it now if necessary. if not os.path.exists(path): with open(path, "wb") as f: f.write(DEFAULT_DHPARAM) diff --git a/mitmproxy/command.py b/mitmproxy/command.py index 48968c90..45141576 100644 --- a/mitmproxy/command.py +++ b/mitmproxy/command.py @@ -54,7 +54,7 @@ class Command: self.has_positional = False for i in sig.parameters.values(): - # This is the kind for *args paramters + # This is the kind for *args parameters if i.kind == i.VAR_POSITIONAL: self.has_positional = True self.paramtypes = [v.annotation for v in sig.parameters.values()] diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py index 9c47985c..e8fc4fbf 100644 --- a/mitmproxy/connections.py +++ b/mitmproxy/connections.py @@ -286,7 +286,8 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): else: path = os.path.join( client_certs, - self.address[0].encode("idna").decode()) + ".pem" + (sni or self.address[0].encode("idna").decode()) + ".pem" + ) if os.path.exists(path): client_cert = path diff --git a/mitmproxy/controller.py b/mitmproxy/controller.py index f39c1b24..beb210ca 100644 --- a/mitmproxy/controller.py +++ b/mitmproxy/controller.py @@ -56,7 +56,7 @@ class Reply: self._state = "start" # "start" -> "taken" -> "committed" - # Holds the reply value. May change before things are actually commited. + # Holds the reply value. May change before things are actually committed. self.value = NO_REPLY @property @@ -66,7 +66,7 @@ class Reply: sequentially through the following lifecycle: 1. start: Initial State. - 2. taken: The reply object has been taken to be commited. + 2. taken: The reply object has been taken to be committed. 3. committed: The reply has been sent back to the requesting party. This attribute is read-only and can only be modified by calling one of @@ -91,7 +91,7 @@ class Reply: def commit(self): """ - Ultimately, messages are commited. This is done either automatically by + Ultimately, messages are committed. This is done either automatically by if the message is not taken or manually by the entity which called .take(). """ diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index 51bd116b..231fa3b8 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -123,7 +123,7 @@ def convert_200_300(data): def convert_300_4(data): data["version"] = 4 - # Ths is an empty migration to transition to the new versioning scheme. + # This is an empty migration to transition to the new versioning scheme. return data diff --git a/mitmproxy/log.py b/mitmproxy/log.py index 3083a000..d2988011 100644 --- a/mitmproxy/log.py +++ b/mitmproxy/log.py @@ -57,5 +57,14 @@ class Log: self.master.add_log(text, level) +LogTierOrder = [ + "error", + "warn", + "info", + "alert", + "debug", +] + + def log_tier(level): return dict(error=0, warn=1, info=2, alert=2, debug=3).get(level) diff --git a/mitmproxy/net/http/cookies.py b/mitmproxy/net/http/cookies.py index 7bef8757..39e8d60f 100644 --- a/mitmproxy/net/http/cookies.py +++ b/mitmproxy/net/http/cookies.py @@ -159,13 +159,17 @@ def _read_set_cookie_pairs(s: str, off=0) -> Tuple[List[TPairs], int]: if len(rhs) <= 3: trail, off = _read_value(s, off + 1, ";,") rhs = rhs + "," + trail - if rhs or lhs: + + # as long as there's a "=", we consider it a pair + pairs.append([lhs, rhs]) + + elif lhs: pairs.append([lhs, rhs]) - # comma marks the beginning of a new cookie - if off < len(s) and s[off] == ",": - cookies.append(pairs) - pairs = [] + # comma marks the beginning of a new cookie + if off < len(s) and s[off] == ",": + cookies.append(pairs) + pairs = [] off += 1 diff --git a/mitmproxy/net/http/http1/read.py b/mitmproxy/net/http/http1/read.py index 0f70b1a7..294e8358 100644 --- a/mitmproxy/net/http/http1/read.py +++ b/mitmproxy/net/http/http1/read.py @@ -43,7 +43,7 @@ def read_request_head(rfile): Raises: exceptions.HttpReadDisconnect: No bytes can be read from rfile. exceptions.HttpSyntaxException: The input is malformed HTTP. - exceptions.HttpException: Any other error occured. + exceptions.HttpException: Any other error occurred. """ timestamp_start = time.time() if hasattr(rfile, "reset_timestamps"): @@ -82,7 +82,7 @@ def read_response_head(rfile): Raises: exceptions.HttpReadDisconnect: No bytes can be read from rfile. exceptions.HttpSyntaxException: The input is malformed HTTP. - exceptions.HttpException: Any other error occured. + exceptions.HttpException: Any other error occurred. """ timestamp_start = time.time() diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 76060548..ce7597a8 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -1,287 +1,28 @@ from typing import Optional, Sequence from mitmproxy import optmanager -from mitmproxy import contentviews from mitmproxy.net import tls -log_verbosity = [ - "error", - "warn", - "info", - "alert", - "debug", -] -APP_HOST = "mitm.it" -APP_PORT = 80 CA_DIR = "~/.mitmproxy" LISTEN_PORT = 8080 -# Some help text style guidelines: -# -# - Should be a single paragraph with no linebreaks. Help will be reflowed by -# tools. -# - Avoid adding information about the data type - we can generate that. - class Options(optmanager.OptManager): - if False: - # This provides type hints for IDEs (e.g. PyCharm) and mypy. - # Autogenerated using test/helper_tools/typehints_for_options.py - add_upstream_certs_to_client_chain = None # type: bool - allow_remote = None # type: bool - anticache = None # type: bool - anticomp = None # type: bool - body_size_limit = None # type: Optional[str] - cadir = None # type: str - certs = None # type: Sequence[str] - ciphers_client = None # type: Optional[str] - ciphers_server = None # type: Optional[str] - client_certs = None # type: Optional[str] - client_replay = None # type: Sequence[str] - console_focus_follow = None # type: bool - console_layout = None # type: str - console_layout_headers = None # type: bool - console_mouse = None # type: bool - console_palette = None # type: str - console_palette_transparent = None # type: bool - default_contentview = None # type: str - flow_detail = None # type: int - http2 = None # type: bool - http2_priority = None # type: bool - ignore_hosts = None # type: Sequence[str] - intercept = None # type: Optional[str] - intercept_active = None # type: bool - keep_host_header = None # type: bool - keepserving = None # type: bool - listen_host = None # type: str - listen_port = None # type: int - mode = None # type: str - onboarding = None # type: bool - onboarding_host = None # type: str - onboarding_port = None # type: int - proxyauth = None # type: Optional[str] - rawtcp = None # type: bool - server_replay_refresh = None # type: bool - replacements = None # type: Sequence[str] - server_replay_kill_extra = None # type: bool - rfile = None # type: Optional[str] - save_stream_file = None # type: Optional[str] - save_stream_filter = None # type: Optional[str] - scripts = None # type: Sequence[str] - server = None # type: bool - server_replay = None # type: Sequence[str] - server_replay_ignore_content = None # type: bool - server_replay_ignore_host = None # type: bool - server_replay_ignore_params = None # type: Sequence[str] - server_replay_ignore_payload_params = None # type: Sequence[str] - server_replay_nopop = None # type: bool - server_replay_use_headers = None # type: Sequence[str] - setheaders = None # type: Sequence[str] - showhost = None # type: bool - spoof_source_address = None # type: bool - ssl_insecure = None # type: bool - ssl_verify_upstream_trusted_ca = None # type: Optional[str] - ssl_verify_upstream_trusted_cadir = None # type: Optional[str] - ssl_version_client = None # type: str - ssl_version_server = None # type: str - stickyauth = None # type: Optional[str] - stickycookie = None # type: Optional[str] - stream_large_bodies = None # type: Optional[str] - stream_websockets = None # type: bool - tcp_hosts = None # type: Sequence[str] - upstream_auth = None # type: Optional[str] - upstream_bind_address = None # type: str - upstream_cert = None # type: bool - verbosity = None # type: str - view_filter = None # type: Optional[str] - view_order = None # type: str - view_order_reversed = None # type: bool - web_debug = None # type: bool - web_iface = None # type: str - web_open_browser = None # type: bool - web_port = None # type: int - websocket = None # type: bool - def __init__(self, **kwargs) -> None: super().__init__() self.add_option( - "onboarding", bool, True, - "Toggle the mitmproxy onboarding app." - ) - self.add_option( - "onboarding_host", str, APP_HOST, - """ - Onboarding app domain. For transparent mode, use an IP when a DNS - entry for the app domain is not present. - """ - ) - self.add_option( - "onboarding_port", int, APP_PORT, - "Port to serve the onboarding app from." - ) - self.add_option( - "anticache", bool, False, - """ - Strip out request headers that might cause the server to return - 304-not-modified. - """ - ) - self.add_option( - "anticomp", bool, False, - "Try to convince servers to send us un-compressed data." - ) - self.add_option( - "client_replay", Sequence[str], [], - "Replay client requests from a saved file." - ) - self.add_option( - "server_replay_kill_extra", bool, False, - "Kill extra requests during replay." - ) - self.add_option( - "keepserving", bool, False, - """ - Continue serving after client playback, server playback or file - read. This option is ignored by interactive tools, which always keep - serving. - """ - ) - self.add_option( "server", bool, True, "Start a proxy server. Enabled by default." ) self.add_option( - "server_replay_nopop", bool, False, - """ - Don't remove flows from server replay state after use. This makes it - possible to replay same response multiple times. - """ - ) - self.add_option( - "server_replay_refresh", bool, True, - """ - Refresh server replay responses by adjusting date, expires and - last-modified headers, as well as adjusting cookie expiration. - """ - ) - self.add_option( - "rfile", Optional[str], None, - "Read flows from file." - ) - self.add_option( - "scripts", Sequence[str], [], - """ - Execute a script. - """ - ) - self.add_option( "showhost", bool, False, "Use the Host header to construct URLs for display." ) - self.add_option( - "replacements", Sequence[str], [], - """ - Replacement patterns of the form "/pattern/regex/replacement", where - the separator can be any character. - """ - ) - self.add_option( - "server_replay_use_headers", Sequence[str], [], - "Request headers to be considered during replay." - ) - self.add_option( - "setheaders", Sequence[str], [], - """ - Header set pattern of the form "/pattern/header/value", where the - separator can be any character. - """ - ) - self.add_option( - "server_replay", Sequence[str], [], - "Replay server responses from a saved file." - ) - self.add_option( - "stickycookie", Optional[str], None, - "Set sticky cookie filter. Matched against requests." - ) - self.add_option( - "stickyauth", Optional[str], None, - "Set sticky auth filter. Matched against requests." - ) - self.add_option( - "stream_large_bodies", Optional[str], None, - """ - Stream data to the client if response body exceeds the given - threshold. If streamed, the body will not be stored in any way. - Understands k/m/g suffixes, i.e. 3m for 3 megabytes. - """ - ) - self.add_option( - "stream_websockets", bool, False, - """ - Stream WebSocket messages between client and server. - Messages are captured and cannot be modified. - """ - ) - self.add_option( - "verbosity", str, 'info', - "Log verbosity.", - choices=log_verbosity - ) - self.add_option( - "default_contentview", str, "auto", - "The default content view mode.", - choices = [i.name.lower() for i in contentviews.views] - ) - self.add_option( - "save_stream_file", Optional[str], None, - "Stream flows to file as they arrive. Prefix path with + to append." - ) - self.add_option( - "save_stream_filter", Optional[str], None, - "Filter which flows are written to file." - ) - self.add_option( - "server_replay_ignore_content", bool, False, - "Ignore request's content while searching for a saved flow to replay." - ) - self.add_option( - "server_replay_ignore_params", Sequence[str], [], - """ - Request's parameters to be ignored while searching for a saved flow - to replay. - """ - ) - self.add_option( - "server_replay_ignore_payload_params", Sequence[str], [], - """ - Request's payload parameters (application/x-www-form-urlencoded or - multipart/form-data) to be ignored while searching for a saved flow - to replay. - """ - ) - self.add_option( - "server_replay_ignore_host", bool, False, - """ - Ignore request's destination host while searching for a saved flow - to replay. - """ - ) # Proxy options self.add_option( - "proxyauth", Optional[str], None, - """ - Require proxy authentication. Format: - "username:pass", - "any" to accept any user/pass combination, - "@path" to use an Apache htpasswd file, - or "ldap[s]:url_server_ldap:dn_auth:password:dn_subtree" for LDAP authentication. - """ - ) - self.add_option( "add_upstream_certs_to_client_chain", bool, False, """ Add all certificates of the upstream server to the certificate chain @@ -289,13 +30,6 @@ class Options(optmanager.OptManager): """ ) self.add_option( - "body_size_limit", Optional[str], None, - """ - Byte size limit of HTTP request and response bodies. Understands - k/m/g suffixes, i.e. 3m for 3 megabytes. - """ - ) - self.add_option( "cadir", str, CA_DIR, "Location of the default mitmproxy CA files." ) @@ -356,13 +90,6 @@ class Options(optmanager.OptManager): "upstream_cert", bool, True, "Connect to upstream server to look up certificate details." ) - self.add_option( - "keep_host_header", bool, False, - """ - Reverse Proxy: Keep the original host header instead of rewriting it - to the reverse proxy target. - """ - ) self.add_option( "http2", bool, True, @@ -396,13 +123,6 @@ class Options(optmanager.OptManager): """ ) self.add_option( - "upstream_auth", Optional[str], None, - """ - Add HTTP Basic authentication to upstream proxy and reverse proxy - requests. Format: username:password. - """ - ) - self.add_option( "ssl_version_client", str, "secure", """ Set supported SSL/TLS versions for client connections. SSLv2, SSLv3 @@ -442,31 +162,4 @@ class Options(optmanager.OptManager): """ ) - self.add_option( - "intercept_active", bool, False, - "Intercept toggle" - ) - - self.add_option( - "intercept", Optional[str], None, - "Intercept filter expression." - ) - - self.add_option( - "view_filter", Optional[str], None, - "Limit which flows are displayed." - ) - - # Dump options - self.add_option( - "flow_detail", int, 1, - """ - The display detail level for flows in mitmdump: 0 (almost quiet) to 3 (very verbose). - 0: shortened request URL, response status code, WebSocket and TCP message notifications. - 1: full request URL with response status code - 2: 1 + HTTP headers - 3: 2 + full response content, content of WebSocket and TCP messages. - """ - ) - self.update(**kwargs) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 01d97af3..bb9e3030 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -327,6 +327,13 @@ class OptManager: return d def make_parser(self, parser, optname, metavar=None, short=None): + """ + Auto-Create a command-line parser entry for a named option. If the + option does not exist, it is ignored. + """ + if optname not in self._options: + return + o = self._options[optname] def mkf(l, s): diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index 1c90a7a0..439c6702 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -359,7 +359,7 @@ class TransparentProxy: packet.dst_addr, packet.dst_port = self.proxy_addr, self.proxy_port packet.direction = pydivert.consts.Direction.INBOUND - # Use any handle thats on the NETWORK layer - request_local may be + # Use any handle that's on the NETWORK layer - request_local may be # unavailable. self.response_handle.send(packet) diff --git a/mitmproxy/proxy/protocol/http.py b/mitmproxy/proxy/protocol/http.py index 076ffa62..99286fa5 100644 --- a/mitmproxy/proxy/protocol/http.py +++ b/mitmproxy/proxy/protocol/http.py @@ -481,7 +481,7 @@ class HttpLayer(base.Layer): if address != self.server_conn.address or tls != self.server_tls: self.set_server(address) self.set_server_tls(tls, address[0]) - # Establish connection is neccessary. + # Establish connection is necessary. if not self.server_conn.connected(): self.connect() else: diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py index cc99a715..1e7c041d 100644 --- a/mitmproxy/proxy/protocol/http2.py +++ b/mitmproxy/proxy/protocol/http2.py @@ -269,7 +269,7 @@ class Http2Layer(base.Layer): def _handle_priority_updated(self, eid, event): if not self.config.options.http2_priority: - self.log("HTTP/2 PRIORITY frame surpressed. Use --http2-priority to enable forwarding.", "debug") + self.log("HTTP/2 PRIORITY frame suppressed. Use --http2-priority to enable forwarding.", "debug") return True if eid in self.streams and self.streams[eid].handled_priority_event is event: @@ -541,7 +541,7 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr # only send priority information if they actually came with the original HeadersFrame # and not if they got updated before/after with a PriorityFrame if not self.config.options.http2_priority: - self.log("HTTP/2 PRIORITY information in HEADERS frame surpressed. Use --http2-priority to enable forwarding.", "debug") + self.log("HTTP/2 PRIORITY information in HEADERS frame suppressed. Use --http2-priority to enable forwarding.", "debug") else: priority_exclusive = self.priority_exclusive priority_depends_on = self._map_depends_on_stream_id(self.server_stream_id, self.priority_depends_on) diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index 876c1162..09ce87ba 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -5,7 +5,7 @@ from mitmproxy import exceptions from mitmproxy.net import tls as net_tls from mitmproxy.proxy.protocol import base -# taken from https://testssl.sh/openssl-rfc.mappping.html +# taken from https://testssl.sh/openssl-rfc.mapping.html CIPHER_ID_NAME_MAP = { 0x00: 'NULL-MD5', 0x01: 'NULL-MD5', diff --git a/mitmproxy/test/taddons.py b/mitmproxy/test/taddons.py index d966f1d5..82a935d2 100644 --- a/mitmproxy/test/taddons.py +++ b/mitmproxy/test/taddons.py @@ -6,7 +6,7 @@ import mitmproxy.options from mitmproxy import addonmanager from mitmproxy import command from mitmproxy import eventsequence -from mitmproxy.addons import script +from mitmproxy.addons import script, core class TestAddons(addonmanager.AddonManager): @@ -59,14 +59,20 @@ class context: provides a number of helper methods for common testing scenarios. """ - def __init__(self, master=None, options=None): + def __init__(self, *addons, options=None, loadcore=True): options = options or mitmproxy.options.Options() - self.master = master or RecordingMaster( + self.master = RecordingMaster( options ) self.options = self.master.options self.wrapped = None + if loadcore: + self.master.addons.add(core.Core()) + + for a in addons: + self.master.addons.add(a) + def ctx(self): """ Returns a new handler context. @@ -134,7 +140,7 @@ class context: def command(self, func, *args): """ - Invoke a command function with a list of string arguments within a command context, mimicing the actual command environment. + Invoke a command function with a list of string arguments within a command context, mimicking the actual command environment. """ cmd = command.Command(self.master.commands, "test.command", func) return cmd.call(args) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index d413ff28..4b7598cf 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -1,10 +1,10 @@ import argparse import os -from mitmproxy import options +from mitmproxy.addons import core -CONFIG_PATH = os.path.join(options.CA_DIR, "config.yaml") +CONFIG_PATH = os.path.join(core.CA_DIR, "config.yaml") def common_options(parser, opts): diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index c73eda42..2b9ff334 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -7,6 +7,7 @@ from mitmproxy import command from mitmproxy import exceptions from mitmproxy import flow from mitmproxy import http +from mitmproxy import log from mitmproxy import contentviews from mitmproxy.utils import strutils import mitmproxy.types @@ -77,13 +78,23 @@ class ConsoleAddon: def load(self, loader): loader.add_option( + "console_default_contentview", str, "auto", + "The default content view mode.", + choices = [i.name.lower() for i in contentviews.views] + ) + loader.add_option( + "console_eventlog_verbosity", str, 'info', + "EventLog verbosity.", + choices=log.LogTierOrder + ) + loader.add_option( "console_layout", str, "single", "Console layout.", choices=sorted(console_layouts), ) loader.add_option( "console_layout_headers", bool, True, - "Show layout comonent headers", + "Show layout component headers", ) loader.add_option( "console_focus_follow", bool, False, @@ -110,15 +121,6 @@ class ConsoleAddon: """ return ["single", "vertical", "horizontal"] - @command.command("console.intercept.toggle") - def intercept_toggle(self) -> None: - """ - Toggles interception on/off leaving intercept filters intact. - """ - ctx.options.update( - intercept_active = not ctx.options.intercept_active - ) - @command.command("console.layout.cycle") def layout_cycle(self) -> None: """ @@ -227,7 +229,7 @@ class ConsoleAddon: ) -> None: """ Prompt the user to choose from a specified list of strings, then - invoke another command with all occurances of {choice} replaced by + invoke another command with all occurrences of {choice} replaced by the choice the user made. """ def callback(opt): @@ -253,7 +255,7 @@ class ConsoleAddon: ) -> None: """ Prompt the user to choose from a list of strings returned by a - command, then invoke another command with all occurances of {choice} + command, then invoke another command with all occurrences of {choice} replaced by the choice the user made. """ choices = ctx.master.commands.call_args(choicecmd, []) @@ -540,7 +542,7 @@ class ConsoleAddon: [ "@focus", "flowview_mode_%s" % idx, - self.master.options.default_contentview, + self.master.options.console_default_contentview, ] ) diff --git a/mitmproxy/tools/console/defaultkeys.py b/mitmproxy/tools/console/defaultkeys.py index c7876288..7f65c1f7 100644 --- a/mitmproxy/tools/console/defaultkeys.py +++ b/mitmproxy/tools/console/defaultkeys.py @@ -26,7 +26,7 @@ def map(km): km.add("ctrl f", "console.nav.pagedown", ["global"], "Page down") km.add("ctrl b", "console.nav.pageup", ["global"], "Page up") - km.add("I", "console.intercept.toggle", ["global"], "Toggle intercept") + km.add("I", "set intercept_active=toggle", ["global"], "Toggle intercept") km.add("i", "console.command.set intercept", ["global"], "Set intercept") km.add("W", "console.command.set save_stream_file", ["global"], "Stream to file") km.add("A", "flow.resume @all", ["flowlist", "flowview"], "Resume all intercepted flows") diff --git a/mitmproxy/tools/console/eventlog.py b/mitmproxy/tools/console/eventlog.py index 8083180d..f3305469 100644 --- a/mitmproxy/tools/console/eventlog.py +++ b/mitmproxy/tools/console/eventlog.py @@ -21,7 +21,7 @@ class EventLog(urwid.ListBox, layoutwidget.LayoutWidget): master.events.sig_add.connect(self.add_event) master.events.sig_refresh.connect(self.refresh_events) - self.master.options.subscribe(self.refresh_events, ["verbosity"]) + self.master.options.subscribe(self.refresh_events, ["console_eventlog_verbosity"]) self.refresh_events() super().__init__(self.walker) @@ -44,7 +44,7 @@ class EventLog(urwid.ListBox, layoutwidget.LayoutWidget): return super().keypress(size, key) def add_event(self, event_store, entry: log.LogEntry): - if log.log_tier(self.master.options.verbosity) < log.log_tier(entry.level): + if log.log_tier(self.master.options.console_eventlog_verbosity) < log.log_tier(entry.level): return txt = "%s: %s" % (entry.level, str(entry.msg)) if entry.level in ("error", "warn", "alert"): diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index da35047e..5da6ef0b 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -10,6 +10,7 @@ import sys import tempfile import traceback import typing # noqa +import contextlib import urwid @@ -86,7 +87,7 @@ class ConsoleMaster(master.Master): ) def sig_add_log(self, event_store, entry: log.LogEntry): - if log.log_tier(self.options.verbosity) < log.log_tier(entry.level): + if log.log_tier(self.options.console_eventlog_verbosity) < log.log_tier(entry.level): return if entry.level in ("error", "warn", "alert"): if self.first_tick: @@ -102,6 +103,16 @@ class ConsoleMaster(master.Master): return callback(*args) self.loop.set_alarm_in(seconds, cb) + @contextlib.contextmanager + def uistopped(self): + self.loop.stop() + try: + yield + finally: + self.loop.start() + self.loop.screen_size = None + self.loop.draw_screen() + def spawn_editor(self, data): text = not isinstance(data, bytes) fd, name = tempfile.mkstemp('', "mproxy", text=text) @@ -111,17 +122,16 @@ class ConsoleMaster(master.Master): c = os.environ.get("EDITOR") or "vi" cmd = shlex.split(c) cmd.append(name) - self.ui.stop() - try: - subprocess.call(cmd) - except: - signals.status_message.send( - message="Can't start editor: %s" % " ".join(c) - ) - else: - with open(name, "r" if text else "rb") as f: - data = f.read() - self.ui.start() + with self.uistopped(): + try: + subprocess.call(cmd) + except: + signals.status_message.send( + message="Can't start editor: %s" % " ".join(c) + ) + else: + with open(name, "r" if text else "rb") as f: + data = f.read() os.unlink(name) return data @@ -153,14 +163,13 @@ class ConsoleMaster(master.Master): c = "less" cmd = shlex.split(c) cmd.append(name) - self.ui.stop() - try: - subprocess.call(cmd, shell=shell) - except: - signals.status_message.send( - message="Can't start external viewer: %s" % " ".join(c) - ) - self.ui.start() + with self.uistopped(): + try: + subprocess.call(cmd, shell=shell) + except: + signals.status_message.send( + message="Can't start external viewer: %s" % " ".join(c) + ) os.unlink(name) def set_palette(self, opts, updated): diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index bdb39013..8553a66f 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -197,10 +197,10 @@ class StatusBar(urwid.WidgetWrap): r.append("[") r.append(("heading_key", "u")) r.append(":%s]" % self.master.options.stickyauth) - if self.master.options.default_contentview != "auto": + if self.master.options.console_default_contentview != "auto": r.append("[") r.append(("heading_key", "M")) - r.append(":%s]" % self.master.options.default_contentview) + r.append(":%s]" % self.master.options.console_default_contentview) if self.master.options.has_changed("view_order"): r.append("[") r.append(("heading_key", "o")) diff --git a/mitmproxy/tools/main.py b/mitmproxy/tools/main.py index 5c2d9f2f..330060f7 100644 --- a/mitmproxy/tools/main.py +++ b/mitmproxy/tools/main.py @@ -46,10 +46,10 @@ def process_options(parser, opts, args): if args.quiet or args.options or args.commands: # also reduce log verbosity if --options or --commands is passed, # we don't want log messages from regular startup then. - args.verbosity = 'error' + args.termlog_verbosity = 'error' args.flow_detail = 0 if args.verbose: - args.verbosity = 'debug' + args.termlog_verbosity = 'debug' args.flow_detail = 2 adict = {} @@ -104,9 +104,7 @@ def run( master.server = server master.addons.trigger("configure", opts.keys()) master.addons.trigger("tick") - remaining = opts.update_known(**unknown) - if remaining and log.log_tier(opts.verbosity) > 1: - print("Ignored options: %s" % remaining) + opts.update_known(**unknown) if args.options: print(optmanager.dump_defaults(opts)) sys.exit(0) diff --git a/mitmproxy/utils/arg_check.py b/mitmproxy/utils/arg_check.py index 873bef06..9c582c4c 100644 --- a/mitmproxy/utils/arg_check.py +++ b/mitmproxy/utils/arg_check.py @@ -11,7 +11,6 @@ DEPRECATED = """ --order --no-mouse --reverse ---socks --http2-priority --no-http2-priority --no-websocket @@ -59,6 +58,7 @@ REPLACED = """ -i -f --filter +--socks """ REPLACEMENTS = { @@ -99,7 +99,8 @@ REPLACEMENTS = { "--replace": "--replacements", "-i": "--intercept", "-f": "--view-filter", - "--filter": "--view-filter" + "--filter": "--view-filter", + "--socks": "--mode socks5" } diff --git a/pathod/pathoc.py b/pathod/pathoc.py index b177d556..18dcccf2 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -352,7 +352,7 @@ class Pathoc(tcp.TCPClient): timeout: If specified None may be yielded instead if timeout is reached. If timeout is None, wait forever. If timeout is 0, return - immedately if nothing is on the queue. + immediately if nothing is on the queue. finish: If true, consume messages until the reader shuts down. Otherwise, return None on timeout. @@ -434,7 +434,7 @@ class Pathoc(tcp.TCPClient): req = language.serve(r, self.wfile, self.settings) self.wfile.flush() - # build a dummy request to read the reponse + # build a dummy request to read the response # ideally this would be returned directly from language.serve dummy_req = net_http.Request( first_line_format="relative", @@ -471,7 +471,7 @@ class Pathoc(tcp.TCPClient): """ Performs a single request. - r: A language.message.Messsage object, or a string representing + r: A language.message.Message object, or a string representing one. Returns Response if we have a non-ignored response. diff --git a/release/README.md b/release/README.md index 2b709318..b2f97aab 100644 --- a/release/README.md +++ b/release/README.md @@ -16,7 +16,11 @@ Make sure run all these steps on the correct branch you want to create a new rel - Attach all files from the new release folder on https://snapshots.mitmproxy.org ## PyPi -- Upload wheel to pypi: `twine upload <mitmproxy-...-.whl` +- `tox -e rtool -- upload-release` + +## Homebrew +- `tox -e rtool -- homebrew-pr` +- The Homebrew maintainers are typically very fast and detect our new relese within a day, but we can be a nice citizen and create the PR ourself. ## Docker - Update docker-releases repo @@ -28,6 +32,13 @@ Make sure run all these steps on the correct branch you want to create a new rel * `2.0.0` for new major versions * `2.0.2` for new patch versions - Update `latest` tag [here](https://hub.docker.com/r/mitmproxy/mitmproxy/~/settings/automated-builds/) +- Check that the build for this tag succeeds [https://hub.docker.com/r/mitmproxy/mitmproxy/builds/](here) +- If build failed: + - Fix it and commit + - `git tag 3.0.2` the new commit + - `git push origin :refs/tags/3.0.2` to delete the old remote tag + - `git push --tags` to push the new tag + - Check the build details page again ## Prepare for next release diff --git a/release/rtool.py b/release/rtool.py index 9050107e..25f37e61 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -3,6 +3,7 @@ import contextlib import fnmatch import os +import sys import platform import re import runpy @@ -286,6 +287,24 @@ def upload_release(username, password, repository): ]) +@cli.command("homebrew-pr") +def homebrew_pr(): + """ + Create a new Homebrew PR + """ + if platform.system() != "Darwin": + print("You need to run this on macOS to create a new Homebrew PR. Sorry.") + sys.exit(1) + + print("Creating a new PR with Homebrew...") + subprocess.check_call([ + "brew", + "bump-formula-pr", + "--url", "https://github.com/mitmproxy/mitmproxy/archive/v{}".format(get_version()), + "mitmproxy", + ]) + + @cli.command("upload-snapshot") @click.option("--host", envvar="SNAPSHOT_HOST", prompt=True) @click.option("--port", envvar="SNAPSHOT_PORT", type=int, default=22) @@ -66,7 +66,6 @@ setup( "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! "click>=6.2, <7", "cryptography>=2.1.4,<2.2", - 'h11>=0.7.0,<0.8', "h2>=3.0.1,<4", "hyperframe>=5.1.0,<6", "kaitaistruct>=0.7,<0.9", @@ -76,7 +75,6 @@ setup( "pyOpenSSL>=17.5,<17.6", "pyparsing>=2.1.3, <2.3", "pyperclip>=1.6.0, <1.7", - "requests>=2.9.1, <3", "ruamel.yaml>=0.13.2, <0.16", "sortedcontainers>=1.5.4, <1.6", "tornado>=4.3, <4.6", @@ -96,12 +94,12 @@ setup( "pytest-timeout>=1.2.1,<2", "pytest-xdist>=1.22,<2", "pytest>=3.3,<4", + "requests>=2.9.1, <3", "tox>=2.3, <3", "rstcheck>=2.2, <4.0", ], 'examples': [ - "beautifulsoup4>=4.4.1, <4.7", - "Pillow>=4.3,<5.1", + "beautifulsoup4>=4.4.1, <4.7" ] } ) diff --git a/test/examples/test_xss_scanner.py b/test/examples/test_xss_scanner.py index 8cf06a2a..1d723d53 100644 --- a/test/examples/test_xss_scanner.py +++ b/test/examples/test_xss_scanner.py @@ -296,12 +296,23 @@ class TestXSSScanner(): assert xss_info == expected_xss_info assert sqli_info is None + def mocked_socket_gethostbyname(domain): + claimed_domains = ["google.com"] + if domain not in claimed_domains: + from socket import gaierror + raise gaierror("[Errno -2] Name or service not known") + else: + return '216.58.221.46' + @pytest.fixture def logger(self): class Logger(): def __init__(self): self.args = [] + def info(self, str): + self.args.append(str) + def error(self, str): self.args.append(str) return Logger() @@ -309,6 +320,7 @@ class TestXSSScanner(): def test_find_unclaimed_URLs(self, monkeypatch, logger): logger.args = [] monkeypatch.setattr("mitmproxy.ctx.log", logger) + monkeypatch.setattr("socket.gethostbyname", self.mocked_socket_gethostbyname) xss.find_unclaimed_URLs("<html><script src=\"http://google.com\"></script></html>", "https://example.com") assert logger.args == [] diff --git a/test/helper_tools/dumperview.py b/test/helper_tools/dumperview.py index d417d767..e17dc77b 100755 --- a/test/helper_tools/dumperview.py +++ b/test/helper_tools/dumperview.py @@ -4,12 +4,11 @@ import click from mitmproxy.addons import dumper from mitmproxy.test import tflow from mitmproxy.test import taddons -from mitmproxy.tools import options def show(flow_detail, flows): d = dumper.Dumper() - with taddons.context(options=options.Options()) as ctx: + with taddons.context() as ctx: ctx.configure(d, flow_detail=flow_detail) for f in flows: ctx.cycle(d, f) diff --git a/test/mitmproxy/addons/test_allowremote.py b/test/mitmproxy/addons/test_allowremote.py index 9fc71525..69019726 100644 --- a/test/mitmproxy/addons/test_allowremote.py +++ b/test/mitmproxy/addons/test_allowremote.py @@ -1,7 +1,7 @@ from unittest import mock import pytest -from mitmproxy.addons import allowremote +from mitmproxy.addons import allowremote, proxyauth from mitmproxy.test import taddons @@ -19,8 +19,8 @@ from mitmproxy.test import taddons ]) def test_allowremote(allow_remote, ip, should_be_killed): ar = allowremote.AllowRemote() - with taddons.context() as tctx: - tctx.master.addons.register(ar) + up = proxyauth.ProxyAuth() + with taddons.context(ar, up) as tctx: tctx.options.allow_remote = allow_remote with mock.patch('mitmproxy.proxy.protocol.base.Layer') as layer: diff --git a/test/mitmproxy/addons/test_anticache.py b/test/mitmproxy/addons/test_anticache.py index 928f2180..d1765fe0 100644 --- a/test/mitmproxy/addons/test_anticache.py +++ b/test/mitmproxy/addons/test_anticache.py @@ -7,7 +7,7 @@ from mitmproxy.test import taddons class TestAntiCache: def test_simple(self): sa = anticache.AntiCache() - with taddons.context() as tctx: + with taddons.context(sa) as tctx: f = tflow.tflow(resp=True) f.request.headers["if-modified-since"] = "test" f.request.headers["if-none-match"] = "test" diff --git a/test/mitmproxy/addons/test_anticomp.py b/test/mitmproxy/addons/test_anticomp.py index 2a6cf3ce..92650332 100644 --- a/test/mitmproxy/addons/test_anticomp.py +++ b/test/mitmproxy/addons/test_anticomp.py @@ -7,7 +7,7 @@ from mitmproxy.test import taddons class TestAntiComp: def test_simple(self): sa = anticomp.AntiComp() - with taddons.context() as tctx: + with taddons.context(sa) as tctx: f = tflow.tflow(resp=True) sa.request(f) diff --git a/test/mitmproxy/addons/test_clientplayback.py b/test/mitmproxy/addons/test_clientplayback.py index 3f990668..f172af83 100644 --- a/test/mitmproxy/addons/test_clientplayback.py +++ b/test/mitmproxy/addons/test_clientplayback.py @@ -24,7 +24,7 @@ class MockThread(): class TestClientPlayback: def test_playback(self): cp = clientplayback.ClientPlayback() - with taddons.context() as tctx: + with taddons.context(cp) as tctx: assert cp.count() == 0 f = tflow.tflow(resp=True) cp.start_replay([f]) @@ -58,7 +58,7 @@ class TestClientPlayback: def test_load_file(self, tmpdir): cp = clientplayback.ClientPlayback() - with taddons.context(): + with taddons.context(cp): fpath = str(tmpdir.join("flows")) tdump(fpath, [tflow.tflow(resp=True)]) cp.load_file(fpath) @@ -68,7 +68,7 @@ class TestClientPlayback: def test_configure(self, tmpdir): cp = clientplayback.ClientPlayback() - with taddons.context() as tctx: + with taddons.context(cp) as tctx: path = str(tmpdir.join("flows")) tdump(path, [tflow.tflow()]) tctx.configure(cp, client_replay=[path]) diff --git a/test/mitmproxy/addons/test_core.py b/test/mitmproxy/addons/test_core.py index 5aa4ef37..b1b105d9 100644 --- a/test/mitmproxy/addons/test_core.py +++ b/test/mitmproxy/addons/test_core.py @@ -1,3 +1,5 @@ +from unittest import mock + from mitmproxy.addons import core from mitmproxy.test import taddons from mitmproxy.test import tflow @@ -7,12 +9,10 @@ import pytest def test_set(): sa = core.Core() - with taddons.context() as tctx: - tctx.master.addons.add(sa) - - assert not tctx.master.options.anticomp - tctx.command(sa.set, "anticomp") - assert tctx.master.options.anticomp + with taddons.context(loadcore=False) as tctx: + assert tctx.master.options.server + tctx.command(sa.set, "server=false") + assert not tctx.master.options.server with pytest.raises(exceptions.CommandError): tctx.command(sa.set, "nonexistent") @@ -20,7 +20,7 @@ def test_set(): def test_resume(): sa = core.Core() - with taddons.context(): + with taddons.context(loadcore=False): f = tflow.tflow() assert not sa.resume([f]) f.intercept() @@ -30,7 +30,7 @@ def test_resume(): def test_mark(): sa = core.Core() - with taddons.context(): + with taddons.context(loadcore=False): f = tflow.tflow() assert not f.marked sa.mark([f], True) @@ -44,7 +44,7 @@ def test_mark(): def test_kill(): sa = core.Core() - with taddons.context(): + with taddons.context(loadcore=False): f = tflow.tflow() f.intercept() assert f.killable @@ -54,7 +54,7 @@ def test_kill(): def test_revert(): sa = core.Core() - with taddons.context(): + with taddons.context(loadcore=False): f = tflow.tflow() f.backup() f.request.content = b"bar" @@ -65,7 +65,7 @@ def test_revert(): def test_flow_set(): sa = core.Core() - with taddons.context(): + with taddons.context(loadcore=False): f = tflow.tflow(resp=True) assert sa.flow_set_options() @@ -101,7 +101,7 @@ def test_flow_set(): def test_encoding(): sa = core.Core() - with taddons.context(): + with taddons.context(loadcore=False): f = tflow.tflow() assert sa.encode_options() sa.encode([f], "request", "deflate") @@ -128,28 +128,23 @@ def test_options(tmpdir): p = str(tmpdir.join("path")) sa = core.Core() with taddons.context() as tctx: - tctx.options.stickycookie = "foo" - assert tctx.options.stickycookie == "foo" - sa.options_reset() - assert tctx.options.stickycookie is None - - tctx.options.stickycookie = "foo" - tctx.options.stickyauth = "bar" - sa.options_reset_one("stickycookie") - assert tctx.options.stickycookie is None - assert tctx.options.stickyauth == "bar" + tctx.options.listen_host = "foo" + assert tctx.options.listen_host == "foo" + sa.options_reset_one("listen_host") + assert tctx.options.listen_host != "foo" with pytest.raises(exceptions.CommandError): sa.options_reset_one("unknown") + tctx.options.listen_host = "foo" sa.options_save(p) with pytest.raises(exceptions.CommandError): sa.options_save("/") sa.options_reset() - assert tctx.options.stickyauth is None + assert tctx.options.listen_host == "" sa.options_load(p) - assert tctx.options.stickyauth == "bar" + assert tctx.options.listen_host == "foo" sa.options_load("/nonexistent") @@ -157,3 +152,40 @@ def test_options(tmpdir): f.write("'''") with pytest.raises(exceptions.CommandError): sa.options_load(p) + + +def test_validation_simple(): + sa = core.Core() + with taddons.context() as tctx: + with pytest.raises(exceptions.OptionsError): + tctx.configure(sa, body_size_limit = "invalid") + tctx.configure(sa, body_size_limit = "1m") + + with pytest.raises(exceptions.OptionsError, match="mutually exclusive"): + tctx.configure( + sa, + add_upstream_certs_to_client_chain = True, + upstream_cert = False + ) + with pytest.raises(exceptions.OptionsError, match="Invalid mode"): + tctx.configure( + sa, + mode = "Flibble" + ) + + +@mock.patch("mitmproxy.platform.original_addr", None) +def test_validation_no_transparent(): + sa = core.Core() + with taddons.context() as tctx: + with pytest.raises(Exception, match="Transparent mode not supported"): + tctx.configure(sa, mode = "transparent") + + +@mock.patch("mitmproxy.platform.original_addr") +def test_validation_modes(m): + sa = core.Core() + with taddons.context() as tctx: + tctx.configure(sa, mode = "reverse:http://localhost") + with pytest.raises(Exception, match="Invalid server specification"): + tctx.configure(sa, mode = "reverse:")
\ No newline at end of file diff --git a/test/mitmproxy/addons/test_core_option_validation.py b/test/mitmproxy/addons/test_core_option_validation.py deleted file mode 100644 index cd5d4dfa..00000000 --- a/test/mitmproxy/addons/test_core_option_validation.py +++ /dev/null @@ -1,42 +0,0 @@ -from mitmproxy import exceptions -from mitmproxy.addons import core_option_validation -from mitmproxy.test import taddons -import pytest -from unittest import mock - - -def test_simple(): - sa = core_option_validation.CoreOptionValidation() - with taddons.context() as tctx: - with pytest.raises(exceptions.OptionsError): - tctx.configure(sa, body_size_limit = "invalid") - tctx.configure(sa, body_size_limit = "1m") - - with pytest.raises(exceptions.OptionsError, match="mutually exclusive"): - tctx.configure( - sa, - add_upstream_certs_to_client_chain = True, - upstream_cert = False - ) - with pytest.raises(exceptions.OptionsError, match="Invalid mode"): - tctx.configure( - sa, - mode = "Flibble" - ) - - -@mock.patch("mitmproxy.platform.original_addr", None) -def test_no_transparent(): - sa = core_option_validation.CoreOptionValidation() - with taddons.context() as tctx: - with pytest.raises(Exception, match="Transparent mode not supported"): - tctx.configure(sa, mode = "transparent") - - -@mock.patch("mitmproxy.platform.original_addr") -def test_modes(m): - sa = core_option_validation.CoreOptionValidation() - with taddons.context() as tctx: - tctx.configure(sa, mode = "reverse:http://localhost") - with pytest.raises(Exception, match="Invalid server specification"): - tctx.configure(sa, mode = "reverse:") diff --git a/test/mitmproxy/addons/test_dumper.py b/test/mitmproxy/addons/test_dumper.py index fb80f3ce..228bacf8 100644 --- a/test/mitmproxy/addons/test_dumper.py +++ b/test/mitmproxy/addons/test_dumper.py @@ -10,13 +10,12 @@ from mitmproxy.test import tutils from mitmproxy.addons import dumper from mitmproxy import exceptions from mitmproxy import http -from mitmproxy import options def test_configure(): d = dumper.Dumper() - with taddons.context(options=options.Options()) as ctx: - ctx.configure(d, view_filter="~b foo") + with taddons.context(d) as ctx: + ctx.configure(d, dumper_filter="~b foo") assert d.filter f = tflow.tflow(resp=True) @@ -24,17 +23,17 @@ def test_configure(): f.response.content = b"foo" assert d.match(f) - ctx.configure(d, view_filter=None) + ctx.configure(d, dumper_filter=None) assert not d.filter with pytest.raises(exceptions.OptionsError): - ctx.configure(d, view_filter="~~") + ctx.configure(d, dumper_filter="~~") assert not d.filter def test_simple(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=options.Options()) as ctx: + with taddons.context(d) as ctx: ctx.configure(d, flow_detail=0) d.response(tflow.tflow(resp=True)) assert not sio.getvalue() @@ -102,7 +101,7 @@ def test_echo_body(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=options.Options()) as ctx: + with taddons.context(d) as ctx: ctx.configure(d, flow_detail=3) d._echo_message(f.response) t = sio.getvalue() @@ -112,7 +111,7 @@ def test_echo_body(): def test_echo_request_line(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=options.Options()) as ctx: + with taddons.context(d) as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.tflow(client_conn=None, server_conn=True, resp=True) f.request.is_replay = True @@ -147,8 +146,8 @@ class TestContentView: view_auto.side_effect = exceptions.ContentViewException("") sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=options.Options()) as ctx: - ctx.configure(d, flow_detail=4, verbosity='debug') + with taddons.context(d) as ctx: + ctx.configure(d, flow_detail=4) d.response(tflow.tflow()) assert ctx.master.has_log("content viewer failed") @@ -156,7 +155,7 @@ class TestContentView: def test_tcp(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=options.Options()) as ctx: + with taddons.context(d) as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.ttcpflow() d.tcp_message(f) @@ -171,7 +170,7 @@ def test_tcp(): def test_websocket(): sio = io.StringIO() d = dumper.Dumper(sio) - with taddons.context(options=options.Options()) as ctx: + with taddons.context(d) as ctx: ctx.configure(d, flow_detail=3, showhost=True) f = tflow.twebsocketflow() d.websocket_message(f) diff --git a/test/mitmproxy/addons/test_intercept.py b/test/mitmproxy/addons/test_intercept.py index d4999eb5..b3d24626 100644 --- a/test/mitmproxy/addons/test_intercept.py +++ b/test/mitmproxy/addons/test_intercept.py @@ -1,7 +1,6 @@ import pytest from mitmproxy.addons import intercept -from mitmproxy import options from mitmproxy import exceptions from mitmproxy.test import taddons from mitmproxy.test import tflow @@ -9,7 +8,7 @@ from mitmproxy.test import tflow def test_simple(): r = intercept.Intercept() - with taddons.context(options=options.Options()) as tctx: + with taddons.context(r) as tctx: assert not r.filt tctx.configure(r, intercept="~q") assert r.filt diff --git a/test/mitmproxy/addons/test_keepserving.py b/test/mitmproxy/addons/test_keepserving.py index 70f7e1e6..2869d097 100644 --- a/test/mitmproxy/addons/test_keepserving.py +++ b/test/mitmproxy/addons/test_keepserving.py @@ -4,7 +4,6 @@ from mitmproxy.test import taddons def test_keepserving(): ks = keepserving.KeepServing() - - with taddons.context() as tctx: + with taddons.context(ks) as tctx: ks.event_processing_complete() assert tctx.master.should_exit.is_set() diff --git a/test/mitmproxy/addons/test_onboarding.py b/test/mitmproxy/addons/test_onboarding.py index 474e6c3c..810ddef1 100644 --- a/test/mitmproxy/addons/test_onboarding.py +++ b/test/mitmproxy/addons/test_onboarding.py @@ -2,7 +2,6 @@ import pytest from mitmproxy.addons import onboarding from mitmproxy.test import taddons -from mitmproxy import options from .. import tservers @@ -11,25 +10,28 @@ class TestApp(tservers.HTTPProxyTest): return [onboarding.Onboarding()] def test_basic(self): - with taddons.context() as tctx: - tctx.configure(self.addons()[0]) + ob = onboarding.Onboarding() + with taddons.context(ob) as tctx: + tctx.configure(ob) assert self.app("/").status_code == 200 @pytest.mark.parametrize("ext", ["pem", "p12"]) def test_cert(self, ext): - with taddons.context() as tctx: - tctx.configure(self.addons()[0]) + ob = onboarding.Onboarding() + with taddons.context(ob) as tctx: + tctx.configure(ob) resp = self.app("/cert/%s" % ext) assert resp.status_code == 200 assert resp.content @pytest.mark.parametrize("ext", ["pem", "p12"]) def test_head(self, ext): - with taddons.context() as tctx: - tctx.configure(self.addons()[0]) + ob = onboarding.Onboarding() + with taddons.context(ob) as tctx: + tctx.configure(ob) p = self.pathoc() with p.connect(): - resp = p.request("head:'http://%s/cert/%s'" % (options.APP_HOST, ext)) + resp = p.request("head:'http://%s/cert/%s'" % (tctx.options.onboarding_host, ext)) assert resp.status_code == 200 assert "Content-Length" in resp.headers assert not resp.content diff --git a/test/mitmproxy/addons/test_proxyauth.py b/test/mitmproxy/addons/test_proxyauth.py index 97259d1c..7816dd18 100644 --- a/test/mitmproxy/addons/test_proxyauth.py +++ b/test/mitmproxy/addons/test_proxyauth.py @@ -49,7 +49,7 @@ class TestProxyAuth: ]) def test_is_proxy_auth(self, mode, expected): up = proxyauth.ProxyAuth() - with taddons.context() as ctx: + with taddons.context(up, loadcore=False) as ctx: ctx.options.mode = mode assert up.is_proxy_auth() is expected @@ -75,7 +75,7 @@ class TestProxyAuth: def test_check(self): up = proxyauth.ProxyAuth() - with taddons.context() as ctx: + with taddons.context(up) as ctx: ctx.configure(up, proxyauth="any", mode="regular") f = tflow.tflow() assert not up.check(f) @@ -133,7 +133,7 @@ class TestProxyAuth: def test_authenticate(self): up = proxyauth.ProxyAuth() - with taddons.context() as ctx: + with taddons.context(up, loadcore=False) as ctx: ctx.configure(up, proxyauth="any", mode="regular") f = tflow.tflow() @@ -165,7 +165,7 @@ class TestProxyAuth: def test_configure(self): up = proxyauth.ProxyAuth() - with taddons.context() as ctx: + with taddons.context(up) as ctx: with pytest.raises(exceptions.OptionsError): ctx.configure(up, proxyauth="foo") @@ -223,7 +223,7 @@ class TestProxyAuth: def test_handlers(self): up = proxyauth.ProxyAuth() - with taddons.context() as ctx: + with taddons.context(up) as ctx: ctx.configure(up, proxyauth="any", mode="regular") f = tflow.tflow() diff --git a/test/mitmproxy/addons/test_readfile.py b/test/mitmproxy/addons/test_readfile.py index 813aa10e..0439862a 100644 --- a/test/mitmproxy/addons/test_readfile.py +++ b/test/mitmproxy/addons/test_readfile.py @@ -41,7 +41,7 @@ class TestReadFile: @mock.patch('mitmproxy.master.Master.load_flow') def test_configure(self, mck, tmpdir, data, corrupt_data): rf = readfile.ReadFile() - with taddons.context() as tctx: + with taddons.context(rf) as tctx: tf = tmpdir.join("tfile") tf.write(data.getvalue()) @@ -58,7 +58,7 @@ class TestReadFile: @mock.patch('mitmproxy.master.Master.load_flow') def test_corrupt(self, mck, corrupt_data): rf = readfile.ReadFile() - with taddons.context() as tctx: + with taddons.context(rf) as tctx: with pytest.raises(exceptions.FlowReadException): rf.load_flows(io.BytesIO(b"qibble")) assert not mck.called @@ -71,7 +71,7 @@ class TestReadFile: def test_nonexisting_file(self): rf = readfile.ReadFile() - with taddons.context() as tctx: + with taddons.context(rf) as tctx: with pytest.raises(exceptions.FlowReadException): rf.load_flows_from_path("nonexistent") assert len(tctx.master.logs) == 1 @@ -82,7 +82,7 @@ class TestReadFileStdin: @mock.patch('sys.stdin') def test_stdin(self, stdin, load_flow, data, corrupt_data): rf = readfile.ReadFileStdin() - with taddons.context() as tctx: + with taddons.context(rf) as tctx: stdin.buffer = data tctx.configure(rf, rfile="-") assert not load_flow.called @@ -97,7 +97,7 @@ class TestReadFileStdin: @mock.patch('mitmproxy.master.Master.load_flow') def test_normal(self, load_flow, tmpdir, data): rf = readfile.ReadFileStdin() - with taddons.context(): + with taddons.context(rf): tfile = tmpdir.join("tfile") tfile.write(data.getvalue()) rf.load_flows_from_path(str(tfile)) diff --git a/test/mitmproxy/addons/test_replace.py b/test/mitmproxy/addons/test_replace.py index 9002afb5..9c1f7f79 100644 --- a/test/mitmproxy/addons/test_replace.py +++ b/test/mitmproxy/addons/test_replace.py @@ -18,7 +18,7 @@ class TestReplace: def test_configure(self): r = replace.Replace() - with taddons.context() as tctx: + with taddons.context(r) as tctx: tctx.configure(r, replacements=["one/two/three"]) with pytest.raises(Exception, match="Invalid filter pattern"): tctx.configure(r, replacements=["/~b/two/three"]) @@ -28,7 +28,7 @@ class TestReplace: def test_simple(self): r = replace.Replace() - with taddons.context() as tctx: + with taddons.context(r) as tctx: tctx.configure( r, replacements=[ @@ -48,7 +48,7 @@ class TestReplace: def test_order(self): r = replace.Replace() - with taddons.context() as tctx: + with taddons.context(r) as tctx: tctx.configure( r, replacements=[ @@ -67,7 +67,7 @@ class TestReplace: class TestReplaceFile: def test_simple(self, tmpdir): r = replace.Replace() - with taddons.context() as tctx: + with taddons.context(r) as tctx: tmpfile = tmpdir.join("replacement") tmpfile.write("bar") tctx.configure( @@ -81,7 +81,7 @@ class TestReplaceFile: def test_nonexistent(self, tmpdir): r = replace.Replace() - with taddons.context() as tctx: + with taddons.context(r) as tctx: with pytest.raises(Exception, match="Invalid file path"): tctx.configure( r, diff --git a/test/mitmproxy/addons/test_save.py b/test/mitmproxy/addons/test_save.py index 2dee708f..4486ff78 100644 --- a/test/mitmproxy/addons/test_save.py +++ b/test/mitmproxy/addons/test_save.py @@ -5,14 +5,13 @@ from mitmproxy.test import tflow from mitmproxy import io from mitmproxy import exceptions -from mitmproxy import options from mitmproxy.addons import save from mitmproxy.addons import view def test_configure(tmpdir): sa = save.Save() - with taddons.context(options=options.Options()) as tctx: + with taddons.context() as tctx: with pytest.raises(exceptions.OptionsError): tctx.configure(sa, save_stream_file=str(tmpdir)) with pytest.raises(Exception, match="Invalid filter"): diff --git a/test/mitmproxy/addons/test_script.py b/test/mitmproxy/addons/test_script.py index 78a5be6c..dc21e6fd 100644 --- a/test/mitmproxy/addons/test_script.py +++ b/test/mitmproxy/addons/test_script.py @@ -171,7 +171,7 @@ class TestScriptLoader: "mitmproxy/data/addonscripts/recorder/recorder.py" ) sc = script.ScriptLoader() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: sc.script_run([tflow.tflow(resp=True)], rp) debug = [i.msg for i in tctx.master.logs if i.level == "debug"] assert debug == [ @@ -183,13 +183,13 @@ class TestScriptLoader: def test_script_run_nonexistent(self): sc = script.ScriptLoader() - with taddons.context(): + with taddons.context(sc): with pytest.raises(exceptions.CommandError): sc.script_run([tflow.tflow(resp=True)], "/") def test_simple(self): sc = script.ScriptLoader() - with taddons.context() as tctx: + with taddons.context(loadcore=False) as tctx: tctx.master.addons.add(sc) sc.running() assert len(tctx.master.addons) == 1 @@ -208,8 +208,7 @@ class TestScriptLoader: def test_dupes(self): sc = script.ScriptLoader() - with taddons.context() as tctx: - tctx.master.addons.add(sc) + with taddons.context(sc) as tctx: with pytest.raises(exceptions.OptionsError): tctx.configure( sc, @@ -232,7 +231,7 @@ class TestScriptLoader: def test_load_err(self): sc = script.ScriptLoader() - with taddons.context() as tctx: + with taddons.context(sc, loadcore=False) as tctx: tctx.configure(sc, scripts=[ tutils.test_data.path("mitmproxy/data/addonscripts/load_error.py") ]) @@ -242,7 +241,7 @@ class TestScriptLoader: pass # this is expected and normally guarded. # on the next tick we should not fail however. tctx.invoke(sc, "tick") - assert len(tctx.master.addons) == 0 + assert len(tctx.master.addons) == 1 def test_order(self): rec = tutils.test_data.path("mitmproxy/data/addonscripts/recorder") diff --git a/test/mitmproxy/addons/test_serverplayback.py b/test/mitmproxy/addons/test_serverplayback.py index 32249a88..0bc28ac8 100644 --- a/test/mitmproxy/addons/test_serverplayback.py +++ b/test/mitmproxy/addons/test_serverplayback.py @@ -19,7 +19,7 @@ def tdump(path, flows): def test_load_file(tmpdir): s = serverplayback.ServerPlayback() - with taddons.context(): + with taddons.context(s): fpath = str(tmpdir.join("flows")) tdump(fpath, [tflow.tflow(resp=True)]) s.load_file(fpath) @@ -30,7 +30,7 @@ def test_load_file(tmpdir): def test_config(tmpdir): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: fpath = str(tmpdir.join("flows")) tdump(fpath, [tflow.tflow(resp=True)]) tctx.configure(s, server_replay=[fpath]) @@ -41,7 +41,7 @@ def test_config(tmpdir): def test_tick(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: s.stop = True s.final_flow = tflow.tflow() s.final_flow.live = False @@ -51,7 +51,7 @@ def test_tick(): def test_server_playback(): sp = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(sp) as tctx: tctx.configure(sp) f = tflow.tflow(resp=True) @@ -70,7 +70,7 @@ def test_server_playback(): def test_ignore_host(): sp = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(sp) as tctx: tctx.configure(sp, server_replay_ignore_host=True) r = tflow.tflow(resp=True) @@ -85,7 +85,7 @@ def test_ignore_host(): def test_ignore_content(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure(s, server_replay_ignore_content=False) r = tflow.tflow(resp=True) @@ -113,7 +113,7 @@ def test_ignore_content(): def test_ignore_content_wins_over_params(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure( s, server_replay_ignore_content=True, @@ -137,7 +137,7 @@ def test_ignore_content_wins_over_params(): def test_ignore_payload_params_other_content_type(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure( s, server_replay_ignore_content=False, @@ -161,7 +161,7 @@ def test_ignore_payload_params_other_content_type(): def test_hash(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure(s) r = tflow.tflow() @@ -181,7 +181,7 @@ def test_hash(): def test_headers(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure(s, server_replay_use_headers=["foo"]) r = tflow.tflow(resp=True) @@ -200,7 +200,7 @@ def test_headers(): def test_load(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure(s) r = tflow.tflow(resp=True) @@ -227,7 +227,7 @@ def test_load(): def test_load_with_server_replay_nopop(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure(s, server_replay_nopop=True) r = tflow.tflow(resp=True) @@ -245,7 +245,7 @@ def test_load_with_server_replay_nopop(): def test_ignore_params(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure( s, server_replay_ignore_params=["param1", "param2"] @@ -266,7 +266,7 @@ def test_ignore_params(): def thash(r, r2, setter): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: s = serverplayback.ServerPlayback() tctx.configure( s, @@ -328,7 +328,7 @@ def test_ignore_payload_params(): def test_server_playback_full(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure( s, server_replay_refresh = True, @@ -360,7 +360,7 @@ def test_server_playback_full(): def test_server_playback_kill(): s = serverplayback.ServerPlayback() - with taddons.context() as tctx: + with taddons.context(s) as tctx: tctx.configure( s, server_replay_refresh = True, diff --git a/test/mitmproxy/addons/test_setheaders.py b/test/mitmproxy/addons/test_setheaders.py index 3aaee7f4..2d5e9e3c 100644 --- a/test/mitmproxy/addons/test_setheaders.py +++ b/test/mitmproxy/addons/test_setheaders.py @@ -19,14 +19,14 @@ class TestSetHeaders: def test_configure(self): sh = setheaders.SetHeaders() - with taddons.context() as tctx: + with taddons.context(sh) as tctx: with pytest.raises(Exception, match="Invalid setheader filter pattern"): tctx.configure(sh, setheaders = ["/~b/one/two"]) tctx.configure(sh, setheaders = ["/foo/bar/voing"]) def test_setheaders(self): sh = setheaders.SetHeaders() - with taddons.context() as tctx: + with taddons.context(sh) as tctx: tctx.configure( sh, setheaders = [ diff --git a/test/mitmproxy/addons/test_stickyauth.py b/test/mitmproxy/addons/test_stickyauth.py index ef7d0793..7b422fdd 100644 --- a/test/mitmproxy/addons/test_stickyauth.py +++ b/test/mitmproxy/addons/test_stickyauth.py @@ -9,7 +9,7 @@ from mitmproxy import exceptions def test_configure(): r = stickyauth.StickyAuth() - with taddons.context() as tctx: + with taddons.context(r) as tctx: tctx.configure(r, stickyauth="~s") with pytest.raises(exceptions.OptionsError): tctx.configure(r, stickyauth="~~") @@ -20,7 +20,7 @@ def test_configure(): def test_simple(): r = stickyauth.StickyAuth() - with taddons.context() as tctx: + with taddons.context(r) as tctx: tctx.configure(r, stickyauth=".*") f = tflow.tflow(resp=True) f.request.headers["authorization"] = "foo" diff --git a/test/mitmproxy/addons/test_stickycookie.py b/test/mitmproxy/addons/test_stickycookie.py index f77d019d..4493e9cc 100644 --- a/test/mitmproxy/addons/test_stickycookie.py +++ b/test/mitmproxy/addons/test_stickycookie.py @@ -15,7 +15,7 @@ def test_domain_match(): class TestStickyCookie: def test_config(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: with pytest.raises(Exception, match="invalid filter"): tctx.configure(sc, stickycookie="~b") @@ -26,7 +26,7 @@ class TestStickyCookie: def test_simple(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") f = tflow.tflow(resp=True) f.response.headers["set-cookie"] = "foo=bar" @@ -50,7 +50,7 @@ class TestStickyCookie: def test_response(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") c = "SSID=mooo; domain=.google.com, FOO=bar; Domain=.google.com; Path=/; " \ @@ -68,7 +68,7 @@ class TestStickyCookie: def test_response_multiple(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") # Test setting of multiple cookies @@ -82,7 +82,7 @@ class TestStickyCookie: def test_response_weird(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") # Test setting of weird cookie keys @@ -100,7 +100,7 @@ class TestStickyCookie: def test_response_overwrite(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") # Test overwriting of a cookie value @@ -115,7 +115,7 @@ class TestStickyCookie: def test_response_delete(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") # Test that a cookie is be deleted @@ -127,7 +127,7 @@ class TestStickyCookie: def test_request(self): sc = stickycookie.StickyCookie() - with taddons.context() as tctx: + with taddons.context(sc) as tctx: tctx.configure(sc, stickycookie=".*") f = self._response(sc, "SSID=mooo", "www.google.com") diff --git a/test/mitmproxy/addons/test_streambodies.py b/test/mitmproxy/addons/test_streambodies.py index 426ec9ae..b980ea0b 100644 --- a/test/mitmproxy/addons/test_streambodies.py +++ b/test/mitmproxy/addons/test_streambodies.py @@ -7,7 +7,7 @@ import pytest def test_simple(): sa = streambodies.StreamBodies() - with taddons.context() as tctx: + with taddons.context(sa) as tctx: with pytest.raises(exceptions.OptionsError): tctx.configure(sa, stream_large_bodies = "invalid") tctx.configure(sa, stream_large_bodies = "10") diff --git a/test/mitmproxy/addons/test_termlog.py b/test/mitmproxy/addons/test_termlog.py index 027bdfeb..6c95df0c 100644 --- a/test/mitmproxy/addons/test_termlog.py +++ b/test/mitmproxy/addons/test_termlog.py @@ -3,7 +3,6 @@ import pytest from mitmproxy.addons import termlog from mitmproxy import log -from mitmproxy.options import Options from mitmproxy.test import taddons @@ -16,7 +15,8 @@ class TestTermLog: ]) def test_output(self, outfile, expected_out, expected_err, capfd): t = termlog.TermLog(outfile=outfile) - with taddons.context(options=Options(verbosity='info')) as tctx: + with taddons.context(t) as tctx: + tctx.options.termlog_verbosity = "info" tctx.configure(t) t.log(log.LogEntry("one", "info")) t.log(log.LogEntry("two", "debug")) diff --git a/test/mitmproxy/addons/test_upstream_auth.py b/test/mitmproxy/addons/test_upstream_auth.py index c7342bb5..ae037693 100644 --- a/test/mitmproxy/addons/test_upstream_auth.py +++ b/test/mitmproxy/addons/test_upstream_auth.py @@ -9,7 +9,7 @@ from mitmproxy.addons import upstream_auth def test_configure(): up = upstream_auth.UpstreamAuth() - with taddons.context() as tctx: + with taddons.context(up) as tctx: tctx.configure(up, upstream_auth="test:test") assert up.auth == b"Basic" + b" " + base64.b64encode(b"test:test") @@ -29,7 +29,7 @@ def test_configure(): def test_simple(): up = upstream_auth.UpstreamAuth() - with taddons.context() as tctx: + with taddons.context(up) as tctx: tctx.configure(up, upstream_auth="foo:bar") f = tflow.tflow() @@ -41,7 +41,7 @@ def test_simple(): up.requestheaders(f) assert "proxy-authorization" not in f.request.headers - tctx.configure(up, mode="reverse") + tctx.configure(up, mode="reverse:127.0.0.1") f = tflow.tflow() f.mode = "transparent" up.requestheaders(f) diff --git a/test/mitmproxy/net/http/test_cookies.py b/test/mitmproxy/net/http/test_cookies.py index e12b0f00..74233cca 100644 --- a/test/mitmproxy/net/http/test_cookies.py +++ b/test/mitmproxy/net/http/test_cookies.py @@ -143,6 +143,27 @@ def test_cookie_roundtrips(): def test_parse_set_cookie_pairs(): pairs = [ [ + "=", + [[ + ["", ""] + ]] + ], + [ + "=;foo=bar", + [[ + ["", ""], + ["foo", "bar"] + ]] + ], + [ + "=;=;foo=bar", + [[ + ["", ""], + ["", ""], + ["foo", "bar"] + ]] + ], + [ "=uno", [[ ["", "uno"] diff --git a/test/mitmproxy/net/test_tcp.py b/test/mitmproxy/net/test_tcp.py index 8c012e42..e862d0ad 100644 --- a/test/mitmproxy/net/test_tcp.py +++ b/test/mitmproxy/net/test_tcp.py @@ -485,7 +485,7 @@ class TestSSLDisconnect(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): c.convert_to_tls() - # Excercise SSL.ZeroReturnError + # Exercise SSL.ZeroReturnError c.rfile.read(10) c.close() with pytest.raises(exceptions.TcpDisconnect): diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index 194a57c9..d9aa03b4 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -10,6 +10,7 @@ import h2 from mitmproxy import options import mitmproxy.net +from mitmproxy.addons import core from ...net import tservers as net_tservers from mitmproxy import exceptions from mitmproxy.net.http import http1, http2 @@ -90,6 +91,7 @@ class _Http2TestBase: def setup_class(cls): cls.options = cls.get_options() tmaster = tservers.TestMaster(cls.options) + tmaster.addons.add(core.Core()) cls.proxy = tservers.ProxyThread(tmaster) cls.proxy.start() diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py index 5cd9601c..661605b7 100644 --- a/test/mitmproxy/proxy/protocol/test_websocket.py +++ b/test/mitmproxy/proxy/protocol/test_websocket.py @@ -6,6 +6,7 @@ import traceback from mitmproxy import options from mitmproxy import exceptions +from mitmproxy.addons import core from mitmproxy.http import HTTPFlow from mitmproxy.websocket import WebSocketFlow @@ -52,6 +53,7 @@ class _WebSocketTestBase: def setup_class(cls): cls.options = cls.get_options() tmaster = tservers.TestMaster(cls.options) + tmaster.addons.add(core.Core()) cls.proxy = tservers.ProxyThread(tmaster) cls.proxy.start() diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 87ec443a..986dfb39 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -10,7 +10,6 @@ from mitmproxy import certs from mitmproxy import exceptions from mitmproxy import http from mitmproxy import options -from mitmproxy.addons import proxyauth from mitmproxy.addons import script from mitmproxy.net import socks from mitmproxy.net import tcp @@ -306,46 +305,6 @@ class TestHTTP(tservers.HTTPProxyTest, CommonMixin): assert self.server.last_log()["request"]["first_line_format"] == "relative" -class TestHTTPAuth(tservers.HTTPProxyTest): - def test_auth(self): - self.master.addons.add(proxyauth.ProxyAuth()) - self.master.addons.trigger( - "configure", self.master.options.keys() - ) - self.master.options.proxyauth = "test:test" - assert self.pathod("202").status_code == 407 - p = self.pathoc() - with p.connect(): - ret = p.request(""" - get - 'http://localhost:%s/p/202' - h'%s'='%s' - """ % ( - self.server.port, - "Proxy-Authorization", - proxyauth.mkauth("test", "test") - )) - assert ret.status_code == 202 - - -class TestHTTPReverseAuth(tservers.ReverseProxyTest): - def test_auth(self): - self.master.addons.add(proxyauth.ProxyAuth()) - self.master.options.proxyauth = "test:test" - assert self.pathod("202").status_code == 401 - p = self.pathoc() - with p.connect(): - ret = p.request(""" - get - '/p/202' - h'%s'='%s' - """ % ( - "Authorization", - proxyauth.mkauth("test", "test") - )) - assert ret.status_code == 202 - - class TestHTTPS(tservers.HTTPProxyTest, CommonMixin, TcpMixin): ssl = True ssloptions = pathod.SSLOptions(request_client_cert=True) diff --git a/test/mitmproxy/test_addonmanager.py b/test/mitmproxy/test_addonmanager.py index 274d2d90..ad56cb22 100644 --- a/test/mitmproxy/test_addonmanager.py +++ b/test/mitmproxy/test_addonmanager.py @@ -107,7 +107,7 @@ def test_loader(): def test_simple(): - with taddons.context() as tctx: + with taddons.context(loadcore=False) as tctx: a = tctx.master.addons assert len(a) == 0 diff --git a/test/mitmproxy/test_optmanager.py b/test/mitmproxy/test_optmanager.py index d9b93227..1c49c0b8 100644 --- a/test/mitmproxy/test_optmanager.py +++ b/test/mitmproxy/test_optmanager.py @@ -351,7 +351,7 @@ def test_dump_defaults(): def test_dump_dicts(): o = options.Options() assert optmanager.dump_dicts(o) - assert optmanager.dump_dicts(o, ['http2', 'anticomp']) + assert optmanager.dump_dicts(o, ['http2', 'listen_port']) class TTypes(optmanager.OptManager): @@ -375,9 +375,13 @@ def test_make_parser(): opts.make_parser(parser, "int", short="c") opts.make_parser(parser, "seqstr", short="d") opts.make_parser(parser, "bool_on", short="e") + with pytest.raises(ValueError): opts.make_parser(parser, "unknown") + # Nonexistent options ignore + opts.make_parser(parser, "nonexistentxxx") + def test_set(): opts = TTypes() diff --git a/test/mitmproxy/tools/console/test_master.py b/test/mitmproxy/tools/console/test_master.py index 6f46ce9e..2879170d 100644 --- a/test/mitmproxy/tools/console/test_master.py +++ b/test/mitmproxy/tools/console/test_master.py @@ -1,20 +1,12 @@ import urwid from mitmproxy import options -from mitmproxy.test import tflow -from mitmproxy.test import tutils from mitmproxy.tools import console from ... import tservers -def test_options(): - assert options.Options(server_replay_kill_extra=True) - - class TestMaster(tservers.MasterTest): def mkmaster(self, **opts): - if "verbosity" not in opts: - opts["verbosity"] = 'warn' o = options.Options(**opts) m = console.master.ConsoleMaster(o) m.addons.trigger("configure", o.keys()) @@ -28,16 +20,3 @@ class TestMaster(tservers.MasterTest): except urwid.ExitMainLoop: pass assert len(m.view) == i - - def test_intercept(self): - """regression test for https://github.com/mitmproxy/mitmproxy/issues/1605""" - m = self.mkmaster(intercept="~b bar") - f = tflow.tflow(req=tutils.treq(content=b"foo")) - m.addons.handle_lifecycle("request", f) - assert not m.view[0].intercepted - f = tflow.tflow(req=tutils.treq(content=b"bar")) - m.addons.handle_lifecycle("request", f) - assert m.view[1].intercepted - f = tflow.tflow(resp=tutils.tresp(content=b"bar")) - m.addons.handle_lifecycle("request", f) - assert m.view[2].intercepted diff --git a/test/mitmproxy/tools/console/test_statusbar.py b/test/mitmproxy/tools/console/test_statusbar.py index 8522eb96..db8a63a7 100644 --- a/test/mitmproxy/tools/console/test_statusbar.py +++ b/test/mitmproxy/tools/console/test_statusbar.py @@ -3,7 +3,9 @@ from mitmproxy.tools.console import statusbar, master def test_statusbar(monkeypatch): - o = options.Options( + o = options.Options() + m = master.ConsoleMaster(o) + m.options.update( setheaders=[":~q:foo:bar"], replacements=[":~q:foo:bar"], ignore_hosts=["example.com", "example.org"], @@ -12,7 +14,7 @@ def test_statusbar(monkeypatch): view_filter="~dst example.com", stickycookie="~dst example.com", stickyauth="~dst example.com", - default_contentview="javascript", + console_default_contentview="javascript", anticache=True, anticomp=True, showhost=True, @@ -21,10 +23,8 @@ def test_statusbar(monkeypatch): upstream_cert=False, stream_large_bodies="3m", mode="transparent", - scripts=["nonexistent"], - save_stream_file="foo", ) - m = master.ConsoleMaster(o) + m.options.update(view_order='url', console_focus_follow=True) monkeypatch.setattr(m.addons.get("clientplayback"), "count", lambda: 42) monkeypatch.setattr(m.addons.get("serverplayback"), "count", lambda: 42) diff --git a/test/mitmproxy/tools/test_dump.py b/test/mitmproxy/tools/test_dump.py index 952c3f4f..f950d719 100644 --- a/test/mitmproxy/tools/test_dump.py +++ b/test/mitmproxy/tools/test_dump.py @@ -10,13 +10,13 @@ from .. import tservers class TestDumpMaster(tservers.MasterTest): - def mkmaster(self, flt, **opts): - o = options.Options(view_filter=flt, verbosity='error', flow_detail=0, **opts) + def mkmaster(self, **opts): + o = options.Options(**opts) m = dump.DumpMaster(o, with_termlog=False, with_dumper=False) return m def test_has_error(self): - m = self.mkmaster(None) + m = self.mkmaster() ent = log.LogEntry("foo", "error") ent.reply = controller.DummyReply() m.addons.trigger("log", ent) diff --git a/test/mitmproxy/tools/web/test_static_viewer.py b/test/mitmproxy/tools/web/test_static_viewer.py index cfe6cd7f..dfc45bc2 100644 --- a/test/mitmproxy/tools/web/test_static_viewer.py +++ b/test/mitmproxy/tools/web/test_static_viewer.py @@ -8,7 +8,7 @@ from mitmproxy import flowfilter from mitmproxy.tools.web.app import flow_to_json from mitmproxy.tools.web import static_viewer -from mitmproxy.addons import save +from mitmproxy.addons import save, readfile def test_save_static(tmpdir): @@ -59,8 +59,9 @@ def test_save_flows_content(ctx, tmpdir): def test_static_viewer(tmpdir): s = static_viewer.StaticViewer() + rf = readfile.ReadFile() sa = save.Save() - with taddons.context() as tctx: + with taddons.context(rf) as tctx: sa.save([tflow.tflow(resp=True)], str(tmpdir.join('foo'))) tctx.master.addons.add(s) tctx.configure(s, web_static_viewer=str(tmpdir), rfile=str(tmpdir.join('foo'))) diff --git a/test/mitmproxy/tservers.py b/test/mitmproxy/tservers.py index dd5bb327..0040b023 100644 --- a/test/mitmproxy/tservers.py +++ b/test/mitmproxy/tservers.py @@ -5,6 +5,7 @@ import sys from unittest import mock import mitmproxy.platform +from mitmproxy.addons import core from mitmproxy.proxy.config import ProxyConfig from mitmproxy.proxy.server import ProxyServer from mitmproxy import controller @@ -132,6 +133,7 @@ class ProxyTestBase: cls.options = cls.get_options() tmaster = cls.masterclass(cls.options) + tmaster.addons.add(core.Core()) cls.proxy = ProxyThread(tmaster) cls.proxy.start() @@ -222,12 +224,12 @@ class HTTPProxyTest(ProxyTestBase): p = pathod.pathoc.Pathoc( ("127.0.0.1", self.proxy.port), True, fp=None ) - with p.connect((options.APP_HOST, options.APP_PORT)): + with p.connect((self.master.options.onboarding_host, self.master.options.onbarding_port)): return p.request("get:'%s'" % page) else: p = self.pathoc() with p.connect(): - return p.request("get:'http://%s%s'" % (options.APP_HOST, page)) + return p.request("get:'http://%s%s'" % (self.master.options.onboarding_host, page)) class TransparentProxyTest(ProxyTestBase): @@ -343,6 +345,7 @@ class ChainProxyTest(ProxyTestBase): for _ in range(cls.n): opts = cls.get_options() tmaster = cls.masterclass(opts) + tmaster.addons.add(core.Core()) proxy = ProxyThread(tmaster) proxy.start() cls.chain.insert(0, proxy) |
