From 9af8f4bb31c94a25780a4189bffa406906249626 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 21 Nov 2016 02:16:20 +0100 Subject: organize examples This commit is largely based on work by Thiago Arrais (@thiagoarrais) and Shane Bradfield (@l33tLumberjack). I wasn't really able to get their PR reasonably merged onto the latest master, so I reapplied their changes manually here and did some further improvements on that. --- docs/features/responsestreaming.rst | 8 +- docs/scripting/overview.rst | 22 +-- examples/README | 31 ---- examples/README.md | 15 ++ examples/add_header.py | 2 - examples/arguments.py | 17 --- examples/change_upstream_proxy.py | 24 --- examples/classes.py | 7 - examples/complex/README.md | 19 +++ examples/complex/change_upstream_proxy.py | 24 +++ examples/complex/dns_spoofing.py | 49 ++++++ examples/complex/dup_and_replay.py | 7 + examples/complex/flowbasic.py | 43 ++++++ examples/complex/full_transparency_shim.c | 87 +++++++++++ examples/complex/har_dump.py | 219 +++++++++++++++++++++++++++ examples/complex/mitmproxywrapper.py | 165 ++++++++++++++++++++ examples/complex/nonblocking.py | 11 ++ examples/complex/remote_debug.py | 19 +++ examples/complex/sslstrip.py | 57 +++++++ examples/complex/stickycookies | 42 +++++ examples/complex/stream.py | 6 + examples/complex/stream_modify.py | 20 +++ examples/complex/tcp_message.py | 27 ++++ examples/complex/tls_passthrough.py | 140 +++++++++++++++++ examples/context_logging.py | 6 - examples/custom_contentviews.py | 70 --------- examples/dns_spoofing.py | 49 ------ examples/dup_and_replay.py | 7 - examples/fail_with_500.py | 3 - examples/flowbasic | 42 ----- examples/flowfilter.py | 21 --- examples/flowwriter.py | 22 --- examples/full_transparency_shim.c | 87 ----------- examples/har_dump.py | 219 --------------------------- examples/iframe_injector.py | 29 ---- examples/mitmproxywrapper.py | 165 -------------------- examples/modify_form.py | 8 - examples/modify_querystring.py | 2 - examples/nonblocking.py | 11 -- examples/proxapp.py | 25 --- examples/read_dumpfile | 21 --- examples/redirect_requests.py | 18 --- examples/remote_debug.py | 19 --- examples/simple/README.md | 18 +++ examples/simple/add_header.py | 2 + examples/simple/add_header_class.py | 7 + examples/simple/custom_contentview.py | 28 ++++ examples/simple/filter_flows.py | 23 +++ examples/simple/io_read_dumpfile.py | 21 +++ examples/simple/io_write_dumpfile.py | 29 ++++ examples/simple/logging.py | 12 ++ examples/simple/modify_body_inject_iframe.py | 29 ++++ examples/simple/modify_form.py | 10 ++ examples/simple/modify_querystring.py | 2 + examples/simple/redirect_requests.py | 11 ++ examples/simple/script_arguments.py | 17 +++ examples/simple/send_reply_from_proxy.py | 17 +++ examples/simple/upsidedownternet.py | 16 ++ examples/simple/wsgi_flask_app.py | 25 +++ examples/sslstrip.py | 53 ------- examples/stickycookies | 42 ----- examples/stream.py | 5 - examples/stream_modify.py | 20 --- examples/stub.py | 87 ----------- examples/tcp_message.py | 27 ---- examples/tls_passthrough.py | 140 ----------------- examples/upsidedownternet.py | 15 -- test/mitmproxy/test_examples.py | 39 ++--- 68 files changed, 1254 insertions(+), 1326 deletions(-) delete mode 100644 examples/README create mode 100644 examples/README.md delete mode 100644 examples/add_header.py delete mode 100644 examples/arguments.py delete mode 100644 examples/change_upstream_proxy.py delete mode 100644 examples/classes.py create mode 100644 examples/complex/README.md create mode 100644 examples/complex/change_upstream_proxy.py create mode 100644 examples/complex/dns_spoofing.py create mode 100644 examples/complex/dup_and_replay.py create mode 100644 examples/complex/flowbasic.py create mode 100644 examples/complex/full_transparency_shim.c create mode 100644 examples/complex/har_dump.py create mode 100644 examples/complex/mitmproxywrapper.py create mode 100644 examples/complex/nonblocking.py create mode 100644 examples/complex/remote_debug.py create mode 100644 examples/complex/sslstrip.py create mode 100644 examples/complex/stickycookies create mode 100644 examples/complex/stream.py create mode 100644 examples/complex/stream_modify.py create mode 100644 examples/complex/tcp_message.py create mode 100644 examples/complex/tls_passthrough.py delete mode 100644 examples/context_logging.py delete mode 100644 examples/custom_contentviews.py delete mode 100644 examples/dns_spoofing.py delete mode 100644 examples/dup_and_replay.py delete mode 100644 examples/fail_with_500.py delete mode 100755 examples/flowbasic delete mode 100644 examples/flowfilter.py delete mode 100644 examples/flowwriter.py delete mode 100644 examples/full_transparency_shim.c delete mode 100644 examples/har_dump.py delete mode 100644 examples/iframe_injector.py delete mode 100644 examples/mitmproxywrapper.py delete mode 100644 examples/modify_form.py delete mode 100644 examples/modify_querystring.py delete mode 100644 examples/nonblocking.py delete mode 100644 examples/proxapp.py delete mode 100644 examples/read_dumpfile delete mode 100644 examples/redirect_requests.py delete mode 100644 examples/remote_debug.py create mode 100644 examples/simple/README.md create mode 100644 examples/simple/add_header.py create mode 100644 examples/simple/add_header_class.py create mode 100644 examples/simple/custom_contentview.py create mode 100644 examples/simple/filter_flows.py create mode 100644 examples/simple/io_read_dumpfile.py create mode 100644 examples/simple/io_write_dumpfile.py create mode 100644 examples/simple/logging.py create mode 100644 examples/simple/modify_body_inject_iframe.py create mode 100644 examples/simple/modify_form.py create mode 100644 examples/simple/modify_querystring.py create mode 100644 examples/simple/redirect_requests.py create mode 100644 examples/simple/script_arguments.py create mode 100644 examples/simple/send_reply_from_proxy.py create mode 100644 examples/simple/upsidedownternet.py create mode 100644 examples/simple/wsgi_flask_app.py delete mode 100644 examples/sslstrip.py delete mode 100755 examples/stickycookies delete mode 100644 examples/stream.py delete mode 100644 examples/stream_modify.py delete mode 100644 examples/stub.py delete mode 100644 examples/tcp_message.py delete mode 100644 examples/tls_passthrough.py delete mode 100644 examples/upsidedownternet.py diff --git a/docs/features/responsestreaming.rst b/docs/features/responsestreaming.rst index 1d5726c4..6fa93271 100644 --- a/docs/features/responsestreaming.rst +++ b/docs/features/responsestreaming.rst @@ -40,8 +40,8 @@ You can also use a script to customize exactly which responses are streamed. Responses that should be tagged for streaming by setting their ``.stream`` attribute to ``True``: -.. literalinclude:: ../../examples/stream.py - :caption: examples/stream.py +.. literalinclude:: ../../examples/complex/stream.py + :caption: examples/complex/stream.py :language: python Implementation Details @@ -59,8 +59,8 @@ Modifying streamed data If the ``.stream`` attribute is callable, ``.stream`` will wrap the generator that yields all chunks. -.. literalinclude:: ../../examples/stream_modify.py - :caption: examples/stream_modify.py +.. literalinclude:: ../../examples/complex/stream_modify.py + :caption: examples/complex/stream_modify.py :language: python .. seealso:: diff --git a/docs/scripting/overview.rst b/docs/scripting/overview.rst index 7e399c9c..7df5532d 100644 --- a/docs/scripting/overview.rst +++ b/docs/scripting/overview.rst @@ -17,8 +17,8 @@ appropriate points of mitmproxy's operation. Here's a complete mitmproxy script that adds a new header to every HTTP response before it is returned to the client: -.. literalinclude:: ../../examples/add_header.py - :caption: :src:`examples/add_header.py` +.. literalinclude:: ../../examples/simple/add_header.py + :caption: :src:`examples/simple/add_header.py` :language: python All events that deal with an HTTP request get an instance of `HTTPFlow @@ -42,8 +42,8 @@ called before anything else happens. You can replace the current script object by returning it from this handler. Here's how this looks when applied to the example above: -.. literalinclude:: ../../examples/classes.py - :caption: :src:`examples/classes.py` +.. literalinclude:: ../../examples/simple/add_header_class.py + :caption: :src:`examples/simple/add_header_class.py` :language: python So here, we're using a module-level script to "boot up" into a class instance. @@ -62,13 +62,13 @@ sophisticated - replace one value with another in all responses. Mitmproxy's `_ method that takes care of all the details for us. -.. literalinclude:: ../../examples/arguments.py - :caption: :src:`examples/arguments.py` +.. literalinclude:: ../../examples/simple/script_arguments.py + :caption: :src:`examples/simple/script_arguments.py` :language: python We can now call this script on the command-line like this: ->>> mitmdump -dd -s "./arguments.py html faketml" +>>> mitmdump -dd -s "./script_arguments.py html faketml" Whenever a handler is called, mitpmroxy rewrites the script environment so that it sees its own arguments as if it was invoked from the command-line. @@ -85,8 +85,8 @@ and mitmproxy console can place script output in the event buffer. Here's how this looks: -.. literalinclude:: ../../examples/context_logging.py - :caption: :src:`examples/context_logging.py` +.. literalinclude:: ../../examples/simple/logging.py + :caption: :src:`examples/simple/logging.py` :language: python The ``ctx`` module also exposes the mitmproxy master object at ``ctx.master`` @@ -126,8 +126,8 @@ It's possible to implement a concurrent mechanism on top of the blocking framework, and mitmproxy includes a handy example of this that is fit for most purposes. You can use it as follows: -.. literalinclude:: ../../examples/nonblocking.py - :caption: :src:`examples/nonblocking.py` +.. literalinclude:: ../../examples/complex/nonblocking.py + :caption: :src:`examples/complex/nonblocking.py` :language: python diff --git a/examples/README b/examples/README deleted file mode 100644 index 90edf468..00000000 --- a/examples/README +++ /dev/null @@ -1,31 +0,0 @@ -Some inline scripts may require additional dependencies, which can be installed using -`pip install mitmproxy[examples]`. - - -# inline script examples -add_header.py Simple script that just adds a header to every request. -change_upstream_proxy.py Dynamically change the upstream proxy -dns_spoofing.py Use mitmproxy in a DNS spoofing scenario. -dup_and_replay.py Duplicates each request, changes it, and then replays the modified request. -fail_with_500.py Turn every response into an Internal Server Error. -filt.py Use mitmproxy's filter expressions in your script. -flowwriter.py Only write selected flows into a mitmproxy dumpfile. -iframe_injector.py Inject configurable iframe into pages. -modify_form.py Modify all form submissions to add a parameter. -modify_querystring.py Modify all query strings to add a parameters. -modify_response_body.py Replace arbitrary strings in all responses -nonblocking.py Demonstrate parallel processing with a blocking script. -proxapp.py How to embed a WSGI app in a mitmproxy server -redirect_requests.py Redirect requests or directly reply to them. -stub.py Script stub with a method definition for every event. -upsidedownternet.py Rewrites traffic to turn images upside down. - - -# mitmproxy examples -flowbasic Basic use of mitmproxy as a library. -stickycookies An example of writing a custom proxy with mitmproxy. - - -# misc -read_dumpfile Read a dumpfile generated by mitmproxy. -mitmproxywrapper.py Bracket mitmproxy run with proxy enable/disable on OS X diff --git a/examples/README.md b/examples/README.md new file mode 100644 index 00000000..f46f322d --- /dev/null +++ b/examples/README.md @@ -0,0 +1,15 @@ +# Mitmproxy Scripting API + +Mitmproxy has a powerful scripting API that allows you to control almost any aspect of traffic being +proxied. In fact, much of mitmproxy’s own core functionality is implemented using the exact same API +exposed to scripters (see [mitmproxy/addons](../mitmproxy/addons)). + +This directory contains some examples of the scripting API. We recommend to start with the +ones in [simple/](./simple). + +| :warning: | If you are browsing this on GitHub, make sure to select the git tag matching your mitmproxy version. | +|------------|------------------------------------------------------------------------------------------------------| + + +Some inline scripts may require additional dependencies, which can be installed using +`pip install mitmproxy[examples]`. \ No newline at end of file diff --git a/examples/add_header.py b/examples/add_header.py deleted file mode 100644 index 3e0b5f1e..00000000 --- a/examples/add_header.py +++ /dev/null @@ -1,2 +0,0 @@ -def response(flow): - flow.response.headers["newheader"] = "foo" diff --git a/examples/arguments.py b/examples/arguments.py deleted file mode 100644 index 70851192..00000000 --- a/examples/arguments.py +++ /dev/null @@ -1,17 +0,0 @@ -import argparse - - -class Replacer: - def __init__(self, src, dst): - self.src, self.dst = src, dst - - def response(self, flow): - flow.response.replace(self.src, self.dst) - - -def start(): - parser = argparse.ArgumentParser() - parser.add_argument("src", type=str) - parser.add_argument("dst", type=str) - args = parser.parse_args() - return Replacer(args.src, args.dst) diff --git a/examples/change_upstream_proxy.py b/examples/change_upstream_proxy.py deleted file mode 100644 index 49d5379f..00000000 --- a/examples/change_upstream_proxy.py +++ /dev/null @@ -1,24 +0,0 @@ -# This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy -# in upstream proxy mode. -# -# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s change_upstream_proxy.py -# -# If you want to change the target server, you should modify flow.request.host and flow.request.port - - -def proxy_address(flow): - # Poor man's loadbalancing: route every second domain through the alternative proxy. - if hash(flow.request.host) % 2 == 1: - return ("localhost", 8082) - else: - return ("localhost", 8081) - - -def request(flow): - if flow.request.method == "CONNECT": - # If the decision is done by domain, one could also modify the server address here. - # We do it after CONNECT here to have the request data available as well. - return - address = proxy_address(flow) - if flow.live: - flow.live.change_upstream_proxy_server(address) diff --git a/examples/classes.py b/examples/classes.py deleted file mode 100644 index 6443798a..00000000 --- a/examples/classes.py +++ /dev/null @@ -1,7 +0,0 @@ -class AddHeader: - def response(self, flow): - flow.response.headers["newheader"] = "foo" - - -def start(): - return AddHeader() diff --git a/examples/complex/README.md b/examples/complex/README.md new file mode 100644 index 00000000..d3b2e77a --- /dev/null +++ b/examples/complex/README.md @@ -0,0 +1,19 @@ +## Complex Examples + +| Filename | Description | +|:-------------------------|:----------------------------------------------------------------------------------------------| +| change_upstream_proxy.py | Dynamically change the upstream proxy. | +| dns_spoofing.py | Use mitmproxy in a DNS spoofing scenario. | +| dup_and_replay.py | Duplicates each request, changes it, and then replays the modified request. | +| flowbasic.py | Basic use of mitmproxy's FlowMaster directly. | +| full_transparency_shim.c | Setuid wrapper that can be used to run mitmproxy in full transparency mode, as a normal user. | +| har_dump.py | Dump flows as HAR files. | +| mitmproxywrapper.py | Bracket mitmproxy run with proxy enable/disable on OS X | +| nonblocking.py | Demonstrate parallel processing with a blocking script | +| remote_debug.py | This script enables remote debugging of the mitmproxy _UI_ with PyCharm. | +| sslstrip.py | sslstrip-like funtionality implemented with mitmproxy | +| stickycookies | An advanced example of using mitmproxy's FlowMaster directly. | +| stream | Enable streaming for all responses. | +| stream_modify.py | Modify a streamed response body. | +| tcp_message.py | Modify a raw TCP connection | +| tls_passthrough.py | Use conditional TLS interception based on a user-defined strategy. | \ No newline at end of file diff --git a/examples/complex/change_upstream_proxy.py b/examples/complex/change_upstream_proxy.py new file mode 100644 index 00000000..49d5379f --- /dev/null +++ b/examples/complex/change_upstream_proxy.py @@ -0,0 +1,24 @@ +# This scripts demonstrates how mitmproxy can switch to a second/different upstream proxy +# in upstream proxy mode. +# +# Usage: mitmdump -U http://default-upstream-proxy.local:8080/ -s change_upstream_proxy.py +# +# If you want to change the target server, you should modify flow.request.host and flow.request.port + + +def proxy_address(flow): + # Poor man's loadbalancing: route every second domain through the alternative proxy. + if hash(flow.request.host) % 2 == 1: + return ("localhost", 8082) + else: + return ("localhost", 8081) + + +def request(flow): + if flow.request.method == "CONNECT": + # If the decision is done by domain, one could also modify the server address here. + # We do it after CONNECT here to have the request data available as well. + return + address = proxy_address(flow) + if flow.live: + flow.live.change_upstream_proxy_server(address) diff --git a/examples/complex/dns_spoofing.py b/examples/complex/dns_spoofing.py new file mode 100644 index 00000000..c020047f --- /dev/null +++ b/examples/complex/dns_spoofing.py @@ -0,0 +1,49 @@ +""" +This inline scripts makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect +connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the +Host header of the HTTP request. +Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't +know the actual target and cannot construct a certificate that looks valid. +Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well. +Using transparent mode is the better option most of the time. + +Usage: + mitmproxy + -p 443 + -s dns_spoofing.py + # Used as the target location if neither SNI nor host header are present. + -R http://example.com/ + mitmdump + -p 80 + -R http://localhost:443/ + + (Setting up a single proxy instance and using iptables to redirect to it + works as well) +""" +import re + +# This regex extracts splits the host header into host and port. +# Handles the edge case of IPv6 addresses containing colons. +# https://bugzilla.mozilla.org/show_bug.cgi?id=45891 +parse_host_header = re.compile(r"^(?P[^:]+|\[.+\])(?::(?P\d+))?$") + + +def request(flow): + if flow.client_conn.ssl_established: + flow.request.scheme = "https" + sni = flow.client_conn.connection.get_servername() + port = 443 + else: + flow.request.scheme = "http" + sni = None + port = 80 + + host_header = flow.request.pretty_host + m = parse_host_header.match(host_header) + if m: + host_header = m.group("host").strip("[]") + if m.group("port"): + port = int(m.group("port")) + + flow.request.host = sni or host_header + flow.request.port = port diff --git a/examples/complex/dup_and_replay.py b/examples/complex/dup_and_replay.py new file mode 100644 index 00000000..bf7c2a4e --- /dev/null +++ b/examples/complex/dup_and_replay.py @@ -0,0 +1,7 @@ +from mitmproxy import ctx + + +def request(flow): + f = ctx.master.state.duplicate_flow(flow) + f.request.path = "/changed" + ctx.master.replay_request(f, block=True) diff --git a/examples/complex/flowbasic.py b/examples/complex/flowbasic.py new file mode 100644 index 00000000..25b0b1a9 --- /dev/null +++ b/examples/complex/flowbasic.py @@ -0,0 +1,43 @@ +#!/usr/bin/env python +""" + This example shows how to build a proxy based on mitmproxy's Flow + primitives. + + Heads Up: In the majority of cases, you want to use inline scripts. + + Note that request and response messages are not automatically replied to, + so we need to implement handlers to do this. +""" +from mitmproxy import controller, options, master +from mitmproxy.proxy import ProxyServer, ProxyConfig + + +class MyMaster(master.Master): + def run(self): + try: + master.Master.run(self) + except KeyboardInterrupt: + self.shutdown() + + @controller.handler + def request(self, f): + print("request", f) + + @controller.handler + def response(self, f): + print("response", f) + + @controller.handler + def error(self, f): + print("error", f) + + @controller.handler + def log(self, l): + print("log", l.msg) + + +opts = options.Options(cadir="~/.mitmproxy/") +config = ProxyConfig(opts) +server = ProxyServer(config) +m = MyMaster(opts, server) +m.run() diff --git a/examples/complex/full_transparency_shim.c b/examples/complex/full_transparency_shim.c new file mode 100644 index 00000000..923eea76 --- /dev/null +++ b/examples/complex/full_transparency_shim.c @@ -0,0 +1,87 @@ +#define _GNU_SOURCE +#include +#include +#include +#include +#include +#include +#include + +/* This setuid wrapper can be used to run mitmproxy in full transparency mode, as a normal user. + * It will set the required capabilities (CAP_NET_RAW), drop privileges, and will then run argv[1] + * with the same capabilities. + * + * It can be compiled as follows: + * gcc examples/mitmproxy_shim.c -o mitmproxy_shim -lcap +*/ + +int set_caps(cap_t cap_struct, cap_value_t *cap_list, size_t bufsize) { + int cap_count = bufsize / sizeof(cap_list[0]); + + if (cap_set_flag(cap_struct, CAP_PERMITTED, cap_count, cap_list, CAP_SET) || + cap_set_flag(cap_struct, CAP_EFFECTIVE, cap_count, cap_list, CAP_SET) || + cap_set_flag(cap_struct, CAP_INHERITABLE, cap_count, cap_list, CAP_SET)) { + if (cap_count < 2) { + fprintf(stderr, "Cannot manipulate capability data structure as user: %s.\n", strerror(errno)); + } else { + fprintf(stderr, "Cannot manipulate capability data structure as root: %s.\n", strerror(errno)); + } + return -1; + } + + if (cap_count < 2) { + if (prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_RAISE, CAP_NET_RAW, 0, 0)) { + fprintf(stderr, "Failed to add CAP_NET_RAW to the ambient set: %s.\n", strerror(errno)); + return -2; + } + } + + if (cap_set_proc(cap_struct)) { + if (cap_count < 2) { + fprintf(stderr, "Cannot set capabilities as user: %s.\n", strerror(errno)); + } else { + fprintf(stderr, "Cannot set capabilities as root: %s.\n", strerror(errno)); + } + return -3; + } + + if (cap_count > 1) { + if (prctl(PR_SET_KEEPCAPS, 1L)) { + fprintf(stderr, "Cannot keep capabilities after dropping privileges: %s.\n", strerror(errno)); + return -4; + } + if (cap_clear(cap_struct)) { + fprintf(stderr, "Cannot clear capability data structure: %s.\n", strerror(errno)); + return -5; + } + } +} + +int main(int argc, char **argv, char **envp) { + cap_t cap_struct = cap_init(); + cap_value_t root_caps[2] = { CAP_NET_RAW, CAP_SETUID }; + cap_value_t user_caps[1] = { CAP_NET_RAW }; + uid_t user = getuid(); + int res; + + if (setresuid(0, 0, 0)) { + fprintf(stderr, "Cannot switch to root: %s.\n", strerror(errno)); + return 1; + } + + if (res = set_caps(cap_struct, root_caps, sizeof(root_caps))) + return res; + + if (setresuid(user, user, user)) { + fprintf(stderr, "Cannot drop root privileges: %s.\n", strerror(errno)); + return 2; + } + + if (res = set_caps(cap_struct, user_caps, sizeof(user_caps))) + return res; + + if (execve(argv[1], argv + 1, envp)) { + fprintf(stderr, "Failed to execute %s: %s\n", argv[1], strerror(errno)); + return 3; + } +} diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py new file mode 100644 index 00000000..aeb154d2 --- /dev/null +++ b/examples/complex/har_dump.py @@ -0,0 +1,219 @@ +""" +This inline script can be used to dump flows as HAR files. +""" + + +import json +import sys +import base64 +import zlib + +from datetime import datetime +import pytz + +import mitmproxy + +from mitmproxy import version +from mitmproxy.utils import strutils +from mitmproxy.net.http import cookies + +HAR = {} + +# A list of server seen till now is maintained so we can avoid +# using 'connect' time for entries that use an existing connection. +SERVERS_SEEN = set() + + +def start(): + """ + Called once on script startup before any other events. + """ + if len(sys.argv) != 2: + raise ValueError( + 'Usage: -s "har_dump.py filename" ' + '(- will output to stdout, filenames ending with .zhar ' + 'will result in compressed har)' + ) + + HAR.update({ + "log": { + "version": "1.2", + "creator": { + "name": "mitmproxy har_dump", + "version": "0.1", + "comment": "mitmproxy version %s" % version.MITMPROXY + }, + "entries": [] + } + }) + + +def response(flow): + """ + Called when a server response has been received. + """ + + # -1 indicates that these values do not apply to current request + ssl_time = -1 + connect_time = -1 + + if flow.server_conn and flow.server_conn not in SERVERS_SEEN: + connect_time = (flow.server_conn.timestamp_tcp_setup - + flow.server_conn.timestamp_start) + + if flow.server_conn.timestamp_ssl_setup is not None: + ssl_time = (flow.server_conn.timestamp_ssl_setup - + flow.server_conn.timestamp_tcp_setup) + + SERVERS_SEEN.add(flow.server_conn) + + # Calculate raw timings from timestamps. DNS timings can not be calculated + # for lack of a way to measure it. The same goes for HAR blocked. + # mitmproxy will open a server connection as soon as it receives the host + # and port from the client connection. So, the time spent waiting is actually + # spent waiting between request.timestamp_end and response.timestamp_start + # thus it correlates to HAR wait instead. + timings_raw = { + 'send': flow.request.timestamp_end - flow.request.timestamp_start, + 'receive': flow.response.timestamp_end - flow.response.timestamp_start, + 'wait': flow.response.timestamp_start - flow.request.timestamp_end, + 'connect': connect_time, + 'ssl': ssl_time, + } + + # HAR timings are integers in ms, so we re-encode the raw timings to that format. + timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()]) + + # full_time is the sum of all timings. + # Timings set to -1 will be ignored as per spec. + full_time = sum(v for v in timings.values() if v > -1) + + started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start)) + + # Response body size and encoding + response_body_size = len(flow.response.raw_content) + response_body_decoded_size = len(flow.response.content) + response_body_compression = response_body_decoded_size - response_body_size + + entry = { + "startedDateTime": started_date_time, + "time": full_time, + "request": { + "method": flow.request.method, + "url": flow.request.url, + "httpVersion": flow.request.http_version, + "cookies": format_request_cookies(flow.request.cookies.fields), + "headers": name_value(flow.request.headers), + "queryString": name_value(flow.request.query or {}), + "headersSize": len(str(flow.request.headers)), + "bodySize": len(flow.request.content), + }, + "response": { + "status": flow.response.status_code, + "statusText": flow.response.reason, + "httpVersion": flow.response.http_version, + "cookies": format_response_cookies(flow.response.cookies.fields), + "headers": name_value(flow.response.headers), + "content": { + "size": response_body_size, + "compression": response_body_compression, + "mimeType": flow.response.headers.get('Content-Type', '') + }, + "redirectURL": flow.response.headers.get('Location', ''), + "headersSize": len(str(flow.response.headers)), + "bodySize": response_body_size, + }, + "cache": {}, + "timings": timings, + } + + # Store binary data as base64 + if strutils.is_mostly_bin(flow.response.content): + entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode() + entry["response"]["content"]["encoding"] = "base64" + else: + entry["response"]["content"]["text"] = flow.response.get_text(strict=False) + + if flow.request.method in ["POST", "PUT", "PATCH"]: + params = [ + {"name": a.decode("utf8", "surrogateescape"), "value": b.decode("utf8", "surrogateescape")} + for a, b in flow.request.urlencoded_form.items(multi=True) + ] + entry["request"]["postData"] = { + "mimeType": flow.request.headers.get("Content-Type", ""), + "text": flow.request.get_text(strict=False), + "params": params + } + + if flow.server_conn.connected(): + entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0]) + + HAR["log"]["entries"].append(entry) + + +def done(): + """ + Called once on script shutdown, after any other events. + """ + dump_file = sys.argv[1] + + json_dump = json.dumps(HAR, indent=2) # type: str + + if dump_file == '-': + mitmproxy.ctx.log(json_dump) + else: + raw = json_dump.encode() # type: bytes + if dump_file.endswith('.zhar'): + raw = zlib.compress(raw, 9) + + with open(dump_file, "wb") as f: + f.write(raw) + + mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump)) + + +def format_datetime(dt): + return dt.replace(tzinfo=pytz.timezone("UTC")).isoformat() + + +def format_cookies(cookie_list): + rv = [] + + for name, value, attrs in cookie_list: + cookie_har = { + "name": name, + "value": value, + } + + # HAR only needs some attributes + for key in ["path", "domain", "comment"]: + if key in attrs: + cookie_har[key] = attrs[key] + + # These keys need to be boolean! + for key in ["httpOnly", "secure"]: + cookie_har[key] = bool(key in attrs) + + # Expiration time needs to be formatted + expire_ts = cookies.get_expiration_ts(attrs) + if expire_ts is not None: + cookie_har["expires"] = format_datetime(datetime.fromtimestamp(expire_ts)) + + rv.append(cookie_har) + + return rv + + +def format_request_cookies(fields): + return format_cookies(cookies.group_cookies(fields)) + + +def format_response_cookies(fields): + return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields) + + +def name_value(obj): + """ + Convert (key, value) pairs to HAR format. + """ + return [{"name": k, "value": v} for k, v in obj.items()] diff --git a/examples/complex/mitmproxywrapper.py b/examples/complex/mitmproxywrapper.py new file mode 100644 index 00000000..eade0fe2 --- /dev/null +++ b/examples/complex/mitmproxywrapper.py @@ -0,0 +1,165 @@ +#!/usr/bin/env python +# +# Helper tool to enable/disable OS X proxy and wrap mitmproxy +# +# Get usage information with: +# +# mitmproxywrapper.py -h +# + +import subprocess +import re +import argparse +import contextlib +import os +import sys + + +class Wrapper: + def __init__(self, port, extra_arguments=None): + self.port = port + self.extra_arguments = extra_arguments + + def run_networksetup_command(self, *arguments): + return subprocess.check_output( + ['sudo', 'networksetup'] + list(arguments)) + + def proxy_state_for_service(self, service): + state = self.run_networksetup_command( + '-getwebproxy', + service).splitlines() + return dict([re.findall(r'([^:]+): (.*)', line)[0] for line in state]) + + def enable_proxy_for_service(self, service): + print('Enabling proxy on {}...'.format(service)) + for subcommand in ['-setwebproxy', '-setsecurewebproxy']: + self.run_networksetup_command( + subcommand, service, '127.0.0.1', str( + self.port)) + + def disable_proxy_for_service(self, service): + print('Disabling proxy on {}...'.format(service)) + for subcommand in ['-setwebproxystate', '-setsecurewebproxystate']: + self.run_networksetup_command(subcommand, service, 'Off') + + def interface_name_to_service_name_map(self): + order = self.run_networksetup_command('-listnetworkserviceorder') + mapping = re.findall( + r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$', + order, + re.MULTILINE) + return dict([(b, a) for (a, b) in mapping]) + + def run_command_with_input(self, command, input): + popen = subprocess.Popen( + command, + stdin=subprocess.PIPE, + stdout=subprocess.PIPE) + (stdout, stderr) = popen.communicate(input) + return stdout + + def primary_interace_name(self): + scutil_script = 'get State:/Network/Global/IPv4\nd.show\n' + stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script) + interface, = re.findall(r'PrimaryInterface\s*:\s*(.+)', stdout) + return interface + + def primary_service_name(self): + return self.interface_name_to_service_name_map()[ + self.primary_interace_name()] + + def proxy_enabled_for_service(self, service): + return self.proxy_state_for_service(service)['Enabled'] == 'Yes' + + def toggle_proxy(self): + new_state = not self.proxy_enabled_for_service( + self.primary_service_name()) + for service_name in self.connected_service_names(): + if self.proxy_enabled_for_service(service_name) and not new_state: + self.disable_proxy_for_service(service_name) + elif not self.proxy_enabled_for_service(service_name) and new_state: + self.enable_proxy_for_service(service_name) + + def connected_service_names(self): + scutil_script = 'list\n' + stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script) + service_ids = re.findall(r'State:/Network/Service/(.+)/IPv4', stdout) + + service_names = [] + for service_id in service_ids: + scutil_script = 'show Setup:/Network/Service/{}\n'.format( + service_id) + stdout = self.run_command_with_input( + '/usr/sbin/scutil', + scutil_script) + service_name, = re.findall(r'UserDefinedName\s*:\s*(.+)', stdout) + service_names.append(service_name) + + return service_names + + def wrap_mitmproxy(self): + with self.wrap_proxy(): + cmd = ['mitmproxy', '-p', str(self.port)] + if self.extra_arguments: + cmd.extend(self.extra_arguments) + subprocess.check_call(cmd) + + def wrap_honeyproxy(self): + with self.wrap_proxy(): + popen = subprocess.Popen('honeyproxy.sh') + try: + popen.wait() + except KeyboardInterrupt: + popen.terminate() + + @contextlib.contextmanager + def wrap_proxy(self): + connected_service_names = self.connected_service_names() + for service_name in connected_service_names: + if not self.proxy_enabled_for_service(service_name): + self.enable_proxy_for_service(service_name) + + yield + + for service_name in connected_service_names: + if self.proxy_enabled_for_service(service_name): + self.disable_proxy_for_service(service_name) + + @classmethod + def ensure_superuser(cls): + if os.getuid() != 0: + print('Relaunching with sudo...') + os.execv('/usr/bin/sudo', ['/usr/bin/sudo'] + sys.argv) + + @classmethod + def main(cls): + parser = argparse.ArgumentParser( + description='Helper tool for OS X proxy configuration and mitmproxy.', + epilog='Any additional arguments will be passed on unchanged to mitmproxy.') + parser.add_argument( + '-t', + '--toggle', + action='store_true', + help='just toggle the proxy configuration') + # parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy') + parser.add_argument( + '-p', + '--port', + type=int, + help='override the default port of 8080', + default=8080) + args, extra_arguments = parser.parse_known_args() + + wrapper = cls(port=args.port, extra_arguments=extra_arguments) + + if args.toggle: + wrapper.toggle_proxy() + # elif args.honeyproxy: + # wrapper.wrap_honeyproxy() + else: + wrapper.wrap_mitmproxy() + + +if __name__ == '__main__': + Wrapper.ensure_superuser() + Wrapper.main() diff --git a/examples/complex/nonblocking.py b/examples/complex/nonblocking.py new file mode 100644 index 00000000..264a1fdb --- /dev/null +++ b/examples/complex/nonblocking.py @@ -0,0 +1,11 @@ +import time + +from mitmproxy.script import concurrent + + +@concurrent # Remove this and see what happens +def request(flow): + # You don't want to use mitmproxy.ctx from a different thread + print("handle request: %s%s" % (flow.request.host, flow.request.path)) + time.sleep(5) + print("start request: %s%s" % (flow.request.host, flow.request.path)) diff --git a/examples/complex/remote_debug.py b/examples/complex/remote_debug.py new file mode 100644 index 00000000..fb864f78 --- /dev/null +++ b/examples/complex/remote_debug.py @@ -0,0 +1,19 @@ +""" +This script enables remote debugging of the mitmproxy *UI* with PyCharm. +For general debugging purposes, it is easier to just debug mitmdump within PyCharm. + +Usage: + - pip install pydevd on the mitmproxy machine + - Open the Run/Debug Configuration dialog box in PyCharm, and select the Python Remote Debug configuration type. + - Debugging works in the way that mitmproxy connects to the debug server on startup. + Specify host and port that mitmproxy can use to reach your PyCharm instance on startup. + - Adjust this inline script accordingly. + - Start debug server in PyCharm + - Set breakpoints + - Start mitmproxy -s remote_debug.py +""" + + +def start(): + import pydevd + pydevd.settrace("localhost", port=5678, stdoutToServer=True, stderrToServer=True) diff --git a/examples/complex/sslstrip.py b/examples/complex/sslstrip.py new file mode 100644 index 00000000..2f60c8b9 --- /dev/null +++ b/examples/complex/sslstrip.py @@ -0,0 +1,57 @@ +""" +This script implements an sslstrip-like attack based on mitmproxy. +https://moxie.org/software/sslstrip/ +""" +import re +import urllib + +# set of SSL/TLS capable hosts +secure_hosts = set() + + +def request(flow): + flow.request.headers.pop('If-Modified-Since', None) + flow.request.headers.pop('Cache-Control', None) + + # do not force https redirection + flow.request.headers.pop('Upgrade-Insecure-Requests', None) + + # proxy connections to SSL-enabled hosts + if flow.request.pretty_host in secure_hosts: + flow.request.scheme = 'https' + flow.request.port = 443 + + # We need to update the request destination to whatever is specified in the host header: + # Having no TLS Server Name Indication from the client and just an IP address as request.host + # in transparent mode, TLS server name certificate validation would fail. + flow.request.host = flow.request.pretty_host + + +def response(flow): + flow.response.headers.pop('Strict-Transport-Security', None) + flow.response.headers.pop('Public-Key-Pins', None) + + # strip links in response body + flow.response.content = flow.response.content.replace(b'https://', b'http://') + + # strip meta tag upgrade-insecure-requests in response body + csp_meta_tag_pattern = b'' + flow.response.content = re.sub(csp_meta_tag_pattern, b'', flow.response.content, flags=re.IGNORECASE) + + # strip links in 'Location' header + if flow.response.headers.get('Location', '').startswith('https://'): + location = flow.response.headers['Location'] + hostname = urllib.parse.urlparse(location).hostname + if hostname: + secure_hosts.add(hostname) + flow.response.headers['Location'] = location.replace('https://', 'http://', 1) + + # strip upgrade-insecure-requests in Content-Security-Policy header + if re.search('upgrade-insecure-requests', flow.response.headers.get('Content-Security-Policy', ''), flags=re.IGNORECASE): + csp = flow.response.headers['Content-Security-Policy'] + flow.response.headers['Content-Security-Policy'] = re.sub('upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) + + # strip secure flag from 'Set-Cookie' headers + cookies = flow.response.headers.get_all('Set-Cookie') + cookies = [re.sub(r';\s*secure\s*', '', s) for s in cookies] + flow.response.headers.set_all('Set-Cookie', cookies) diff --git a/examples/complex/stickycookies b/examples/complex/stickycookies new file mode 100644 index 00000000..4631fa73 --- /dev/null +++ b/examples/complex/stickycookies @@ -0,0 +1,42 @@ +#!/usr/bin/env python +""" +This example builds on mitmproxy's base proxying infrastructure to +implement functionality similar to the "sticky cookies" option. + +Heads Up: In the majority of cases, you want to use inline scripts. +""" +import os +from mitmproxy import controller, proxy, master +from mitmproxy.proxy.server import ProxyServer + + +class StickyMaster(master.Master): + def __init__(self, server): + master.Master.__init__(self, server) + self.stickyhosts = {} + + def run(self): + try: + return master.Master.run(self) + except KeyboardInterrupt: + self.shutdown() + + @controller.handler + def request(self, flow): + hid = (flow.request.host, flow.request.port) + if "cookie" in flow.request.headers: + self.stickyhosts[hid] = flow.request.headers.get_all("cookie") + elif hid in self.stickyhosts: + flow.request.headers.set_all("cookie", self.stickyhosts[hid]) + + @controller.handler + def response(self, flow): + hid = (flow.request.host, flow.request.port) + if "set-cookie" in flow.response.headers: + self.stickyhosts[hid] = flow.response.headers.get_all("set-cookie") + + +config = proxy.ProxyConfig(port=8080) +server = ProxyServer(config) +m = StickyMaster(server) +m.run() diff --git a/examples/complex/stream.py b/examples/complex/stream.py new file mode 100644 index 00000000..1993cf7f --- /dev/null +++ b/examples/complex/stream.py @@ -0,0 +1,6 @@ +def responseheaders(flow): + """ + Enables streaming for all responses. + This is equivalent to passing `--stream 0` to mitmproxy. + """ + flow.response.stream = True diff --git a/examples/complex/stream_modify.py b/examples/complex/stream_modify.py new file mode 100644 index 00000000..5e5da95b --- /dev/null +++ b/examples/complex/stream_modify.py @@ -0,0 +1,20 @@ +""" +This inline script modifies a streamed response. +If you do not need streaming, see the modify_response_body example. +Be aware that content replacement isn't trivial: + - If the transfer encoding isn't chunked, you cannot simply change the content length. + - If you want to replace all occurences of "foobar", make sure to catch the cases + where one chunk ends with [...]foo" and the next starts with "bar[...]. +""" + + +def modify(chunks): + """ + chunks is a generator that can be used to iterate over all chunks. + """ + for chunk in chunks: + yield chunk.replace("foo", "bar") + + +def responseheaders(flow): + flow.response.stream = modify diff --git a/examples/complex/tcp_message.py b/examples/complex/tcp_message.py new file mode 100644 index 00000000..d7c9c42e --- /dev/null +++ b/examples/complex/tcp_message.py @@ -0,0 +1,27 @@ +""" +tcp_message Inline Script Hook API Demonstration +------------------------------------------------ + +* modifies packets containing "foo" to "bar" +* prints various details for each packet. + +example cmdline invocation: +mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py +""" +from mitmproxy.utils import strutils + + +def tcp_message(tcp_msg): + modified_msg = tcp_msg.message.replace("foo", "bar") + + is_modified = False if modified_msg == tcp_msg.message else True + tcp_msg.message = modified_msg + + print( + "[tcp_message{}] from {} {} to {} {}:\r\n{}".format( + " (modified)" if is_modified else "", + "client" if tcp_msg.sender == tcp_msg.client_conn else "server", + tcp_msg.sender.address, + "server" if tcp_msg.receiver == tcp_msg.server_conn else "client", + tcp_msg.receiver.address, strutils.bytes_to_escaped_str(tcp_msg.message)) + ) diff --git a/examples/complex/tls_passthrough.py b/examples/complex/tls_passthrough.py new file mode 100644 index 00000000..40c1051d --- /dev/null +++ b/examples/complex/tls_passthrough.py @@ -0,0 +1,140 @@ +""" +This inline script allows conditional TLS Interception based +on a user-defined strategy. + +Example: + + > mitmdump -s tls_passthrough.py + + 1. curl --proxy http://localhost:8080 https://example.com --insecure + // works - we'll also see the contents in mitmproxy + + 2. curl --proxy http://localhost:8080 https://example.com --insecure + // still works - we'll also see the contents in mitmproxy + + 3. curl --proxy http://localhost:8080 https://example.com + // fails with a certificate error, which we will also see in mitmproxy + + 4. curl --proxy http://localhost:8080 https://example.com + // works again, but mitmproxy does not intercept and we do *not* see the contents + +Authors: Maximilian Hils, Matthew Tuusberg +""" +import collections +import random + +import sys +from enum import Enum + +import mitmproxy +from mitmproxy.exceptions import TlsProtocolException +from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer + + +class InterceptionResult(Enum): + success = True + failure = False + skipped = None + + +class _TlsStrategy: + """ + Abstract base class for interception strategies. + """ + + def __init__(self): + # A server_address -> interception results mapping + self.history = collections.defaultdict(lambda: collections.deque(maxlen=200)) + + def should_intercept(self, server_address): + """ + Returns: + True, if we should attempt to intercept the connection. + False, if we want to employ pass-through instead. + """ + raise NotImplementedError() + + def record_success(self, server_address): + self.history[server_address].append(InterceptionResult.success) + + def record_failure(self, server_address): + self.history[server_address].append(InterceptionResult.failure) + + def record_skipped(self, server_address): + self.history[server_address].append(InterceptionResult.skipped) + + +class ConservativeStrategy(_TlsStrategy): + """ + Conservative Interception Strategy - only intercept if there haven't been any failed attempts + in the history. + """ + + def should_intercept(self, server_address): + if InterceptionResult.failure in self.history[server_address]: + return False + return True + + +class ProbabilisticStrategy(_TlsStrategy): + """ + Fixed probability that we intercept a given connection. + """ + + def __init__(self, p): + self.p = p + super(ProbabilisticStrategy, self).__init__() + + def should_intercept(self, server_address): + return random.uniform(0, 1) < self.p + + +class TlsFeedback(TlsLayer): + """ + Monkey-patch _establish_tls_with_client to get feedback if TLS could be established + successfully on the client connection (which may fail due to cert pinning). + """ + + def _establish_tls_with_client(self): + server_address = self.server_conn.address + + try: + super(TlsFeedback, self)._establish_tls_with_client() + except TlsProtocolException as e: + tls_strategy.record_failure(server_address) + raise e + else: + tls_strategy.record_success(server_address) + + +# inline script hooks below. + +tls_strategy = None + + +def start(): + global tls_strategy + if len(sys.argv) == 2: + tls_strategy = ProbabilisticStrategy(float(sys.argv[1])) + else: + tls_strategy = ConservativeStrategy() + + +def next_layer(next_layer): + """ + This hook does the actual magic - if the next layer is planned to be a TLS layer, + we check if we want to enter pass-through mode instead. + """ + if isinstance(next_layer, TlsLayer) and next_layer._client_tls: + server_address = next_layer.server_conn.address + + if tls_strategy.should_intercept(server_address): + # We try to intercept. + # Monkey-Patch the layer to get feedback from the TLSLayer if interception worked. + next_layer.__class__ = TlsFeedback + else: + # We don't intercept - reply with a pass-through layer and add a "skipped" entry. + mitmproxy.ctx.log("TLS passthrough for %s" % repr(next_layer.server_conn.address), "info") + next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) + next_layer.reply.send(next_layer_replacement) + tls_strategy.record_skipped(server_address) diff --git a/examples/context_logging.py b/examples/context_logging.py deleted file mode 100644 index dccfd8b2..00000000 --- a/examples/context_logging.py +++ /dev/null @@ -1,6 +0,0 @@ -from mitmproxy import ctx - - -def start(): - ctx.log.info("This is some informative text.") - ctx.log.error("This is an error.") diff --git a/examples/custom_contentviews.py b/examples/custom_contentviews.py deleted file mode 100644 index 3558eaca..00000000 --- a/examples/custom_contentviews.py +++ /dev/null @@ -1,70 +0,0 @@ -import string -import lxml.html -import lxml.etree -from mitmproxy import contentviews -from mitmproxy.utils import strutils - - -class ViewPigLatin(contentviews.View): - name = "pig_latin_HTML" - prompt = ("pig latin HTML", "l") - content_types = ["text/html"] - - def __call__(self, data, **metadata): - if strutils.is_xml(data): - parser = lxml.etree.HTMLParser( - strip_cdata=True, - remove_blank_text=True - ) - d = lxml.html.fromstring(data, parser=parser) - docinfo = d.getroottree().docinfo - - def piglify(src): - words = src.split() - ret = '' - for word in words: - idx = -1 - while word[idx] in string.punctuation and (idx * -1) != len(word): - idx -= 1 - if word[0].lower() in 'aeiou': - if idx == -1: - ret += word[0:] + "hay" - else: - ret += word[0:len(word) + idx + 1] + "hay" + word[idx + 1:] - else: - if idx == -1: - ret += word[1:] + word[0] + "ay" - else: - ret += word[1:len(word) + idx + 1] + word[0] + "ay" + word[idx + 1:] - ret += ' ' - return ret.strip() - - def recurse(root): - if hasattr(root, 'text') and root.text: - root.text = piglify(root.text) - if hasattr(root, 'tail') and root.tail: - root.tail = piglify(root.tail) - - if len(root): - for child in root: - recurse(child) - - recurse(d) - - s = lxml.etree.tostring( - d, - pretty_print=True, - doctype=docinfo.doctype - ) - return "HTML", contentviews.format_text(s) - - -pig_view = ViewPigLatin() - - -def start(): - contentviews.add(pig_view) - - -def done(): - contentviews.remove(pig_view) diff --git a/examples/dns_spoofing.py b/examples/dns_spoofing.py deleted file mode 100644 index c020047f..00000000 --- a/examples/dns_spoofing.py +++ /dev/null @@ -1,49 +0,0 @@ -""" -This inline scripts makes it possible to use mitmproxy in scenarios where IP spoofing has been used to redirect -connections to mitmproxy. The way this works is that we rely on either the TLS Server Name Indication (SNI) or the -Host header of the HTTP request. -Of course, this is not foolproof - if an HTTPS connection comes without SNI, we don't -know the actual target and cannot construct a certificate that looks valid. -Similarly, if there's no Host header or a spoofed Host header, we're out of luck as well. -Using transparent mode is the better option most of the time. - -Usage: - mitmproxy - -p 443 - -s dns_spoofing.py - # Used as the target location if neither SNI nor host header are present. - -R http://example.com/ - mitmdump - -p 80 - -R http://localhost:443/ - - (Setting up a single proxy instance and using iptables to redirect to it - works as well) -""" -import re - -# This regex extracts splits the host header into host and port. -# Handles the edge case of IPv6 addresses containing colons. -# https://bugzilla.mozilla.org/show_bug.cgi?id=45891 -parse_host_header = re.compile(r"^(?P[^:]+|\[.+\])(?::(?P\d+))?$") - - -def request(flow): - if flow.client_conn.ssl_established: - flow.request.scheme = "https" - sni = flow.client_conn.connection.get_servername() - port = 443 - else: - flow.request.scheme = "http" - sni = None - port = 80 - - host_header = flow.request.pretty_host - m = parse_host_header.match(host_header) - if m: - host_header = m.group("host").strip("[]") - if m.group("port"): - port = int(m.group("port")) - - flow.request.host = sni or host_header - flow.request.port = port diff --git a/examples/dup_and_replay.py b/examples/dup_and_replay.py deleted file mode 100644 index bf7c2a4e..00000000 --- a/examples/dup_and_replay.py +++ /dev/null @@ -1,7 +0,0 @@ -from mitmproxy import ctx - - -def request(flow): - f = ctx.master.state.duplicate_flow(flow) - f.request.path = "/changed" - ctx.master.replay_request(f, block=True) diff --git a/examples/fail_with_500.py b/examples/fail_with_500.py deleted file mode 100644 index 9710f74a..00000000 --- a/examples/fail_with_500.py +++ /dev/null @@ -1,3 +0,0 @@ -def response(flow): - flow.response.status_code = 500 - flow.response.content = b"" diff --git a/examples/flowbasic b/examples/flowbasic deleted file mode 100755 index cb1e4ea4..00000000 --- a/examples/flowbasic +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -""" - This example shows how to build a proxy based on mitmproxy's Flow - primitives. - - Heads Up: In the majority of cases, you want to use inline scripts. - - Note that request and response messages are not automatically replied to, - so we need to implement handlers to do this. -""" -from mitmproxy import controller, options, master -from mitmproxy.proxy import ProxyServer, ProxyConfig - - -class MyMaster(master.Master): - def run(self): - try: - master.Master.run(self) - except KeyboardInterrupt: - self.shutdown() - - @controller.handler - def request(self, f): - print("request", f) - - @controller.handler - def response(self, f): - print("response", f) - - @controller.handler - def error(self, f): - print("error", f) - - @controller.handler - def log(self, l): - print("log", l.msg) - -opts = options.Options(cadir="~/.mitmproxy/") -config = ProxyConfig(opts) -server = ProxyServer(config) -m = MyMaster(opts, server) -m.run() diff --git a/examples/flowfilter.py b/examples/flowfilter.py deleted file mode 100644 index 34d97275..00000000 --- a/examples/flowfilter.py +++ /dev/null @@ -1,21 +0,0 @@ -# This scripts demonstrates how to use mitmproxy's filter pattern in scripts. -# Usage: mitmdump -s "flowfilter.py FILTER" - -import sys -from mitmproxy import flowfilter - - -class Filter: - def __init__(self, spec): - self.filter = flowfilter.parse(spec) - - def response(self, flow): - if flowfilter.match(self.filter, flow): - print("Flow matches filter:") - print(flow) - - -def start(): - if len(sys.argv) != 2: - raise ValueError("Usage: -s 'filt.py FILTER'") - return Filter(sys.argv[1]) diff --git a/examples/flowwriter.py b/examples/flowwriter.py deleted file mode 100644 index a9768542..00000000 --- a/examples/flowwriter.py +++ /dev/null @@ -1,22 +0,0 @@ -import random -import sys -from mitmproxy import io - - -class Writer: - def __init__(self, path): - if path == "-": - f = sys.stdout - else: - f = open(path, "wb") - self.w = io.FlowWriter(f) - - def response(self, flow): - if random.choice([True, False]): - self.w.add(flow) - - -def start(): - if len(sys.argv) != 2: - raise ValueError('Usage: -s "flowriter.py filename"') - return Writer(sys.argv[1]) diff --git a/examples/full_transparency_shim.c b/examples/full_transparency_shim.c deleted file mode 100644 index 923eea76..00000000 --- a/examples/full_transparency_shim.c +++ /dev/null @@ -1,87 +0,0 @@ -#define _GNU_SOURCE -#include -#include -#include -#include -#include -#include -#include - -/* This setuid wrapper can be used to run mitmproxy in full transparency mode, as a normal user. - * It will set the required capabilities (CAP_NET_RAW), drop privileges, and will then run argv[1] - * with the same capabilities. - * - * It can be compiled as follows: - * gcc examples/mitmproxy_shim.c -o mitmproxy_shim -lcap -*/ - -int set_caps(cap_t cap_struct, cap_value_t *cap_list, size_t bufsize) { - int cap_count = bufsize / sizeof(cap_list[0]); - - if (cap_set_flag(cap_struct, CAP_PERMITTED, cap_count, cap_list, CAP_SET) || - cap_set_flag(cap_struct, CAP_EFFECTIVE, cap_count, cap_list, CAP_SET) || - cap_set_flag(cap_struct, CAP_INHERITABLE, cap_count, cap_list, CAP_SET)) { - if (cap_count < 2) { - fprintf(stderr, "Cannot manipulate capability data structure as user: %s.\n", strerror(errno)); - } else { - fprintf(stderr, "Cannot manipulate capability data structure as root: %s.\n", strerror(errno)); - } - return -1; - } - - if (cap_count < 2) { - if (prctl(PR_CAP_AMBIENT, PR_CAP_AMBIENT_RAISE, CAP_NET_RAW, 0, 0)) { - fprintf(stderr, "Failed to add CAP_NET_RAW to the ambient set: %s.\n", strerror(errno)); - return -2; - } - } - - if (cap_set_proc(cap_struct)) { - if (cap_count < 2) { - fprintf(stderr, "Cannot set capabilities as user: %s.\n", strerror(errno)); - } else { - fprintf(stderr, "Cannot set capabilities as root: %s.\n", strerror(errno)); - } - return -3; - } - - if (cap_count > 1) { - if (prctl(PR_SET_KEEPCAPS, 1L)) { - fprintf(stderr, "Cannot keep capabilities after dropping privileges: %s.\n", strerror(errno)); - return -4; - } - if (cap_clear(cap_struct)) { - fprintf(stderr, "Cannot clear capability data structure: %s.\n", strerror(errno)); - return -5; - } - } -} - -int main(int argc, char **argv, char **envp) { - cap_t cap_struct = cap_init(); - cap_value_t root_caps[2] = { CAP_NET_RAW, CAP_SETUID }; - cap_value_t user_caps[1] = { CAP_NET_RAW }; - uid_t user = getuid(); - int res; - - if (setresuid(0, 0, 0)) { - fprintf(stderr, "Cannot switch to root: %s.\n", strerror(errno)); - return 1; - } - - if (res = set_caps(cap_struct, root_caps, sizeof(root_caps))) - return res; - - if (setresuid(user, user, user)) { - fprintf(stderr, "Cannot drop root privileges: %s.\n", strerror(errno)); - return 2; - } - - if (res = set_caps(cap_struct, user_caps, sizeof(user_caps))) - return res; - - if (execve(argv[1], argv + 1, envp)) { - fprintf(stderr, "Failed to execute %s: %s\n", argv[1], strerror(errno)); - return 3; - } -} diff --git a/examples/har_dump.py b/examples/har_dump.py deleted file mode 100644 index aeb154d2..00000000 --- a/examples/har_dump.py +++ /dev/null @@ -1,219 +0,0 @@ -""" -This inline script can be used to dump flows as HAR files. -""" - - -import json -import sys -import base64 -import zlib - -from datetime import datetime -import pytz - -import mitmproxy - -from mitmproxy import version -from mitmproxy.utils import strutils -from mitmproxy.net.http import cookies - -HAR = {} - -# A list of server seen till now is maintained so we can avoid -# using 'connect' time for entries that use an existing connection. -SERVERS_SEEN = set() - - -def start(): - """ - Called once on script startup before any other events. - """ - if len(sys.argv) != 2: - raise ValueError( - 'Usage: -s "har_dump.py filename" ' - '(- will output to stdout, filenames ending with .zhar ' - 'will result in compressed har)' - ) - - HAR.update({ - "log": { - "version": "1.2", - "creator": { - "name": "mitmproxy har_dump", - "version": "0.1", - "comment": "mitmproxy version %s" % version.MITMPROXY - }, - "entries": [] - } - }) - - -def response(flow): - """ - Called when a server response has been received. - """ - - # -1 indicates that these values do not apply to current request - ssl_time = -1 - connect_time = -1 - - if flow.server_conn and flow.server_conn not in SERVERS_SEEN: - connect_time = (flow.server_conn.timestamp_tcp_setup - - flow.server_conn.timestamp_start) - - if flow.server_conn.timestamp_ssl_setup is not None: - ssl_time = (flow.server_conn.timestamp_ssl_setup - - flow.server_conn.timestamp_tcp_setup) - - SERVERS_SEEN.add(flow.server_conn) - - # Calculate raw timings from timestamps. DNS timings can not be calculated - # for lack of a way to measure it. The same goes for HAR blocked. - # mitmproxy will open a server connection as soon as it receives the host - # and port from the client connection. So, the time spent waiting is actually - # spent waiting between request.timestamp_end and response.timestamp_start - # thus it correlates to HAR wait instead. - timings_raw = { - 'send': flow.request.timestamp_end - flow.request.timestamp_start, - 'receive': flow.response.timestamp_end - flow.response.timestamp_start, - 'wait': flow.response.timestamp_start - flow.request.timestamp_end, - 'connect': connect_time, - 'ssl': ssl_time, - } - - # HAR timings are integers in ms, so we re-encode the raw timings to that format. - timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()]) - - # full_time is the sum of all timings. - # Timings set to -1 will be ignored as per spec. - full_time = sum(v for v in timings.values() if v > -1) - - started_date_time = format_datetime(datetime.utcfromtimestamp(flow.request.timestamp_start)) - - # Response body size and encoding - response_body_size = len(flow.response.raw_content) - response_body_decoded_size = len(flow.response.content) - response_body_compression = response_body_decoded_size - response_body_size - - entry = { - "startedDateTime": started_date_time, - "time": full_time, - "request": { - "method": flow.request.method, - "url": flow.request.url, - "httpVersion": flow.request.http_version, - "cookies": format_request_cookies(flow.request.cookies.fields), - "headers": name_value(flow.request.headers), - "queryString": name_value(flow.request.query or {}), - "headersSize": len(str(flow.request.headers)), - "bodySize": len(flow.request.content), - }, - "response": { - "status": flow.response.status_code, - "statusText": flow.response.reason, - "httpVersion": flow.response.http_version, - "cookies": format_response_cookies(flow.response.cookies.fields), - "headers": name_value(flow.response.headers), - "content": { - "size": response_body_size, - "compression": response_body_compression, - "mimeType": flow.response.headers.get('Content-Type', '') - }, - "redirectURL": flow.response.headers.get('Location', ''), - "headersSize": len(str(flow.response.headers)), - "bodySize": response_body_size, - }, - "cache": {}, - "timings": timings, - } - - # Store binary data as base64 - if strutils.is_mostly_bin(flow.response.content): - entry["response"]["content"]["text"] = base64.b64encode(flow.response.content).decode() - entry["response"]["content"]["encoding"] = "base64" - else: - entry["response"]["content"]["text"] = flow.response.get_text(strict=False) - - if flow.request.method in ["POST", "PUT", "PATCH"]: - params = [ - {"name": a.decode("utf8", "surrogateescape"), "value": b.decode("utf8", "surrogateescape")} - for a, b in flow.request.urlencoded_form.items(multi=True) - ] - entry["request"]["postData"] = { - "mimeType": flow.request.headers.get("Content-Type", ""), - "text": flow.request.get_text(strict=False), - "params": params - } - - if flow.server_conn.connected(): - entry["serverIPAddress"] = str(flow.server_conn.ip_address.address[0]) - - HAR["log"]["entries"].append(entry) - - -def done(): - """ - Called once on script shutdown, after any other events. - """ - dump_file = sys.argv[1] - - json_dump = json.dumps(HAR, indent=2) # type: str - - if dump_file == '-': - mitmproxy.ctx.log(json_dump) - else: - raw = json_dump.encode() # type: bytes - if dump_file.endswith('.zhar'): - raw = zlib.compress(raw, 9) - - with open(dump_file, "wb") as f: - f.write(raw) - - mitmproxy.ctx.log("HAR dump finished (wrote %s bytes to file)" % len(json_dump)) - - -def format_datetime(dt): - return dt.replace(tzinfo=pytz.timezone("UTC")).isoformat() - - -def format_cookies(cookie_list): - rv = [] - - for name, value, attrs in cookie_list: - cookie_har = { - "name": name, - "value": value, - } - - # HAR only needs some attributes - for key in ["path", "domain", "comment"]: - if key in attrs: - cookie_har[key] = attrs[key] - - # These keys need to be boolean! - for key in ["httpOnly", "secure"]: - cookie_har[key] = bool(key in attrs) - - # Expiration time needs to be formatted - expire_ts = cookies.get_expiration_ts(attrs) - if expire_ts is not None: - cookie_har["expires"] = format_datetime(datetime.fromtimestamp(expire_ts)) - - rv.append(cookie_har) - - return rv - - -def format_request_cookies(fields): - return format_cookies(cookies.group_cookies(fields)) - - -def format_response_cookies(fields): - return format_cookies((c[0], c[1].value, c[1].attrs) for c in fields) - - -def name_value(obj): - """ - Convert (key, value) pairs to HAR format. - """ - return [{"name": k, "value": v} for k, v in obj.items()] diff --git a/examples/iframe_injector.py b/examples/iframe_injector.py deleted file mode 100644 index 33d18bbd..00000000 --- a/examples/iframe_injector.py +++ /dev/null @@ -1,29 +0,0 @@ -# Usage: mitmdump -s "iframe_injector.py url" -# (this script works best with --anticache) -import sys -from bs4 import BeautifulSoup - - -class Injector: - def __init__(self, iframe_url): - self.iframe_url = iframe_url - - def response(self, flow): - if flow.request.host in self.iframe_url: - return - html = BeautifulSoup(flow.response.content, "lxml") - if html.body: - iframe = html.new_tag( - "iframe", - src=self.iframe_url, - frameborder=0, - height=0, - width=0) - html.body.insert(0, iframe) - flow.response.content = str(html).encode("utf8") - - -def start(): - if len(sys.argv) != 2: - raise ValueError('Usage: -s "iframe_injector.py url"') - return Injector(sys.argv[1]) diff --git a/examples/mitmproxywrapper.py b/examples/mitmproxywrapper.py deleted file mode 100644 index eade0fe2..00000000 --- a/examples/mitmproxywrapper.py +++ /dev/null @@ -1,165 +0,0 @@ -#!/usr/bin/env python -# -# Helper tool to enable/disable OS X proxy and wrap mitmproxy -# -# Get usage information with: -# -# mitmproxywrapper.py -h -# - -import subprocess -import re -import argparse -import contextlib -import os -import sys - - -class Wrapper: - def __init__(self, port, extra_arguments=None): - self.port = port - self.extra_arguments = extra_arguments - - def run_networksetup_command(self, *arguments): - return subprocess.check_output( - ['sudo', 'networksetup'] + list(arguments)) - - def proxy_state_for_service(self, service): - state = self.run_networksetup_command( - '-getwebproxy', - service).splitlines() - return dict([re.findall(r'([^:]+): (.*)', line)[0] for line in state]) - - def enable_proxy_for_service(self, service): - print('Enabling proxy on {}...'.format(service)) - for subcommand in ['-setwebproxy', '-setsecurewebproxy']: - self.run_networksetup_command( - subcommand, service, '127.0.0.1', str( - self.port)) - - def disable_proxy_for_service(self, service): - print('Disabling proxy on {}...'.format(service)) - for subcommand in ['-setwebproxystate', '-setsecurewebproxystate']: - self.run_networksetup_command(subcommand, service, 'Off') - - def interface_name_to_service_name_map(self): - order = self.run_networksetup_command('-listnetworkserviceorder') - mapping = re.findall( - r'\(\d+\)\s(.*)$\n\(.*Device: (.+)\)$', - order, - re.MULTILINE) - return dict([(b, a) for (a, b) in mapping]) - - def run_command_with_input(self, command, input): - popen = subprocess.Popen( - command, - stdin=subprocess.PIPE, - stdout=subprocess.PIPE) - (stdout, stderr) = popen.communicate(input) - return stdout - - def primary_interace_name(self): - scutil_script = 'get State:/Network/Global/IPv4\nd.show\n' - stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script) - interface, = re.findall(r'PrimaryInterface\s*:\s*(.+)', stdout) - return interface - - def primary_service_name(self): - return self.interface_name_to_service_name_map()[ - self.primary_interace_name()] - - def proxy_enabled_for_service(self, service): - return self.proxy_state_for_service(service)['Enabled'] == 'Yes' - - def toggle_proxy(self): - new_state = not self.proxy_enabled_for_service( - self.primary_service_name()) - for service_name in self.connected_service_names(): - if self.proxy_enabled_for_service(service_name) and not new_state: - self.disable_proxy_for_service(service_name) - elif not self.proxy_enabled_for_service(service_name) and new_state: - self.enable_proxy_for_service(service_name) - - def connected_service_names(self): - scutil_script = 'list\n' - stdout = self.run_command_with_input('/usr/sbin/scutil', scutil_script) - service_ids = re.findall(r'State:/Network/Service/(.+)/IPv4', stdout) - - service_names = [] - for service_id in service_ids: - scutil_script = 'show Setup:/Network/Service/{}\n'.format( - service_id) - stdout = self.run_command_with_input( - '/usr/sbin/scutil', - scutil_script) - service_name, = re.findall(r'UserDefinedName\s*:\s*(.+)', stdout) - service_names.append(service_name) - - return service_names - - def wrap_mitmproxy(self): - with self.wrap_proxy(): - cmd = ['mitmproxy', '-p', str(self.port)] - if self.extra_arguments: - cmd.extend(self.extra_arguments) - subprocess.check_call(cmd) - - def wrap_honeyproxy(self): - with self.wrap_proxy(): - popen = subprocess.Popen('honeyproxy.sh') - try: - popen.wait() - except KeyboardInterrupt: - popen.terminate() - - @contextlib.contextmanager - def wrap_proxy(self): - connected_service_names = self.connected_service_names() - for service_name in connected_service_names: - if not self.proxy_enabled_for_service(service_name): - self.enable_proxy_for_service(service_name) - - yield - - for service_name in connected_service_names: - if self.proxy_enabled_for_service(service_name): - self.disable_proxy_for_service(service_name) - - @classmethod - def ensure_superuser(cls): - if os.getuid() != 0: - print('Relaunching with sudo...') - os.execv('/usr/bin/sudo', ['/usr/bin/sudo'] + sys.argv) - - @classmethod - def main(cls): - parser = argparse.ArgumentParser( - description='Helper tool for OS X proxy configuration and mitmproxy.', - epilog='Any additional arguments will be passed on unchanged to mitmproxy.') - parser.add_argument( - '-t', - '--toggle', - action='store_true', - help='just toggle the proxy configuration') - # parser.add_argument('--honeyproxy', action='store_true', help='run honeyproxy instead of mitmproxy') - parser.add_argument( - '-p', - '--port', - type=int, - help='override the default port of 8080', - default=8080) - args, extra_arguments = parser.parse_known_args() - - wrapper = cls(port=args.port, extra_arguments=extra_arguments) - - if args.toggle: - wrapper.toggle_proxy() - # elif args.honeyproxy: - # wrapper.wrap_honeyproxy() - else: - wrapper.wrap_mitmproxy() - - -if __name__ == '__main__': - Wrapper.ensure_superuser() - Wrapper.main() diff --git a/examples/modify_form.py b/examples/modify_form.py deleted file mode 100644 index b63a1586..00000000 --- a/examples/modify_form.py +++ /dev/null @@ -1,8 +0,0 @@ -def request(flow): - if flow.request.urlencoded_form: - flow.request.urlencoded_form["mitmproxy"] = "rocks" - else: - # This sets the proper content type and overrides the body. - flow.request.urlencoded_form = [ - ("foo", "bar") - ] diff --git a/examples/modify_querystring.py b/examples/modify_querystring.py deleted file mode 100644 index ee8a89ad..00000000 --- a/examples/modify_querystring.py +++ /dev/null @@ -1,2 +0,0 @@ -def request(flow): - flow.request.query["mitmproxy"] = "rocks" diff --git a/examples/nonblocking.py b/examples/nonblocking.py deleted file mode 100644 index 264a1fdb..00000000 --- a/examples/nonblocking.py +++ /dev/null @@ -1,11 +0,0 @@ -import time - -from mitmproxy.script import concurrent - - -@concurrent # Remove this and see what happens -def request(flow): - # You don't want to use mitmproxy.ctx from a different thread - print("handle request: %s%s" % (flow.request.host, flow.request.path)) - time.sleep(5) - print("start request: %s%s" % (flow.request.host, flow.request.path)) diff --git a/examples/proxapp.py b/examples/proxapp.py deleted file mode 100644 index f95c41e5..00000000 --- a/examples/proxapp.py +++ /dev/null @@ -1,25 +0,0 @@ -""" -This example shows how to graft a WSGI app onto mitmproxy. In this -instance, we're using the Flask framework (http://flask.pocoo.org/) to expose -a single simplest-possible page. -""" -from flask import Flask -from mitmproxy.addons import wsgiapp - -app = Flask("proxapp") - - -@app.route('/') -def hello_world(): - return 'Hello World!' - - -def start(): - # Host app at the magic domain "proxapp" on port 80. Requests to this - # domain and port combination will now be routed to the WSGI app instance. - return wsgiapp.WSGIApp(app, "proxapp", 80) - - # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design. - # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set) - # but won't send any data. - # mitmproxy.ctx.master.apps.add(app, "example.com", 443) diff --git a/examples/read_dumpfile b/examples/read_dumpfile deleted file mode 100644 index e0e9064a..00000000 --- a/examples/read_dumpfile +++ /dev/null @@ -1,21 +0,0 @@ -#!/usr/bin/env python -# -# Simple script showing how to read a mitmproxy dump file -# - -from mitmproxy import flow -from mitmproxy.exceptions import FlowReadException -import pprint -import sys - -with open(sys.argv[1], "rb") as logfile: - freader = io.FlowReader(logfile) - pp = pprint.PrettyPrinter(indent=4) - try: - for f in freader.stream(): - print(f) - print(f.request.host) - pp.pprint(f.get_state()) - print("") - except FlowReadException as e: - print("Flow file corrupted: {}".format(e)) diff --git a/examples/redirect_requests.py b/examples/redirect_requests.py deleted file mode 100644 index c28042db..00000000 --- a/examples/redirect_requests.py +++ /dev/null @@ -1,18 +0,0 @@ -""" -This example shows two ways to redirect flows to other destinations. -""" -from mitmproxy import http - - -def request(flow): - # pretty_host takes the "Host" header of the request into account, - # which is useful in transparent mode where we usually only have the IP - # otherwise. - - # Method 1: Answer with a locally generated response - if flow.request.pretty_host.endswith("example.com"): - flow.response = http.HTTPResponse.make(200, b"Hello World", {"Content-Type": "text/html"}) - - # Method 2: Redirect the request to a different server - if flow.request.pretty_host.endswith("example.org"): - flow.request.host = "mitmproxy.org" diff --git a/examples/remote_debug.py b/examples/remote_debug.py deleted file mode 100644 index fb864f78..00000000 --- a/examples/remote_debug.py +++ /dev/null @@ -1,19 +0,0 @@ -""" -This script enables remote debugging of the mitmproxy *UI* with PyCharm. -For general debugging purposes, it is easier to just debug mitmdump within PyCharm. - -Usage: - - pip install pydevd on the mitmproxy machine - - Open the Run/Debug Configuration dialog box in PyCharm, and select the Python Remote Debug configuration type. - - Debugging works in the way that mitmproxy connects to the debug server on startup. - Specify host and port that mitmproxy can use to reach your PyCharm instance on startup. - - Adjust this inline script accordingly. - - Start debug server in PyCharm - - Set breakpoints - - Start mitmproxy -s remote_debug.py -""" - - -def start(): - import pydevd - pydevd.settrace("localhost", port=5678, stdoutToServer=True, stderrToServer=True) diff --git a/examples/simple/README.md b/examples/simple/README.md new file mode 100644 index 00000000..52f15627 --- /dev/null +++ b/examples/simple/README.md @@ -0,0 +1,18 @@ +## Simple Examples + +| Filename | Description | +|:-----------------------------|:---------------------------------------------------------------------------| +| add_header.py | Simple script that just adds a header to every request. | +| custom_contentview.py | Add a custom content view to the mitmproxy UI. | +| filter_flows.py | This script demonstrates how to use mitmproxy's filter pattern in scripts. | +| io_read_dumpfile.py | Read a dumpfile generated by mitmproxy. | +| io_write_dumpfile.py | Only write selected flows into a mitmproxy dumpfile. | +| logging.py | Use mitmproxy's logging API. | +| modify_body_inject_iframe.py | Inject configurable iframe into pages. | +| modify_form.py | Modify HTTP form submissions. | +| modify_querystring.py | Modify HTTP query strings. | +| redirect_requests.py | Redirect a request to a different server. | +| script_arguments.py | Add arguments to a script. | +| send_reply_from_proxy.py | Send a HTTP response directly from the proxy. | +| upsidedownternet.py | Turn all images upside down. | +| wsgi_flask_app.py | Embed a WSGI app into mitmproxy. | \ No newline at end of file diff --git a/examples/simple/add_header.py b/examples/simple/add_header.py new file mode 100644 index 00000000..3e0b5f1e --- /dev/null +++ b/examples/simple/add_header.py @@ -0,0 +1,2 @@ +def response(flow): + flow.response.headers["newheader"] = "foo" diff --git a/examples/simple/add_header_class.py b/examples/simple/add_header_class.py new file mode 100644 index 00000000..6443798a --- /dev/null +++ b/examples/simple/add_header_class.py @@ -0,0 +1,7 @@ +class AddHeader: + def response(self, flow): + flow.response.headers["newheader"] = "foo" + + +def start(): + return AddHeader() diff --git a/examples/simple/custom_contentview.py b/examples/simple/custom_contentview.py new file mode 100644 index 00000000..35216397 --- /dev/null +++ b/examples/simple/custom_contentview.py @@ -0,0 +1,28 @@ +""" +This example shows how one can add a custom contentview to mitmproxy. +The content view API is explained in the mitmproxy.contentviews module. +""" +from mitmproxy import contentviews + + +class ViewSwapCase(contentviews.View): + name = "swapcase" + + # We don't have a good solution for the keyboard shortcut yet - + # you manually need to find a free letter. Contributions welcome :) + prompt = ("swap case text", "p") + content_types = ["text/plain"] + + def __call__(self, data: bytes, **metadata): + return "case-swapped text", contentviews.format_text(data.swapcase()) + + +view = ViewSwapCase() + + +def start(): + contentviews.add(view) + + +def done(): + contentviews.remove(view) diff --git a/examples/simple/filter_flows.py b/examples/simple/filter_flows.py new file mode 100644 index 00000000..29d0a9b8 --- /dev/null +++ b/examples/simple/filter_flows.py @@ -0,0 +1,23 @@ +""" +This scripts demonstrates how to use mitmproxy's filter pattern in scripts. +Usage: + mitmdump -s "flowfilter.py FILTER" +""" +import sys +from mitmproxy import flowfilter + + +class Filter: + def __init__(self, spec): + self.filter = flowfilter.parse(spec) + + def response(self, flow): + if flowfilter.match(self.filter, flow): + print("Flow matches filter:") + print(flow) + + +def start(): + if len(sys.argv) != 2: + raise ValueError("Usage: -s 'filt.py FILTER'") + return Filter(sys.argv[1]) diff --git a/examples/simple/io_read_dumpfile.py b/examples/simple/io_read_dumpfile.py new file mode 100644 index 00000000..edbbe2dd --- /dev/null +++ b/examples/simple/io_read_dumpfile.py @@ -0,0 +1,21 @@ +#!/usr/bin/env python +# +# Simple script showing how to read a mitmproxy dump file +# + +from mitmproxy import io +from mitmproxy.exceptions import FlowReadException +import pprint +import sys + +with open(sys.argv[1], "rb") as logfile: + freader = io.FlowReader(logfile) + pp = pprint.PrettyPrinter(indent=4) + try: + for f in freader.stream(): + print(f) + print(f.request.host) + pp.pprint(f.get_state()) + print("") + except FlowReadException as e: + print("Flow file corrupted: {}".format(e)) diff --git a/examples/simple/io_write_dumpfile.py b/examples/simple/io_write_dumpfile.py new file mode 100644 index 00000000..ff1fd0f4 --- /dev/null +++ b/examples/simple/io_write_dumpfile.py @@ -0,0 +1,29 @@ +""" +This script how to generate a mitmproxy dump file, +as it would also be generated by passing `-w` to mitmproxy. +In contrast to `-w`, this gives you full control over which +flows should be saved and also allows you to rotate files or log +to multiple files in parallel. +""" +import random +import sys +from mitmproxy import io + + +class Writer: + def __init__(self, path): + if path == "-": + f = sys.stdout + else: + f = open(path, "wb") + self.w = io.FlowWriter(f) + + def response(self, flow): + if random.choice([True, False]): + self.w.add(flow) + + +def start(): + if len(sys.argv) != 2: + raise ValueError('Usage: -s "flowriter.py filename"') + return Writer(sys.argv[1]) diff --git a/examples/simple/logging.py b/examples/simple/logging.py new file mode 100644 index 00000000..ab1baf75 --- /dev/null +++ b/examples/simple/logging.py @@ -0,0 +1,12 @@ +""" +It is recommended to use `ctx.log` for logging within a script. +This goes to the event log in mitmproxy and to stdout in mitmdump. + +If you want to help us out: https://github.com/mitmproxy/mitmproxy/issues/1530 :-) +""" +from mitmproxy import ctx + + +def start(): + ctx.log.info("This is some informative text.") + ctx.log.error("This is an error.") diff --git a/examples/simple/modify_body_inject_iframe.py b/examples/simple/modify_body_inject_iframe.py new file mode 100644 index 00000000..33d18bbd --- /dev/null +++ b/examples/simple/modify_body_inject_iframe.py @@ -0,0 +1,29 @@ +# Usage: mitmdump -s "iframe_injector.py url" +# (this script works best with --anticache) +import sys +from bs4 import BeautifulSoup + + +class Injector: + def __init__(self, iframe_url): + self.iframe_url = iframe_url + + def response(self, flow): + if flow.request.host in self.iframe_url: + return + html = BeautifulSoup(flow.response.content, "lxml") + if html.body: + iframe = html.new_tag( + "iframe", + src=self.iframe_url, + frameborder=0, + height=0, + width=0) + html.body.insert(0, iframe) + flow.response.content = str(html).encode("utf8") + + +def start(): + if len(sys.argv) != 2: + raise ValueError('Usage: -s "iframe_injector.py url"') + return Injector(sys.argv[1]) diff --git a/examples/simple/modify_form.py b/examples/simple/modify_form.py new file mode 100644 index 00000000..b425efb0 --- /dev/null +++ b/examples/simple/modify_form.py @@ -0,0 +1,10 @@ +def request(flow): + if flow.request.urlencoded_form: + # If there's already a form, one can just add items to the dict: + flow.request.urlencoded_form["mitmproxy"] = "rocks" + else: + # One can also just pass new form data. + # This sets the proper content type and overrides the body. + flow.request.urlencoded_form = [ + ("foo", "bar") + ] diff --git a/examples/simple/modify_querystring.py b/examples/simple/modify_querystring.py new file mode 100644 index 00000000..ee8a89ad --- /dev/null +++ b/examples/simple/modify_querystring.py @@ -0,0 +1,2 @@ +def request(flow): + flow.request.query["mitmproxy"] = "rocks" diff --git a/examples/simple/redirect_requests.py b/examples/simple/redirect_requests.py new file mode 100644 index 00000000..51876df7 --- /dev/null +++ b/examples/simple/redirect_requests.py @@ -0,0 +1,11 @@ +""" +This example shows two ways to redirect flows to another server. +""" + + +def request(flow): + # pretty_host takes the "Host" header of the request into account, + # which is useful in transparent mode where we usually only have the IP + # otherwise. + if flow.request.pretty_host == "example.org": + flow.request.host = "mitmproxy.org" diff --git a/examples/simple/script_arguments.py b/examples/simple/script_arguments.py new file mode 100644 index 00000000..70851192 --- /dev/null +++ b/examples/simple/script_arguments.py @@ -0,0 +1,17 @@ +import argparse + + +class Replacer: + def __init__(self, src, dst): + self.src, self.dst = src, dst + + def response(self, flow): + flow.response.replace(self.src, self.dst) + + +def start(): + parser = argparse.ArgumentParser() + parser.add_argument("src", type=str) + parser.add_argument("dst", type=str) + args = parser.parse_args() + return Replacer(args.src, args.dst) diff --git a/examples/simple/send_reply_from_proxy.py b/examples/simple/send_reply_from_proxy.py new file mode 100644 index 00000000..bef2e7e7 --- /dev/null +++ b/examples/simple/send_reply_from_proxy.py @@ -0,0 +1,17 @@ +""" +This example shows how to send a reply from the proxy immediately +without sending any data to the remote server. +""" +from mitmproxy import http + + +def request(flow): + # pretty_url takes the "Host" header of the request into account, which + # is useful in transparent mode where we usually only have the IP otherwise. + + if flow.request.pretty_url == "http://example.com/path": + flow.response = http.HTTPResponse.make( + 200, # (optional) status code + b"Hello World", # (optional) content + {"Content-Type": "text/html"} # (optional) headers + ) diff --git a/examples/simple/upsidedownternet.py b/examples/simple/upsidedownternet.py new file mode 100644 index 00000000..8ba450ab --- /dev/null +++ b/examples/simple/upsidedownternet.py @@ -0,0 +1,16 @@ +""" +This script rotates all images passing through the proxy by 180 degrees. +""" +import io + +from PIL import Image + + +def response(flow): + if flow.response.headers.get("content-type", "").startswith("image"): + s = io.BytesIO(flow.response.content) + img = Image.open(s).rotate(180) + s2 = io.BytesIO() + img.save(s2, "png") + flow.response.content = s2.getvalue() + flow.response.headers["content-type"] = "image/png" diff --git a/examples/simple/wsgi_flask_app.py b/examples/simple/wsgi_flask_app.py new file mode 100644 index 00000000..f95c41e5 --- /dev/null +++ b/examples/simple/wsgi_flask_app.py @@ -0,0 +1,25 @@ +""" +This example shows how to graft a WSGI app onto mitmproxy. In this +instance, we're using the Flask framework (http://flask.pocoo.org/) to expose +a single simplest-possible page. +""" +from flask import Flask +from mitmproxy.addons import wsgiapp + +app = Flask("proxapp") + + +@app.route('/') +def hello_world(): + return 'Hello World!' + + +def start(): + # Host app at the magic domain "proxapp" on port 80. Requests to this + # domain and port combination will now be routed to the WSGI app instance. + return wsgiapp.WSGIApp(app, "proxapp", 80) + + # SSL works too, but the magic domain needs to be resolvable from the mitmproxy machine due to mitmproxy's design. + # mitmproxy will connect to said domain and use serve its certificate (unless --no-upstream-cert is set) + # but won't send any data. + # mitmproxy.ctx.master.apps.add(app, "example.com", 443) diff --git a/examples/sslstrip.py b/examples/sslstrip.py deleted file mode 100644 index 9a090c0c..00000000 --- a/examples/sslstrip.py +++ /dev/null @@ -1,53 +0,0 @@ -import re -import urllib - -# set of SSL/TLS capable hosts -secure_hosts = set() - - -def request(flow): - flow.request.headers.pop('If-Modified-Since', None) - flow.request.headers.pop('Cache-Control', None) - - # do not force https redirection - flow.request.headers.pop('Upgrade-Insecure-Requests', None) - - # proxy connections to SSL-enabled hosts - if flow.request.pretty_host in secure_hosts: - flow.request.scheme = 'https' - flow.request.port = 443 - - # We need to update the request destination to whatever is specified in the host header: - # Having no TLS Server Name Indication from the client and just an IP address as request.host - # in transparent mode, TLS server name certificate validation would fail. - flow.request.host = flow.request.pretty_host - - -def response(flow): - flow.response.headers.pop('Strict-Transport-Security', None) - flow.response.headers.pop('Public-Key-Pins', None) - - # strip links in response body - flow.response.content = flow.response.content.replace('https://', 'http://') - - # strip meta tag upgrade-insecure-requests in response body - csp_meta_tag_pattern = b'' - flow.response.content = re.sub(csp_meta_tag_pattern, b'', flow.response.content, flags=re.IGNORECASE) - - # strip links in 'Location' header - if flow.response.headers.get('Location', '').startswith('https://'): - location = flow.response.headers['Location'] - hostname = urllib.parse.urlparse(location).hostname - if hostname: - secure_hosts.add(hostname) - flow.response.headers['Location'] = location.replace('https://', 'http://', 1) - - # strip upgrade-insecure-requests in Content-Security-Policy header - if re.search('upgrade-insecure-requests', flow.response.headers.get('Content-Security-Policy', ''), flags=re.IGNORECASE): - csp = flow.response.headers['Content-Security-Policy'] - flow.response.headers['Content-Security-Policy'] = re.sub('upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) - - # strip secure flag from 'Set-Cookie' headers - cookies = flow.response.headers.get_all('Set-Cookie') - cookies = [re.sub(r';\s*secure\s*', '', s) for s in cookies] - flow.response.headers.set_all('Set-Cookie', cookies) diff --git a/examples/stickycookies b/examples/stickycookies deleted file mode 100755 index a0ee90ff..00000000 --- a/examples/stickycookies +++ /dev/null @@ -1,42 +0,0 @@ -#!/usr/bin/env python -""" -This example builds on mitmproxy's base proxying infrastructure to -implement functionality similar to the "sticky cookies" option. - -Heads Up: In the majority of cases, you want to use inline scripts. -""" -import os -from mitmproxy import controller, proxy -from mitmproxy.proxy.server import ProxyServer - - -class StickyMaster(master.Master): - def __init__(self, server): - master.Master.__init__(self, server) - self.stickyhosts = {} - - def run(self): - try: - return master.Master.run(self) - except KeyboardInterrupt: - self.shutdown() - - @controller.handler - def request(self, flow): - hid = (flow.request.host, flow.request.port) - if "cookie" in flow.request.headers: - self.stickyhosts[hid] = flow.request.headers.get_all("cookie") - elif hid in self.stickyhosts: - flow.request.headers.set_all("cookie", self.stickyhosts[hid]) - - @controller.handler - def response(self, flow): - hid = (flow.request.host, flow.request.port) - if "set-cookie" in flow.response.headers: - self.stickyhosts[hid] = flow.response.headers.get_all("set-cookie") - - -config = proxy.ProxyConfig(port=8080) -server = ProxyServer(config) -m = StickyMaster(server) -m.run() diff --git a/examples/stream.py b/examples/stream.py deleted file mode 100644 index 8598f329..00000000 --- a/examples/stream.py +++ /dev/null @@ -1,5 +0,0 @@ -def responseheaders(flow): - """ - Enables streaming for all responses. - """ - flow.response.stream = True diff --git a/examples/stream_modify.py b/examples/stream_modify.py deleted file mode 100644 index 5e5da95b..00000000 --- a/examples/stream_modify.py +++ /dev/null @@ -1,20 +0,0 @@ -""" -This inline script modifies a streamed response. -If you do not need streaming, see the modify_response_body example. -Be aware that content replacement isn't trivial: - - If the transfer encoding isn't chunked, you cannot simply change the content length. - - If you want to replace all occurences of "foobar", make sure to catch the cases - where one chunk ends with [...]foo" and the next starts with "bar[...]. -""" - - -def modify(chunks): - """ - chunks is a generator that can be used to iterate over all chunks. - """ - for chunk in chunks: - yield chunk.replace("foo", "bar") - - -def responseheaders(flow): - flow.response.stream = modify diff --git a/examples/stub.py b/examples/stub.py deleted file mode 100644 index 4f5061e2..00000000 --- a/examples/stub.py +++ /dev/null @@ -1,87 +0,0 @@ -import mitmproxy -""" - This is a script stub, with definitions for all events. -""" - - -def start(): - """ - Called once on script startup before any other events - """ - mitmproxy.ctx.log("start") - - -def configure(options, updated): - """ - Called once on script startup before any other events, and whenever options changes. - """ - mitmproxy.ctx.log("configure") - - -def clientconnect(root_layer): - """ - Called when a client initiates a connection to the proxy. Note that a - connection can correspond to multiple HTTP requests - """ - mitmproxy.ctx.log("clientconnect") - - -def request(flow): - """ - Called when a client request has been received. - """ - mitmproxy.ctx.log("request") - - -def serverconnect(server_conn): - """ - Called when the proxy initiates a connection to the target server. Note that a - connection can correspond to multiple HTTP requests - """ - mitmproxy.ctx.log("serverconnect") - - -def responseheaders(flow): - """ - Called when the response headers for a server response have been received, - but the response body has not been processed yet. Can be used to tell mitmproxy - to stream the response. - """ - mitmproxy.ctx.log("responseheaders") - - -def response(flow): - """ - Called when a server response has been received. - """ - mitmproxy.ctx.log("response") - - -def error(flow): - """ - Called when a flow error has occured, e.g. invalid server responses, or - interrupted connections. This is distinct from a valid server HTTP error - response, which is simply a response with an HTTP error code. - """ - mitmproxy.ctx.log("error") - - -def serverdisconnect(server_conn): - """ - Called when the proxy closes the connection to the target server. - """ - mitmproxy.ctx.log("serverdisconnect") - - -def clientdisconnect(root_layer): - """ - Called when a client disconnects from the proxy. - """ - mitmproxy.ctx.log("clientdisconnect") - - -def done(): - """ - Called once on script shutdown, after any other events. - """ - mitmproxy.ctx.log("done") diff --git a/examples/tcp_message.py b/examples/tcp_message.py deleted file mode 100644 index d7c9c42e..00000000 --- a/examples/tcp_message.py +++ /dev/null @@ -1,27 +0,0 @@ -""" -tcp_message Inline Script Hook API Demonstration ------------------------------------------------- - -* modifies packets containing "foo" to "bar" -* prints various details for each packet. - -example cmdline invocation: -mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py -""" -from mitmproxy.utils import strutils - - -def tcp_message(tcp_msg): - modified_msg = tcp_msg.message.replace("foo", "bar") - - is_modified = False if modified_msg == tcp_msg.message else True - tcp_msg.message = modified_msg - - print( - "[tcp_message{}] from {} {} to {} {}:\r\n{}".format( - " (modified)" if is_modified else "", - "client" if tcp_msg.sender == tcp_msg.client_conn else "server", - tcp_msg.sender.address, - "server" if tcp_msg.receiver == tcp_msg.server_conn else "client", - tcp_msg.receiver.address, strutils.bytes_to_escaped_str(tcp_msg.message)) - ) diff --git a/examples/tls_passthrough.py b/examples/tls_passthrough.py deleted file mode 100644 index 40c1051d..00000000 --- a/examples/tls_passthrough.py +++ /dev/null @@ -1,140 +0,0 @@ -""" -This inline script allows conditional TLS Interception based -on a user-defined strategy. - -Example: - - > mitmdump -s tls_passthrough.py - - 1. curl --proxy http://localhost:8080 https://example.com --insecure - // works - we'll also see the contents in mitmproxy - - 2. curl --proxy http://localhost:8080 https://example.com --insecure - // still works - we'll also see the contents in mitmproxy - - 3. curl --proxy http://localhost:8080 https://example.com - // fails with a certificate error, which we will also see in mitmproxy - - 4. curl --proxy http://localhost:8080 https://example.com - // works again, but mitmproxy does not intercept and we do *not* see the contents - -Authors: Maximilian Hils, Matthew Tuusberg -""" -import collections -import random - -import sys -from enum import Enum - -import mitmproxy -from mitmproxy.exceptions import TlsProtocolException -from mitmproxy.proxy.protocol import TlsLayer, RawTCPLayer - - -class InterceptionResult(Enum): - success = True - failure = False - skipped = None - - -class _TlsStrategy: - """ - Abstract base class for interception strategies. - """ - - def __init__(self): - # A server_address -> interception results mapping - self.history = collections.defaultdict(lambda: collections.deque(maxlen=200)) - - def should_intercept(self, server_address): - """ - Returns: - True, if we should attempt to intercept the connection. - False, if we want to employ pass-through instead. - """ - raise NotImplementedError() - - def record_success(self, server_address): - self.history[server_address].append(InterceptionResult.success) - - def record_failure(self, server_address): - self.history[server_address].append(InterceptionResult.failure) - - def record_skipped(self, server_address): - self.history[server_address].append(InterceptionResult.skipped) - - -class ConservativeStrategy(_TlsStrategy): - """ - Conservative Interception Strategy - only intercept if there haven't been any failed attempts - in the history. - """ - - def should_intercept(self, server_address): - if InterceptionResult.failure in self.history[server_address]: - return False - return True - - -class ProbabilisticStrategy(_TlsStrategy): - """ - Fixed probability that we intercept a given connection. - """ - - def __init__(self, p): - self.p = p - super(ProbabilisticStrategy, self).__init__() - - def should_intercept(self, server_address): - return random.uniform(0, 1) < self.p - - -class TlsFeedback(TlsLayer): - """ - Monkey-patch _establish_tls_with_client to get feedback if TLS could be established - successfully on the client connection (which may fail due to cert pinning). - """ - - def _establish_tls_with_client(self): - server_address = self.server_conn.address - - try: - super(TlsFeedback, self)._establish_tls_with_client() - except TlsProtocolException as e: - tls_strategy.record_failure(server_address) - raise e - else: - tls_strategy.record_success(server_address) - - -# inline script hooks below. - -tls_strategy = None - - -def start(): - global tls_strategy - if len(sys.argv) == 2: - tls_strategy = ProbabilisticStrategy(float(sys.argv[1])) - else: - tls_strategy = ConservativeStrategy() - - -def next_layer(next_layer): - """ - This hook does the actual magic - if the next layer is planned to be a TLS layer, - we check if we want to enter pass-through mode instead. - """ - if isinstance(next_layer, TlsLayer) and next_layer._client_tls: - server_address = next_layer.server_conn.address - - if tls_strategy.should_intercept(server_address): - # We try to intercept. - # Monkey-Patch the layer to get feedback from the TLSLayer if interception worked. - next_layer.__class__ = TlsFeedback - else: - # We don't intercept - reply with a pass-through layer and add a "skipped" entry. - mitmproxy.ctx.log("TLS passthrough for %s" % repr(next_layer.server_conn.address), "info") - next_layer_replacement = RawTCPLayer(next_layer.ctx, ignore=True) - next_layer.reply.send(next_layer_replacement) - tls_strategy.record_skipped(server_address) diff --git a/examples/upsidedownternet.py b/examples/upsidedownternet.py deleted file mode 100644 index d4de7e25..00000000 --- a/examples/upsidedownternet.py +++ /dev/null @@ -1,15 +0,0 @@ -import io -from PIL import Image - - -def response(flow): - if flow.response.headers.get("content-type", "").startswith("image"): - try: - s = io.StringIO(flow.response.content) - img = Image.open(s).rotate(180) - s2 = io.StringIO() - img.save(s2, "png") - flow.response.content = s2.getvalue() - flow.response.headers["content-type"] = "image/png" - except: # Unknown image types etc. - pass diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py index 7f194a18..94637350 100644 --- a/test/mitmproxy/test_examples.py +++ b/test/mitmproxy/test_examples.py @@ -40,30 +40,29 @@ def tscript(cmd, args=""): class TestScripts(mastertest.MasterTest): def test_add_header(self): - m, _ = tscript("add_header.py") + m, _ = tscript("simple/add_header.py") f = tflow.tflow(resp=tutils.tresp()) m.response(f) assert f.response.headers["newheader"] == "foo" def test_custom_contentviews(self): - m, sc = tscript("custom_contentviews.py") - pig = contentviews.get("pig_latin_HTML") - _, fmt = pig(b"test!") - assert any(b'esttay!' in val[0][1] for val in fmt) - assert not pig(b"gobbledygook") + m, sc = tscript("simple/custom_contentview.py") + swapcase = contentviews.get("swapcase") + _, fmt = swapcase(b"Test!") + assert any(b'tEST!' in val[0][1] for val in fmt) def test_iframe_injector(self): with tutils.raises(ScriptError): - tscript("iframe_injector.py") + tscript("simple/modify_body_inject_iframe.py") - m, sc = tscript("iframe_injector.py", "http://example.org/evil_iframe") + m, sc = tscript("simple/modify_body_inject_iframe.py", "http://example.org/evil_iframe") f = tflow.tflow(resp=tutils.tresp(content=b"mitmproxy")) m.response(f) content = f.response.content assert b'iframe' in content and b'evil_iframe' in content def test_modify_form(self): - m, sc = tscript("modify_form.py") + m, sc = tscript("simple/modify_form.py") form_header = Headers(content_type="application/x-www-form-urlencoded") f = tflow.tflow(req=tutils.treq(headers=form_header)) @@ -76,7 +75,7 @@ class TestScripts(mastertest.MasterTest): assert list(f.request.urlencoded_form.items()) == [(b"foo", b"bar")] def test_modify_querystring(self): - m, sc = tscript("modify_querystring.py") + m, sc = tscript("simple/modify_querystring.py") f = tflow.tflow(req=tutils.treq(path="/search?q=term")) m.request(f) @@ -87,17 +86,23 @@ class TestScripts(mastertest.MasterTest): assert f.request.query["mitmproxy"] == "rocks" def test_arguments(self): - m, sc = tscript("arguments.py", "mitmproxy rocks") + m, sc = tscript("simple/script_arguments.py", "mitmproxy rocks") f = tflow.tflow(resp=tutils.tresp(content=b"I <3 mitmproxy")) m.response(f) assert f.response.content == b"I <3 rocks" def test_redirect_requests(self): - m, sc = tscript("redirect_requests.py") + m, sc = tscript("simple/redirect_requests.py") f = tflow.tflow(req=tutils.treq(host="example.org")) m.request(f) assert f.request.host == "mitmproxy.org" + def test_send_reply_from_proxy(self): + m, sc = tscript("simple/send_reply_from_proxy.py") + f = tflow.tflow(req=tutils.treq(host="example.com", port=80)) + m.request(f) + assert f.response.content == b"Hello World" + class TestHARDump: @@ -115,13 +120,13 @@ class TestHARDump: def test_no_file_arg(self): with tutils.raises(ScriptError): - tscript("har_dump.py") + tscript("complex/har_dump.py") def test_simple(self): with tutils.tmpdir() as tdir: path = os.path.join(tdir, "somefile") - m, sc = tscript("har_dump.py", shlex.quote(path)) + m, sc = tscript("complex/har_dump.py", shlex.quote(path)) m.addons.invoke(m, "response", self.flow()) m.addons.remove(sc) @@ -134,7 +139,7 @@ class TestHARDump: with tutils.tmpdir() as tdir: path = os.path.join(tdir, "somefile") - m, sc = tscript("har_dump.py", shlex.quote(path)) + m, sc = tscript("complex/har_dump.py", shlex.quote(path)) m.addons.invoke(m, "response", self.flow(resp_content=b"foo" + b"\xFF" * 10)) m.addons.remove(sc) @@ -144,7 +149,7 @@ class TestHARDump: assert har["log"]["entries"][0]["response"]["content"]["encoding"] == "base64" def test_format_cookies(self): - m, sc = tscript("har_dump.py", "-") + m, sc = tscript("complex/har_dump.py", "-") format_cookies = sc.ns.ns["format_cookies"] CA = cookies.CookieAttrs @@ -174,7 +179,7 @@ class TestHARDump: with tutils.tmpdir() as tdir: path = os.path.join(tdir, "somefile") - m, sc = tscript("har_dump.py", shlex.quote(path)) + m, sc = tscript("complex/har_dump.py", shlex.quote(path)) m.addons.invoke(m, "response", f) m.addons.remove(sc) -- cgit v1.2.3