diff options
29 files changed, 677 insertions, 314 deletions
@@ -1,3 +1,55 @@ +6 November 2015: mitmproxy 0.14 + + * Statistics: 399 commits, 13 contributors, 79 closed issues, 37 closed + PRs, 103 days + + * Docs: Greatly updated docs now hosted on ReadTheDocs! + http://docs.mitmproxy.org + + * Docs: Fixed Typos, updated URLs etc. (Nick Badger, Ben Lerner, Choongwoo + Han, onlywade, Jurriaan Bremer) + + * mitmdump: Colorized TTY output + + * mitmdump: Use mitmproxy's content views for human-readable output (Chris + Czub) + + * mitmproxy and mitmdump: Support for displaying UTF8 contents + + * mitmproxy: add command line switch to disable mouse interaction (Timothy + Elliott) + + * mitmproxy: bug fixes (Choongwoo Han, sethp-jive, FreeArtMan) + + * mitmweb: bug fixes (Colin Bendell) + + * libmproxy: Add ability to fall back to TCP passthrough for non-HTTP + connections. + + * libmproxy: Avoid double-connect in case of TLS Server Name Indication. + This yields a massive speedup for TLS handshakes. + + * libmproxy: Prevent unneccessary upstream connections (macmantrl) + + * Inline Scripts: New API for HTTP Headers: + http://docs.mitmproxy.org/en/latest/dev/models.html#netlib.http.Headers + + * Inline Scripts: Properly handle exceptions in `done` hook + + * Inline Scripts: Allow relative imports, provide `__file__` + + * Examples: Add probabilistic TLS passthrough as an inline script + + * netlib: Refactored HTTP protocol handling code + + * netlib: ALPN support + + * netlib: fixed a bug in the optional certificate verification. + + * netlib: Initial Python 3.5 support (this is the first prerequisite for + 3.x support in mitmproxy) + + 24 July 2015: mitmproxy 0.13 * Upstream certificate validation. See the --verify-upstream-cert, diff --git a/CONTRIBUTORS b/CONTRIBUTORS index 3d056fb8..5389db5f 100644 --- a/CONTRIBUTORS +++ b/CONTRIBUTORS @@ -1,6 +1,7 @@ - 1112 Aldo Cortesi - 569 Maximilian Hils - 79 Marcelo Glezer + 1122 Aldo Cortesi + 765 Maximilian Hils + 80 Marcelo Glezer + 48 Thomas Kriechbaumer 28 Jim Shaver 18 Henrik Nordstrom 13 Thomas Roth @@ -13,41 +14,55 @@ 8 Jason A. Novak 8 Rouli 7 Alexis Hildebrandt - 6 Thomas Kriechbaumer 5 Brad Peabody - 5 Matthias Urlichs 5 Tomaz Muraus + 5 Choongwoo Han + 5 Matthias Urlichs 5 elitest 5 iroiro123 - 4 Bryan Bishop 4 Marc Liyanage 4 Valtteri Virtanen 4 Wade 524 4 Youhei Sakurai 4 root - 3 Chris Neasbitt + 4 Chris Czub + 4 Bryan Bishop + 3 Kyle Manna 3 David Weinstein 3 Eli Shvartsman - 3 Kyle Manna + 3 Chris Neasbitt 3 Zack B - 2 Bennett Blodinger - 2 Choongwoo Han + 2 Rob Wills + 2 isra17 + 2 israel + 2 Michael Frister 2 Heikki Hannikainen - 2 Jaime Soriano Pastor + 2 Nick Badger 2 Jim Lloyd + 2 Terry Long + 2 Wade Catron 2 Krzysztof Bielicki 2 Mark E. Haase - 2 Michael Frister - 2 Rob Wills 2 alts - 2 isra17 - 2 israel + 2 Bennett Blodinger + 2 Jaime Soriano Pastor + 2 Colin Bendell + 1 joebowbeer + 1 meeee + 1 michaeljau + 1 peralta + 1 phil plante + 1 sentient07 + 1 sethp-jive + 1 vzvu3k6k 1 Andy Smith + 1 Ben Lerner 1 Dan Wilbraham 1 David Shaw 1 Doug Lethin 1 Eric Entzel 1 Felix Wolfsteller + 1 FreeArtMan 1 Gabriel Kirkpatrick 1 Henrik Nordström 1 Ivaylo Popov @@ -79,19 +94,12 @@ 1 Suyash 1 Tarashish Mishra 1 TearsDontFalls - 1 Terry Long + 1 Timothy Elliott 1 Ulrich Petri 1 Vyacheslav Bakhmutov - 1 Wade Catron 1 Yuangxuan Wang 1 capt8bit 1 davidpshaw 1 deployable + 1 gecko655 1 jlhonora - 1 joebowbeer - 1 meeee - 1 michaeljau - 1 peralta - 1 phil plante - 1 sentient07 - 1 vzvu3k6k diff --git a/docs/certinstall.rst b/docs/certinstall.rst index 542c6dd2..5ec7b7ce 100644 --- a/docs/certinstall.rst +++ b/docs/certinstall.rst @@ -90,20 +90,28 @@ Chrome on Linux See https://code.google.com/p/chromium/wiki/LinuxCertManagement -More on mitmproxy certificates ------------------------------- +The mitmproxy certificate authority +----------------------------------- The first time **mitmproxy** or **mitmdump** is run, the mitmproxy Certificate Authority (CA) is created in the config directory (``~/.mitmproxy`` by default). This CA is used for on-the-fly generation of dummy certificates for each of the SSL sites that your client visits. Since your browser won't trust the -mitmproxy CA out of the box , you will see an SSL certificate warning every +mitmproxy CA out of the box, you will see an SSL certificate warning every time you visit a new SSL domain through mitmproxy. When you are testing a single site through a browser, just accepting the bogus SSL cert manually is not too much trouble, but there are a many circumstances where you will want to configure your testing system or browser to trust the mitmproxy CA as a -signing root authority. For security reasons, the mitmproxy CA is generated uniquely on the first -start and is not shared between mitmproxy installations on different devices. +signing root authority. For security reasons, the mitmproxy CA is generated uniquely on the first start and is not shared between mitmproxy installations on different devices. + +Certificate Pinning +^^^^^^^^^^^^^^^^^^^ + +Some applications employ `Certificate Pinning`_ to prevent man-in-the-middle attacks. +This means that **mitmproxy** and **mitmdump's** certificates will not be +accepted by these applications without modifying them. It is recommended to use the +:ref:`passthrough` feature in order to prevent **mitmproxy** and **mitmdump** from intercepting +traffic to these specific domains. If you want to intercept the pinned connections, you need to patch the application manually. For Android and (jailbroken) iOS devices, various tools exist to accomplish this. CA and cert files @@ -172,3 +180,5 @@ If you visit example.org, mitmproxy looks for a file named ``example.org.pem`` i directory and uses this as the client cert. The certificate file needs to be in the PEM format and should contain both the unencrypted private key and the certificate. + +.. _Certificate Pinning: http://security.stackexchange.com/questions/29988/what-is-certificate-pinning/
\ No newline at end of file diff --git a/examples/custom_contentviews.py b/examples/custom_contentviews.py new file mode 100644 index 00000000..17920e51 --- /dev/null +++ b/examples/custom_contentviews.py @@ -0,0 +1,68 @@ +import string +import lxml.html +import lxml.etree +from libmproxy import utils, contentviews + + +class ViewPigLatin(contentviews.View): + name = "pig_latin_HTML" + prompt = ("pig latin HTML", "l") + content_types = ["text/html"] + + def __call__(self, data, **metadata): + if utils.isXML(data): + parser = lxml.etree.HTMLParser( + strip_cdata=True, + remove_blank_text=True + ) + d = lxml.html.fromstring(data, parser=parser) + docinfo = d.getroottree().docinfo + + def piglify(src): + words = string.split(src) + ret = '' + for word in words: + idx = -1 + while word[idx] in string.punctuation and (idx * -1) != len(word): idx -= 1 + if word[0].lower() in 'aeiou': + if idx == -1: + ret += word[0:] + "hay" + else: + ret += word[0:len(word) + idx + 1] + "hay" + word[idx + 1:] + else: + if idx == -1: + ret += word[1:] + word[0] + "ay" + else: + ret += word[1:len(word) + idx + 1] + word[0] + "ay" + word[idx + 1:] + ret += ' ' + return ret.strip() + + def recurse(root): + if hasattr(root, 'text') and root.text: + root.text = piglify(root.text) + if hasattr(root, 'tail') and root.tail: + root.tail = piglify(root.tail) + + if len(root): + for child in root: + recurse(child) + + recurse(d) + + s = lxml.etree.tostring( + d, + pretty_print=True, + doctype=docinfo.doctype + ) + return "HTML", contentviews.format_text(s) + + +pig_view = ViewPigLatin() + + +def start(context, argv): + context.add_contentview(pig_view) + + +def stop(context): + context.remove_contentview(pig_view) diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 3bc0c091..cef2013e 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -315,8 +315,8 @@ class ConsoleMaster(flow.FlowMaster): signals.add_event("Running script on flow: %s" % command, "debug") try: - s = script.Script(command, self) - except script.ScriptError as v: + s = script.Script(command, script.ScriptContext(self)) + except script.ScriptException as v: signals.status_message.send( message = "Error loading script." ) @@ -731,3 +731,9 @@ class ConsoleMaster(flow.FlowMaster): if f: self.process_flow(f) return f + + def handle_script_change(self, script): + if super(ConsoleMaster, self).handle_script_change(script): + signals.status_message.send(message='"{}" reloaded.'.format(script.filename)) + else: + signals.status_message.send(message='Error reloading "{}".'.format(script.filename))
\ No newline at end of file diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 1a72fa2a..12fdfe27 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -256,7 +256,7 @@ def copy_flow_format_data(part, scope, flow): return None, "Request content is missing" with decoded(flow.request): if part == "h": - data += flow.client_conn.protocol.assemble(flow.request) + data += netlib.http.http1.assemble_request(flow.request) elif part == "c": data += flow.request.content else: @@ -269,7 +269,7 @@ def copy_flow_format_data(part, scope, flow): return None, "Response content is missing" with decoded(flow.response): if part == "h": - data += flow.client_conn.protocol.assemble(flow.response) + data += netlib.http.http1.assemble_response(flow.response) elif part == "c": data += flow.response.content else: diff --git a/libmproxy/console/grideditor.py b/libmproxy/console/grideditor.py index d32ce5b4..237eea28 100644 --- a/libmproxy/console/grideditor.py +++ b/libmproxy/console/grideditor.py @@ -636,7 +636,7 @@ class ScriptEditor(GridEditor): def is_error(self, col, val): try: script.Script.parse_command(val) - except script.ScriptError as v: + except script.ScriptException as v: return str(v) diff --git a/libmproxy/contentviews.py b/libmproxy/contentviews.py index 9af08033..b2459563 100644 --- a/libmproxy/contentviews.py +++ b/libmproxy/contentviews.py @@ -228,7 +228,8 @@ class ViewHTML(View): s = lxml.etree.tostring( d, pretty_print=True, - doctype=docinfo.doctype + doctype=docinfo.doctype, + encoding='utf8' ) return "HTML", format_text(s) @@ -479,34 +480,9 @@ class ViewWBXML(View): return None -views = [ - ViewAuto(), - ViewRaw(), - ViewHex(), - ViewJSON(), - ViewXML(), - ViewWBXML(), - ViewHTML(), - ViewHTMLOutline(), - ViewJavaScript(), - ViewCSS(), - ViewURLEncoded(), - ViewMultipart(), - ViewImage(), -] -if pyamf: - views.append(ViewAMF()) - -if ViewProtobuf.is_available(): - views.append(ViewProtobuf()) - +views = [] content_types_map = {} -for i in views: - for ct in i.content_types: - l = content_types_map.setdefault(ct, []) - l.append(i) - -view_prompts = [i.prompt for i in views] +view_prompts = [] def get_by_shortcut(c): @@ -515,6 +491,58 @@ def get_by_shortcut(c): return i +def add(view): + # TODO: auto-select a different name (append an integer?) + for i in views: + if i.name == view.name: + raise ContentViewException("Duplicate view: " + view.name) + + # TODO: the UI should auto-prompt for a replacement shortcut + for prompt in view_prompts: + if prompt[1] == view.prompt[1]: + raise ContentViewException("Duplicate view shortcut: " + view.prompt[1]) + + views.append(view) + + for ct in view.content_types: + l = content_types_map.setdefault(ct, []) + l.append(view) + + view_prompts.append(view.prompt) + + +def remove(view): + for ct in view.content_types: + l = content_types_map.setdefault(ct, []) + l.remove(view) + + if not len(l): + del content_types_map[ct] + + view_prompts.remove(view.prompt) + views.remove(view) + + +add(ViewAuto()) +add(ViewRaw()) +add(ViewHex()) +add(ViewJSON()) +add(ViewXML()) +add(ViewWBXML()) +add(ViewHTML()) +add(ViewHTMLOutline()) +add(ViewJavaScript()) +add(ViewCSS()) +add(ViewURLEncoded()) +add(ViewMultipart()) +add(ViewImage()) + +if pyamf: + add(ViewAMF()) + +if ViewProtobuf.is_available(): + add(ViewProtobuf()) + def get(name): for i in views: if i.name == name: diff --git a/libmproxy/exceptions.py b/libmproxy/exceptions.py index 8f23bd92..e2bde980 100644 --- a/libmproxy/exceptions.py +++ b/libmproxy/exceptions.py @@ -48,3 +48,7 @@ class ContentViewException(ProxyException): class ReplayException(ProxyException): pass + + +class ScriptException(ProxyException): + pass
\ No newline at end of file diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 55a4dbcf..97d72992 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -10,11 +10,9 @@ import os import re import urlparse - from netlib import wsgi from netlib.exceptions import HttpException from netlib.http import CONTENT_MISSING, Headers, http1 -import netlib.http from . import controller, tnetstring, filt, script, version from .onboarding import app from .proxy.config import HostMatcher @@ -663,18 +661,21 @@ class FlowMaster(controller.Master): def unload_script(self, script_obj): try: script_obj.unload() - except script.ScriptError as e: + except script.ScriptException as e: self.add_event("Script error:\n" + str(e), "error") + script.reloader.unwatch(script_obj) self.scripts.remove(script_obj) - - def load_script(self, command): + + def load_script(self, command, use_reloader=True): """ Loads a script. Returns an error description if something went wrong. """ try: - s = script.Script(command, self) - except script.ScriptError as v: + s = script.Script(command, script.ScriptContext(self)) + if use_reloader: + script.reloader.watch(s, lambda: self.masterq.put(("script_change", s))) + except script.ScriptException as v: return v.args[0] self.scripts.append(s) @@ -682,7 +683,7 @@ class FlowMaster(controller.Master): if script_obj and not self.pause_scripts: try: script_obj.run(name, *args, **kwargs) - except script.ScriptError as e: + except script.ScriptException as e: self.add_event("Script error:\n" + str(e), "error") def run_script_hook(self, name, *args, **kwargs): @@ -1021,6 +1022,34 @@ class FlowMaster(controller.Master): def handle_accept_intercept(self, f): self.state.update_flow(f) + def handle_script_change(self, s): + """ + Handle a script whose contents have been changed on the file system. + + Args: + s (script.Script): the changed script + + Returns: + True, if reloading was successful. + False, otherwise. + """ + ok = True + # We deliberately do not want to fail here. + # In the worst case, we have an "empty" script object. + try: + s.unload() + except script.ScriptException as e: + ok = False + self.add_event('Error reloading "{}": {}'.format(s.filename, str(e))) + try: + s.load() + except script.ScriptException as e: + ok = False + self.add_event('Error reloading "{}": {}'.format(s.filename, str(e))) + else: + self.add_event('"{}" reloaded.'.format(s.filename)) + return ok + def shutdown(self): self.unload_scripts() controller.Master.shutdown(self) @@ -1037,7 +1066,6 @@ class FlowMaster(controller.Master): self.stream.fo.close() self.stream = None - def read_flows_from_paths(paths): """ Given a list of filepaths, read all flows and return a list of them. diff --git a/libmproxy/main.py b/libmproxy/main.py index 23cb487c..3c908ed9 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -2,6 +2,7 @@ from __future__ import print_function, absolute_import import os import signal import sys +import thread from netlib.version_check import check_pyopenssl_version, check_mitmproxy_version from . import version, cmdline from .exceptions import ServerException @@ -62,7 +63,7 @@ def mitmproxy(args=None): # pragma: nocover m = console.ConsoleMaster(server, console_options) try: m.run() - except KeyboardInterrupt: + except (KeyboardInterrupt, thread.error): pass @@ -97,7 +98,7 @@ def mitmdump(args=None): # pragma: nocover except dump.DumpError as e: print("mitmdump: %s" % e, file=sys.stderr) sys.exit(1) - except KeyboardInterrupt: + except (KeyboardInterrupt, thread.error): pass @@ -125,5 +126,5 @@ def mitmweb(args=None): # pragma: nocover m = web.WebMaster(server, web_options) try: m.run() - except KeyboardInterrupt: + except (KeyboardInterrupt, thread.error): pass diff --git a/libmproxy/platform/osx.py b/libmproxy/platform/osx.py index c5922850..2824718e 100644 --- a/libmproxy/platform/osx.py +++ b/libmproxy/platform/osx.py @@ -19,8 +19,17 @@ class Resolver(object): def original_addr(self, csock): peer = csock.getpeername() - stxt = subprocess.check_output(self.STATECMD, stderr=subprocess.STDOUT) - if "sudo: a password is required" in stxt: + try: + stxt = subprocess.check_output(self.STATECMD, stderr=subprocess.STDOUT) + except subprocess.CalledProcessError, e: + if "sudo: a password is required" in e.output: + insufficient_priv = True + else: + raise RuntimeError("Error getting pfctl state: " + repr(e)) + else: + insufficient_priv = "sudo: a password is required" in stxt + + if insufficient_priv: raise RuntimeError( "Insufficient privileges to access pfctl. " "See http://mitmproxy.org/doc/transparent/osx.html for details.") diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 8740927e..8d6a53c6 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -309,7 +309,10 @@ class HttpLayer(Layer): self.log("request", "debug", [repr(request)]) # Handle Proxy Authentication - if not self.authenticate(request): + # Proxy Authentication conceptually does not work in transparent mode. + # We catch this misconfiguration on startup. Here, we sort out requests + # after a successful CONNECT request (which do not need to be validated anymore) + if self.mode != "transparent" and not self.authenticate(request): return # Make sure that the incoming request matches our expectations diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index cd9eda5a..0b45d83e 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -141,6 +141,9 @@ def process_proxy_options(parser, options): if options.auth_nonanonymous or options.auth_singleuser or options.auth_htpasswd: + if options.transparent_proxy: + return parser.error("Proxy Authentication not supported in transparent mode.") + if options.socks_proxy: return parser.error( "Proxy Authentication not supported in SOCKS mode. " diff --git a/libmproxy/script.py b/libmproxy/script.py deleted file mode 100644 index 8bfacb38..00000000 --- a/libmproxy/script.py +++ /dev/null @@ -1,193 +0,0 @@ -from __future__ import absolute_import -import os -import traceback -import threading -import shlex -import sys - - -class ScriptError(Exception): - pass - - -class ScriptContext: - """ - The script context should be used to interact with the global mitmproxy state from within a - script. - """ - def __init__(self, master): - self._master = master - - def log(self, message, level="info"): - """ - Logs an event. - - By default, only events with level "error" get displayed. This can be controlled with the "-v" switch. - How log messages are handled depends on the front-end. mitmdump will print them to stdout, - mitmproxy sends output to the eventlog for display ("e" keyboard shortcut). - """ - self._master.add_event(message, level) - - def kill_flow(self, f): - """ - Kills a flow immediately. No further data will be sent to the client or the server. - """ - f.kill(self._master) - - def duplicate_flow(self, f): - """ - Returns a duplicate of the specified flow. The flow is also - injected into the current state, and is ready for editing, replay, - etc. - """ - self._master.pause_scripts = True - f = self._master.duplicate_flow(f) - self._master.pause_scripts = False - return f - - def replay_request(self, f): - """ - Replay the request on the current flow. The response will be added - to the flow object. - """ - return self._master.replay_request(f, block=True, run_scripthooks=False) - - @property - def app_registry(self): - return self._master.apps - - -class Script: - """ - Script object representing an inline script. - """ - - def __init__(self, command, master): - self.args = self.parse_command(command) - self.ctx = ScriptContext(master) - self.ns = None - self.load() - - @classmethod - def parse_command(cls, command): - if not command or not command.strip(): - raise ScriptError("Empty script command.") - if os.name == "nt": # Windows: escape all backslashes in the path. - backslashes = shlex.split(command, posix=False)[0].count("\\") - command = command.replace("\\", "\\\\", backslashes) - args = shlex.split(command) - args[0] = os.path.expanduser(args[0]) - if not os.path.exists(args[0]): - raise ScriptError( - ("Script file not found: %s.\r\n" - "If your script path contains spaces, " - "make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") % - args[0]) - elif os.path.isdir(args[0]): - raise ScriptError("Not a file: %s" % args[0]) - return args - - def load(self): - """ - Loads an inline script. - - Returns: - The return value of self.run("start", ...) - - Raises: - ScriptError on failure - """ - if self.ns is not None: - self.unload() - script_dir = os.path.dirname(os.path.abspath(self.args[0])) - ns = {'__file__': os.path.abspath(self.args[0])} - sys.path.append(script_dir) - try: - execfile(self.args[0], ns, ns) - except Exception as e: - # Python 3: use exception chaining, https://www.python.org/dev/peps/pep-3134/ - raise ScriptError(traceback.format_exc(e)) - sys.path.pop() - self.ns = ns - return self.run("start", self.args) - - def unload(self): - ret = self.run("done") - self.ns = None - return ret - - def run(self, name, *args, **kwargs): - """ - Runs an inline script hook. - - Returns: - The return value of the method. - None, if the script does not provide the method. - - Raises: - ScriptError if there was an exception. - """ - f = self.ns.get(name) - if f: - try: - return f(self.ctx, *args, **kwargs) - except Exception as e: - raise ScriptError(traceback.format_exc(e)) - else: - return None - - -class ReplyProxy(object): - def __init__(self, original_reply, script_thread): - self.original_reply = original_reply - self.script_thread = script_thread - self._ignore_call = True - self.lock = threading.Lock() - - def __call__(self, *args, **kwargs): - with self.lock: - if self._ignore_call: - self.script_thread.start() - self._ignore_call = False - return - self.original_reply(*args, **kwargs) - - def __getattr__(self, k): - return getattr(self.original_reply, k) - - -def _handle_concurrent_reply(fn, o, *args, **kwargs): - # Make first call to o.reply a no op and start the script thread. - # We must not start the script thread before, as this may lead to a nasty race condition - # where the script thread replies a different response before the normal reply, which then gets swallowed. - - def run(): - fn(*args, **kwargs) - # If the script did not call .reply(), we have to do it now. - reply_proxy() - - script_thread = ScriptThread(target=run) - - reply_proxy = ReplyProxy(o.reply, script_thread) - o.reply = reply_proxy - - -class ScriptThread(threading.Thread): - name = "ScriptThread" - - -def concurrent(fn): - if fn.func_name in ( - "request", - "response", - "error", - "clientconnect", - "serverconnect", - "clientdisconnect", - "next_layer"): - def _concurrent(ctx, obj): - _handle_concurrent_reply(fn, obj, ctx, obj) - - return _concurrent - raise NotImplementedError( - "Concurrent decorator not supported for '%s' method." % fn.func_name) diff --git a/libmproxy/script/__init__.py b/libmproxy/script/__init__.py new file mode 100644 index 00000000..8bcdc5a2 --- /dev/null +++ b/libmproxy/script/__init__.py @@ -0,0 +1,13 @@ +from .script import Script +from .script_context import ScriptContext +from .concurrent import concurrent +from ..exceptions import ScriptException +from . import reloader + +__all__ = [ + "Script", + "ScriptContext", + "concurrent", + "ScriptException", + "reloader" +]
\ No newline at end of file diff --git a/libmproxy/script/concurrent.py b/libmproxy/script/concurrent.py new file mode 100644 index 00000000..bee2d43b --- /dev/null +++ b/libmproxy/script/concurrent.py @@ -0,0 +1,62 @@ +""" +This module provides a @concurrent decorator primitive to +offload computations from mitmproxy's main master thread. +""" +from __future__ import absolute_import, print_function, division +import threading + + +class ReplyProxy(object): + def __init__(self, original_reply, script_thread): + self.original_reply = original_reply + self.script_thread = script_thread + self._ignore_call = True + self.lock = threading.Lock() + + def __call__(self, *args, **kwargs): + with self.lock: + if self._ignore_call: + self.script_thread.start() + self._ignore_call = False + return + self.original_reply(*args, **kwargs) + + def __getattr__(self, k): + return getattr(self.original_reply, k) + + +def _handle_concurrent_reply(fn, o, *args, **kwargs): + # Make first call to o.reply a no op and start the script thread. + # We must not start the script thread before, as this may lead to a nasty race condition + # where the script thread replies a different response before the normal reply, which then gets swallowed. + + def run(): + fn(*args, **kwargs) + # If the script did not call .reply(), we have to do it now. + reply_proxy() + + script_thread = ScriptThread(target=run) + + reply_proxy = ReplyProxy(o.reply, script_thread) + o.reply = reply_proxy + + +class ScriptThread(threading.Thread): + name = "ScriptThread" + + +def concurrent(fn): + if fn.func_name in ( + "request", + "response", + "error", + "clientconnect", + "serverconnect", + "clientdisconnect", + "next_layer"): + def _concurrent(ctx, obj): + _handle_concurrent_reply(fn, obj, ctx, obj) + + return _concurrent + raise NotImplementedError( + "Concurrent decorator not supported for '%s' method." % fn.func_name) diff --git a/libmproxy/script/reloader.py b/libmproxy/script/reloader.py new file mode 100644 index 00000000..b867238f --- /dev/null +++ b/libmproxy/script/reloader.py @@ -0,0 +1,37 @@ +import os +from watchdog.events import PatternMatchingEventHandler +from watchdog.observers import Observer + +_observers = {} + + +def watch(script, callback): + script_dir = os.path.dirname(os.path.abspath(script.args[0])) + event_handler = _ScriptModificationHandler(callback) + observer = Observer() + observer.schedule(event_handler, script_dir) + observer.start() + _observers[script] = observer + + +def unwatch(script): + observer = _observers.pop(script, None) + if observer: + observer.stop() + + +class _ScriptModificationHandler(PatternMatchingEventHandler): + def __init__(self, callback): + # We could enumerate all relevant *.py files (as werkzeug does it), + # but our case looks like it isn't as simple as enumerating sys.modules. + # This should be good enough for now. + super(_ScriptModificationHandler, self).__init__( + ignore_directories=True, + patterns=["*.py"] + ) + self.callback = callback + + def on_modified(self, event): + self.callback() + +__all__ = ["watch", "unwatch"]
\ No newline at end of file diff --git a/libmproxy/script/script.py b/libmproxy/script/script.py new file mode 100644 index 00000000..498caf94 --- /dev/null +++ b/libmproxy/script/script.py @@ -0,0 +1,97 @@ +""" +The script object representing mitmproxy inline scripts. +Script objects know nothing about mitmproxy or mitmproxy's API - this knowledge is provided +by the mitmproxy-specific ScriptContext. +""" +from __future__ import absolute_import, print_function, division +import os +import shlex +import traceback +import sys +from ..exceptions import ScriptException + + +class Script(object): + """ + Script object representing an inline script. + """ + + def __init__(self, command, context): + self.command = command + self.args = self.parse_command(command) + self.ctx = context + self.ns = None + self.load() + + @property + def filename(self): + return self.args[0] + + @staticmethod + def parse_command(command): + if not command or not command.strip(): + raise ScriptException("Empty script command.") + if os.name == "nt": # Windows: escape all backslashes in the path. + backslashes = shlex.split(command, posix=False)[0].count("\\") + command = command.replace("\\", "\\\\", backslashes) + args = shlex.split(command) + args[0] = os.path.expanduser(args[0]) + if not os.path.exists(args[0]): + raise ScriptException( + ("Script file not found: %s.\r\n" + "If your script path contains spaces, " + "make sure to wrap it in additional quotes, e.g. -s \"'./foo bar/baz.py' --args\".") % + args[0]) + elif os.path.isdir(args[0]): + raise ScriptException("Not a file: %s" % args[0]) + return args + + def load(self): + """ + Loads an inline script. + + Returns: + The return value of self.run("start", ...) + + Raises: + ScriptException on failure + """ + if self.ns is not None: + self.unload() + script_dir = os.path.dirname(os.path.abspath(self.args[0])) + self.ns = {'__file__': os.path.abspath(self.args[0])} + sys.path.append(script_dir) + try: + execfile(self.args[0], self.ns, self.ns) + except Exception as e: + # Python 3: use exception chaining, https://www.python.org/dev/peps/pep-3134/ + raise ScriptException(traceback.format_exc(e)) + finally: + sys.path.pop() + return self.run("start", self.args) + + def unload(self): + try: + return self.run("done") + finally: + self.ns = None + + def run(self, name, *args, **kwargs): + """ + Runs an inline script hook. + + Returns: + The return value of the method. + None, if the script does not provide the method. + + Raises: + ScriptException if there was an exception. + """ + f = self.ns.get(name) + if f: + try: + return f(self.ctx, *args, **kwargs) + except Exception as e: + raise ScriptException(traceback.format_exc(e)) + else: + return None
\ No newline at end of file diff --git a/libmproxy/script/script_context.py b/libmproxy/script/script_context.py new file mode 100644 index 00000000..d8748cb2 --- /dev/null +++ b/libmproxy/script/script_context.py @@ -0,0 +1,59 @@ +""" +The mitmproxy script context provides an API to inline scripts. +""" +from __future__ import absolute_import, print_function, division +from .. import contentviews + + +class ScriptContext(object): + """ + The script context should be used to interact with the global mitmproxy state from within a + script. + """ + + def __init__(self, master): + self._master = master + + def log(self, message, level="info"): + """ + Logs an event. + + By default, only events with level "error" get displayed. This can be controlled with the "-v" switch. + How log messages are handled depends on the front-end. mitmdump will print them to stdout, + mitmproxy sends output to the eventlog for display ("e" keyboard shortcut). + """ + self._master.add_event(message, level) + + def kill_flow(self, f): + """ + Kills a flow immediately. No further data will be sent to the client or the server. + """ + f.kill(self._master) + + def duplicate_flow(self, f): + """ + Returns a duplicate of the specified flow. The flow is also + injected into the current state, and is ready for editing, replay, + etc. + """ + self._master.pause_scripts = True + f = self._master.duplicate_flow(f) + self._master.pause_scripts = False + return f + + def replay_request(self, f): + """ + Replay the request on the current flow. The response will be added + to the flow object. + """ + return self._master.replay_request(f, block=True, run_scripthooks=False) + + @property + def app_registry(self): + return self._master.apps + + def add_contentview(self, view_obj): + contentviews.add(view_obj) + + def remove_contentview(self, view_obj): + contentviews.remove(view_obj) diff --git a/libmproxy/version.py b/libmproxy/version.py index 0af60af5..5ed89732 100644 --- a/libmproxy/version.py +++ b/libmproxy/version.py @@ -1,6 +1,6 @@ from __future__ import (absolute_import, print_function, division) -IVERSION = (0, 13, 1) +IVERSION = (0, 14, 1) VERSION = ".".join(str(i) for i in IVERSION) MINORVERSION = ".".join(str(i) for i in IVERSION[:2]) NAME = "mitmproxy" diff --git a/release/mitmdump.spec b/release/mitmdump.spec index 5ea6d0ce..e3a6905d 100644 --- a/release/mitmdump.spec +++ b/release/mitmdump.spec @@ -4,7 +4,7 @@ from glob import glob VENV = "../release/venv" -a = Analysis(['../mitmproxy/mitmdump'], +a = Analysis(['../../mitmproxy/mitmdump'], hiddenimports=[], hookspath=None, runtime_hooks=None, diff --git a/release/mitmproxy.spec b/release/mitmproxy.spec index a5a75d75..1f0a515b 100644 --- a/release/mitmproxy.spec +++ b/release/mitmproxy.spec @@ -4,7 +4,7 @@ from glob import glob VENV = "../release/venv" -a = Analysis(['../mitmproxy/mitmproxy'], +a = Analysis(['../../mitmproxy/mitmproxy'], hiddenimports=[], hookspath=None, runtime_hooks=None, @@ -15,25 +15,26 @@ with open(os.path.join(here, 'README.rst'), encoding='utf-8') as f: # Core dependencies deps = { "netlib>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION), - "pyasn1>0.1.2", - "tornado>=4.0.2", - "configargparse>=0.9.3", - "pyperclip>=1.5.8", - "blinker>=1.3", - "pyparsing>=1.5.2", - "html2text>=2015.4.14", - "construct>=2.5.2", - "six>=1.9.0", - "lxml>=3.3.6", - "Pillow>=2.3.0", + "pyasn1~=0.1.9", + "tornado~=4.3.0", + "configargparse~=0.10.0", + "pyperclip~=1.5.22", + "blinker~=1.4", + "pyparsing~=2.0.5", + "html2text~=2015.11.4", + "construct~=2.5.2", + "six~=1.10.0", + "lxml~=3.4.4", + "Pillow~=3.0.0", + "watchdog~=0.8.3", } # A script -> additional dependencies dict. scripts = { "mitmproxy": { - "urwid>=1.3", + "urwid~=1.3.1", }, "mitmdump": { - "click>=5.1", + "click~=5.1", }, "mitmweb": set() } @@ -50,9 +51,9 @@ dev_deps = { "sphinxcontrib-documentedlist>=0.2", } example_deps = { - "pytz", - "harparser", - "beautifulsoup4", + "pytz~=2015.7", + "harparser~=0.2", + "beautifulsoup4~=4.4.1", } # Add *all* script dependencies to developer dependencies. for script_deps in scripts.values(): @@ -61,14 +62,14 @@ for script_deps in scripts.values(): # Remove mitmproxy for Windows support. if os.name == "nt": del scripts["mitmproxy"] - deps.add("pydivert>=0.0.7") # Transparent proxying on Windows + deps.add("pydivert~=0.0.7") # Transparent proxying on Windows # Add dependencies for available scripts as core dependencies. for script_deps in scripts.values(): deps.update(script_deps) if sys.version_info < (3, 4): - example_deps.add("enum34") + example_deps.add("enum34~=1.0.4") console_scripts = ["%s = libmproxy.main:%s" % (s, s) for s in scripts.keys()] @@ -107,9 +108,9 @@ setup( extras_require={ 'dev': list(dev_deps), 'contentviews': [ - "pyamf>=0.6.1", - "protobuf>=2.5.0", - "cssutils>=1.0" + "pyamf~=0.7.2", + "protobuf~=2.6.1", + "cssutils~=1.0.1" ], 'examples': list(example_deps) } diff --git a/test/test_contentview.py b/test/test_contentview.py index 97608520..2a70b414 100644 --- a/test/test_contentview.py +++ b/test/test_contentview.py @@ -210,6 +210,21 @@ Larry assert "decoded gzip" in r[0] assert "Raw" in r[0] + def test_add_cv(self): + class TestContentView(cv.View): + name = "test" + prompt = ("t", "test") + + tcv = TestContentView() + cv.add(tcv) + + # repeated addition causes exception + tutils.raises( + ContentViewException, + cv.add, + tcv + ) + if pyamf: def test_view_amf_request(): @@ -233,7 +248,7 @@ if cv.ViewProtobuf.is_available(): p = tutils.test_data.path("data/protobuf01") content_type, output = v(file(p, "rb").read()) assert content_type == "Protobuf" - assert output[0].text == '1: "3bbc333c-e61c-433b-819a-0b9a8cc103b8"' + assert output.next()[0][1] == '1: "3bbc333c-e61c-433b-819a-0b9a8cc103b8"' def test_get_by_shortcut(): diff --git a/test/test_custom_contentview.py b/test/test_custom_contentview.py new file mode 100644 index 00000000..4b5a3e53 --- /dev/null +++ b/test/test_custom_contentview.py @@ -0,0 +1,52 @@ +from libmproxy import script, flow +import libmproxy.contentviews as cv +from netlib.http import Headers + + +def test_custom_views(): + class ViewNoop(cv.View): + name = "noop" + prompt = ("noop", "n") + content_types = ["text/none"] + + def __call__(self, data, **metadata): + return "noop", cv.format_text(data) + + + view_obj = ViewNoop() + + cv.add(view_obj) + + assert cv.get("noop") + + r = cv.get_content_view( + cv.get("noop"), + "[1, 2, 3]", + headers=Headers( + content_type="text/plain" + ) + ) + assert "noop" in r[0] + + # now try content-type matching + r = cv.get_content_view( + cv.get("Auto"), + "[1, 2, 3]", + headers=Headers( + content_type="text/none" + ) + ) + assert "noop" in r[0] + + # now try removing the custom view + cv.remove(view_obj) + r = cv.get_content_view( + cv.get("Auto"), + "[1, 2, 3]", + headers=Headers( + content_type="text/none" + ) + ) + assert "noop" not in r[0] + + diff --git a/test/test_examples.py b/test/test_examples.py index dce257cf..2a30f9d5 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -22,7 +22,7 @@ def test_load_scripts(): if "modify_response_body" in f: f += " foo bar" # two arguments required try: - s = script.Script(f, tmaster) # Loads the script file. + s = script.Script(f, script.ScriptContext(tmaster)) # Loads the script file. except Exception as v: if "ImportError" not in str(v): raise diff --git a/test/test_script.py b/test/test_script.py index 1b0e5a5b..fbe3e107 100644 --- a/test/test_script.py +++ b/test/test_script.py @@ -9,13 +9,13 @@ def test_simple(): s = flow.State() fm = flow.FlowMaster(None, s) sp = tutils.test_data.path("scripts/a.py") - p = script.Script("%s --var 40" % sp, fm) + p = script.Script("%s --var 40" % sp, script.ScriptContext(fm)) assert "here" in p.ns assert p.run("here") == 41 assert p.run("here") == 42 - tutils.raises(script.ScriptError, p.run, "errargs") + tutils.raises(script.ScriptException, p.run, "errargs") # Check reload p.load() @@ -36,29 +36,30 @@ def test_duplicate_flow(): def test_err(): s = flow.State() fm = flow.FlowMaster(None, s) + sc = script.ScriptContext(fm) tutils.raises( "not found", - script.Script, "nonexistent", fm + script.Script, "nonexistent", sc ) tutils.raises( "not a file", - script.Script, tutils.test_data.path("scripts"), fm + script.Script, tutils.test_data.path("scripts"), sc ) tutils.raises( - script.ScriptError, - script.Script, tutils.test_data.path("scripts/syntaxerr.py"), fm + script.ScriptException, + script.Script, tutils.test_data.path("scripts/syntaxerr.py"), sc ) tutils.raises( - script.ScriptError, - script.Script, tutils.test_data.path("scripts/loaderr.py"), fm + script.ScriptException, + script.Script, tutils.test_data.path("scripts/loaderr.py"), sc ) - scr = script.Script(tutils.test_data.path("scripts/unloaderr.py"), fm) - tutils.raises(script.ScriptError, scr.unload) + scr = script.Script(tutils.test_data.path("scripts/unloaderr.py"), sc) + tutils.raises(script.ScriptException, scr.unload) def test_concurrent(): @@ -84,7 +85,7 @@ def test_concurrent2(): fm = flow.FlowMaster(None, s) s = script.Script( tutils.test_data.path("scripts/concurrent_decorator.py"), - fm) + script.ScriptContext(fm)) s.load() m = mock.Mock() @@ -125,5 +126,6 @@ def test_command_parsing(): s = flow.State() fm = flow.FlowMaster(None, s) absfilepath = os.path.normcase(tutils.test_data.path("scripts/a.py")) - s = script.Script(absfilepath, fm) + s = script.Script(absfilepath, script.ScriptContext(fm)) assert os.path.isfile(s.args[0]) + diff --git a/test/test_server.py b/test/test_server.py index 0bf8b9c7..83b6423a 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -516,9 +516,8 @@ class TestProxy(tservers.HTTPProxTest): assert f.status_code == 304 response = self.master.state.view[0].response - # time.sleep might be a little bit shorter than a second, - # we observed up to 0.93s on appveyor. - assert 0.8 <= response.timestamp_end - response.timestamp_start + # timestamp_start might fire a bit late, so we play safe and only require 300ms. + assert 0.3 <= response.timestamp_end - response.timestamp_start def test_request_timestamps(self): # test that we notice a delay between timestamps in request object @@ -537,9 +536,8 @@ class TestProxy(tservers.HTTPProxTest): request, response = self.master.state.view[ 0].request, self.master.state.view[0].response assert response.status_code == 304 # sanity test for our low level request - # time.sleep might be a little bit shorter than a second, - # we observed up to 0.93s on appveyor. - assert 0.8 < (request.timestamp_end - request.timestamp_start) + # timestamp_start might fire a bit late, so we play safe and only require 300ms. + assert 0.3 < (request.timestamp_end - request.timestamp_start) def test_request_tcp_setup_timestamp_presence(self): # tests that the client_conn a tcp connection has a tcp_setup_timestamp |