diff options
Diffstat (limited to 'libmproxy')
| -rw-r--r-- | libmproxy/cmdline.py | 316 | ||||
| -rw-r--r-- | libmproxy/console/__init__.py | 37 | ||||
| -rw-r--r-- | libmproxy/console/flowlist.py | 6 | ||||
| -rw-r--r-- | libmproxy/console/flowview.py | 5 | ||||
| -rw-r--r-- | libmproxy/console/grideditor.py | 13 | ||||
| -rw-r--r-- | libmproxy/console/help.py | 1 | ||||
| -rw-r--r-- | libmproxy/dump.py | 15 | ||||
| -rw-r--r-- | libmproxy/filt.py | 4 | ||||
| -rw-r--r-- | libmproxy/flow.py | 93 | ||||
| -rw-r--r-- | libmproxy/main.py | 165 | ||||
| -rw-r--r-- | libmproxy/onboarding/app.py | 4 | ||||
| -rw-r--r-- | libmproxy/onboarding/templates/index.html | 11 | ||||
| -rw-r--r-- | libmproxy/platform/windows.py | 4 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 425 | ||||
| -rw-r--r-- | libmproxy/protocol/primitives.py | 19 | ||||
| -rw-r--r-- | libmproxy/protocol/tcp.py | 35 | ||||
| -rw-r--r-- | libmproxy/proxy/config.py | 91 | ||||
| -rw-r--r-- | libmproxy/proxy/primitives.py | 81 | ||||
| -rw-r--r-- | libmproxy/proxy/server.py | 42 | ||||
| -rw-r--r-- | libmproxy/version.py | 6 | ||||
| -rw-r--r-- | libmproxy/web/static/flows.json | 28 |
21 files changed, 936 insertions, 465 deletions
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index f6cd1ab8..b892f1fd 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -1,9 +1,9 @@ from __future__ import absolute_import +import os import re -import argparse -from argparse import ArgumentTypeError +import configargparse from netlib import http -from . import filt, utils +from . import filt, utils, version from .proxy import config APP_HOST = "mitm.it" @@ -23,7 +23,9 @@ def _parse_hook(s): elif len(parts) == 3: patt, a, b = parts else: - raise ParseException("Malformed hook specifier - too few clauses: %s" % s) + raise ParseException( + "Malformed hook specifier - too few clauses: %s" % s + ) if not a: raise ParseException("Empty clause: %s" % str(patt)) @@ -102,7 +104,9 @@ def parse_server_spec(url): p = http.parse_url(normalized_url) if not p or not p[1]: - raise ArgumentTypeError("Invalid server specification: %s" % url) + raise configargparse.ArgumentTypeError( + "Invalid server specification: %s" % url + ) if url.lower().startswith("https2http"): ssl = [True, False] @@ -131,17 +135,19 @@ def get_common_options(options): try: p = parse_replace_hook(i) except ParseException, e: - raise ArgumentTypeError(e.message) + raise configargparse.ArgumentTypeError(e.message) reps.append(p) for i in options.replace_file: try: patt, rex, path = parse_replace_hook(i) except ParseException, e: - raise ArgumentTypeError(e.message) + raise configargparse.ArgumentTypeError(e.message) try: v = open(path, "rb").read() except IOError, e: - raise ArgumentTypeError("Could not read replace file: %s" % path) + raise configargparse.ArgumentTypeError( + "Could not read replace file: %s" % path + ) reps.append((patt, rex, v)) setheaders = [] @@ -149,7 +155,7 @@ def get_common_options(options): try: p = parse_setheader(i) except ParseException, e: - raise ArgumentTypeError(e.message) + raise configargparse.ArgumentTypeError(e.message) setheaders.append(p) return dict( @@ -183,14 +189,23 @@ def get_common_options(options): def common_options(parser): parser.add_argument( + '--version', + action= 'version', + version= "%(prog)s" + " " + version.VERSION + ) + parser.add_argument( "--anticache", action="store_true", dest="anticache", default=False, - help="Strip out request headers that might cause the server to return 304-not-modified." + + help=""" + Strip out request headers that might cause the server to return + 304-not-modified. + """ ) parser.add_argument( - "--confdir", - action="store", type=str, dest="confdir", default='~/.mitmproxy', - help="Configuration directory. (~/.mitmproxy)" + "--cadir", + action="store", type=str, dest="cadir", default=config.CA_DIR, + help="Location of the default mitmproxy CA files. (%s)"%config.CA_DIR ) parser.add_argument( "--host", @@ -198,112 +213,150 @@ def common_options(parser): help="Use the Host header to construct URLs for display." ) parser.add_argument( - "-q", + "-q", "--quiet", action="store_true", dest="quiet", help="Quiet." ) parser.add_argument( - "-r", + "-r", "--read-flows", action="store", dest="rfile", default=None, help="Read flows from file." ) parser.add_argument( - "-s", + "-s", "--script", action="append", type=str, dest="scripts", default=[], metavar='"script.py --bar"', - help="Run a script. Surround with quotes to pass script arguments. Can be passed multiple times." + help=""" + Run a script. Surround with quotes to pass script arguments. Can be + passed multiple times. + """ ) parser.add_argument( - "-t", - action="store", dest="stickycookie_filt", default=None, metavar="FILTER", + "-t", "--stickycookie", + action="store", + dest="stickycookie_filt", + default=None, + metavar="FILTER", help="Set sticky cookie filter. Matched against requests." ) parser.add_argument( - "-u", + "-u", "--stickyauth", action="store", dest="stickyauth_filt", default=None, metavar="FILTER", help="Set sticky auth filter. Matched against requests." ) parser.add_argument( - "-v", + "-v", "--verbose", action="store_const", dest="verbose", default=1, const=2, help="Increase event log verbosity." ) parser.add_argument( - "-w", + "-w", "--wfile", action="store", dest="wfile", default=None, help="Write flows to file." ) parser.add_argument( - "-z", + "-z", "--anticomp", action="store_true", dest="anticomp", default=False, help="Try to convince servers to send us un-compressed data." ) parser.add_argument( - "-Z", + "-Z", "--body-size-limit", action="store", dest="body_size_limit", default=None, metavar="SIZE", - help="Byte size limit of HTTP request and response bodies." \ + help="Byte size limit of HTTP request and response bodies." " Understands k/m/g suffixes, i.e. 3m for 3 megabytes." ) parser.add_argument( "--stream", action="store", dest="stream_large_bodies", default=None, metavar="SIZE", - help="Stream data to the client if response body exceeds the given threshold. " - "If streamed, the body will not be stored in any way. Understands k/m/g suffixes, i.e. 3m for 3 megabytes." + help=""" + Stream data to the client if response body exceeds the given + threshold. If streamed, the body will not be stored in any way. + Understands k/m/g suffixes, i.e. 3m for 3 megabytes. + """ ) group = parser.add_argument_group("Proxy Options") - # We could make a mutually exclusive group out of -R, -U, -T, but we don't do that because - # - --upstream-server should be in that group as well, but it's already in a different group. - # - our own error messages are more helpful + # We could make a mutually exclusive group out of -R, -U, -T, but we don't + # do that because - --upstream-server should be in that group as well, but + # it's already in a different group. - our own error messages are more + # helpful group.add_argument( - "-b", + "-b", "--bind-address", action="store", type=str, dest="addr", default='', help="Address to bind proxy to (defaults to all interfaces)" ) group.add_argument( "-I", "--ignore", - action="append", type=str, dest="ignore", default=[], + action="append", type=str, dest="ignore_hosts", default=[], metavar="HOST", - help="Ignore host and forward all traffic without processing it. " - "In transparent mode, it is recommended to use an IP address (range), not the hostname. " - "In regular mode, only SSL traffic is ignored and the hostname should be used. " - "The supplied value is interpreted as a regular expression and matched on the ip or the hostname. " - "Can be passed multiple times. " + help=""" + Ignore host and forward all traffic without processing it. In + transparent mode, it is recommended to use an IP address (range), + not the hostname. In regular mode, only SSL traffic is ignored and + the hostname should be used. The supplied value is interpreted as a + regular expression and matched on the ip or the hostname. Can be + passed multiple times. + """ ) group.add_argument( - "-n", + "--tcp", + action="append", type=str, dest="tcp_hosts", default=[], + metavar="HOST", + help=""" + Generic TCP SSL proxy mode for all hosts that match the pattern. + Similar to --ignore, but SSL connections are intercepted. The + communication contents are printed to the event log in verbose mode. + """ + ) + group.add_argument( + "-n", "--no-server", action="store_true", dest="no_server", help="Don't start a proxy server." ) group.add_argument( - "-p", + "-p", "--port", action="store", type=int, dest="port", default=8080, help="Proxy service port." ) group.add_argument( - "-R", - action="store", type=parse_server_spec, dest="reverse_proxy", default=None, - help="Forward all requests to upstream HTTP server: http[s][2http[s]]://host[:port]" + "-R", "--reverse", + action="store", + type=parse_server_spec, + dest="reverse_proxy", + default=None, + help=""" + Forward all requests to upstream HTTP server: + http[s][2http[s]]://host[:port] + """ ) group.add_argument( - "-T", + "--socks", + action="store_true", dest="socks_proxy", default=False, + help="Set SOCKS5 proxy mode." + ) + group.add_argument( + "-T", "--transparent", action="store_true", dest="transparent_proxy", default=False, help="Set transparent proxy mode." ) group.add_argument( - "-U", - action="store", type=parse_server_spec, dest="upstream_proxy", default=None, + "-U", "--upstream", + action="store", + type=parse_server_spec, + dest="upstream_proxy", + default=None, help="Forward all requests to upstream proxy server: http://host[:port]" ) group = parser.add_argument_group( "Advanced Proxy Options", """ - The following options allow a custom adjustment of the proxy behavior. - Normally, you don't want to use these options directly and use the provided wrappers instead (-R, -U, -T). - """.strip() + The following options allow a custom adjustment of the proxy + behavior. Normally, you don't want to use these options directly and + use the provided wrappers instead (-R, -U, -T). + """ ) group.add_argument( "--http-form-in", dest="http_form_in", default=None, @@ -318,38 +371,44 @@ def common_options(parser): group = parser.add_argument_group("Onboarding App") group.add_argument( - "-a", + "-a", "--noapp", action="store_false", dest="app", default=True, help="Disable the mitmproxy onboarding app." ) group.add_argument( "--app-host", action="store", dest="app_host", default=APP_HOST, metavar="host", - help="Domain to serve the onboarding app from. For transparent mode, use an IP when\ - a DNS entry for the app domain is not present. Default: %s" % APP_HOST - + help=""" + Domain to serve the onboarding app from. For transparent mode, use + an IP when a DNS entry for the app domain is not present. Default: + %s + """ % APP_HOST ) group.add_argument( "--app-port", - action="store", dest="app_port", default=APP_PORT, type=int, metavar="80", + action="store", + dest="app_port", + default=APP_PORT, + type=int, + metavar="80", help="Port to serve the onboarding app from." ) group = parser.add_argument_group("Client Replay") group.add_argument( - "-c", + "-c", "--client-replay", action="store", dest="client_replay", default=None, metavar="PATH", help="Replay client requests from a saved file." ) group = parser.add_argument_group("Server Replay") group.add_argument( - "-S", + "-S", "--server-replay", action="store", dest="server_replay", default=None, metavar="PATH", help="Replay server responses from a saved file." ) group.add_argument( - "-k", + "-k", "--kill", action="store_true", dest="kill", default=False, help="Kill extra requests during replay." ) @@ -362,8 +421,10 @@ def common_options(parser): group.add_argument( "--norefresh", action="store_true", dest="norefresh", default=False, - help="Disable response refresh, " - "which updates times in cookies and headers for replayed responses." + help=""" + Disable response refresh, which updates times in cookies and headers + for replayed responses. + """ ) group.add_argument( "--no-pop", @@ -374,14 +435,18 @@ def common_options(parser): group.add_argument( "--replay-ignore-content", action="store_true", dest="replay_ignore_content", default=False, - help="Ignore request's content while searching for a saved flow to replay" + help=""" + Ignore request's content while searching for a saved flow to replay + """ ) group.add_argument( "--replay-ignore-param", action="append", dest="replay_ignore_params", type=str, - help="Request's parameters to be ignored while searching for a saved flow to replay" - "Can be passed multiple times." - ) + help=""" + Request's parameters to be ignored while searching for a saved flow + to replay. Can be passed multiple times. + """ + ) group = parser.add_argument_group( "Replacements", @@ -399,9 +464,12 @@ def common_options(parser): ) group.add_argument( "--replace-from-file", - action="append", type=str, dest="replace_file", default=[], - metavar="PATH", - help="Replacement pattern, where the replacement clause is a path to a file." + action = "append", type=str, dest="replace_file", default=[], + metavar = "PATH", + help = """ + Replacement pattern, where the replacement clause is a path to a + file. + """ ) group = parser.add_argument_group( @@ -437,7 +505,10 @@ def common_options(parser): "--singleuser", action="store", dest="auth_singleuser", type=str, metavar="USER", - help="Allows access to a a single user, specified in the form username:password." + help=""" + Allows access to a a single user, specified in the form + username:password. + """ ) user_specification_group.add_argument( "--htpasswd", @@ -447,3 +518,116 @@ def common_options(parser): ) config.ssl_option_group(parser) + + +def mitmproxy(): + # Don't import libmproxy.console for mitmdump, urwid is not available on all + # platforms. + from .console import palettes + + parser = configargparse.ArgumentParser( + usage="%(prog)s [options]", + args_for_setting_config_path = ["--conf"], + default_config_files = [ + os.path.join(config.CA_DIR, "common.conf"), + os.path.join(config.CA_DIR, "mitmproxy.conf") + ], + add_config_file_help = True, + add_env_var_help = True + ) + common_options(parser) + parser.add_argument( + "--palette", type=str, default="dark", + action="store", dest="palette", + help="Select color palette: " + ", ".join(palettes.palettes.keys()) + ) + parser.add_argument( + "-e", "--eventlog", + action="store_true", dest="eventlog", + help="Show event log." + ) + group = parser.add_argument_group( + "Filters", + "See help in mitmproxy for filter expression syntax." + ) + group.add_argument( + "-i", "--intercept", action="store", + type=str, dest="intercept", default=None, + help="Intercept filter expression." + ) + return parser + + +def mitmdump(): + parser = configargparse.ArgumentParser( + usage="%(prog)s [options] [filter]", + args_for_setting_config_path = ["--conf"], + default_config_files = [ + os.path.join(config.CA_DIR, "common.conf"), + os.path.join(config.CA_DIR, "mitmdump.conf") + ], + add_config_file_help = True, + add_env_var_help = True + ) + + common_options(parser) + parser.add_argument( + "--keepserving", + action= "store_true", dest="keepserving", default=False, + help= """ + Continue serving after client playback or file read. We exit by + default. + """ + ) + parser.add_argument( + "-d", "--detail", + action="count", dest="flow_detail", default=1, + help="Increase flow detail display level. Can be passed multiple times." + ) + parser.add_argument('args', nargs="...") + return parser + + +def mitmweb(): + parser = configargparse.ArgumentParser( + usage="%(prog)s [options]", + args_for_setting_config_path = ["--conf"], + default_config_files = [ + os.path.join(config.CA_DIR, "common.conf"), + os.path.join(config.CA_DIR, "mitmweb.conf") + ], + add_config_file_help = True, + add_env_var_help = True + ) + + group = parser.add_argument_group("Mitmweb") + group.add_argument( + "--wport", + action="store", type=int, dest="wport", default=8081, + metavar="PORT", + help="Mitmweb port." + ) + group.add_argument( + "--wiface", + action="store", dest="wiface", default="127.0.0.1", + metavar="IFACE", + help="Mitmweb interface." + ) + group.add_argument( + "--wdebug", + action="store_true", dest="wdebug", + help="Turn on mitmweb debugging" + ) + + common_options(parser) + group = parser.add_argument_group( + "Filters", + "See help in mitmproxy for filter expression syntax." + ) + group.add_argument( + "-i", "--intercept", action="store", + type=str, dest="intercept", default=None, + help="Intercept filter expression." + ) + return parser + diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 9c4b4827..e6bc9b41 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -129,10 +129,14 @@ class StatusBar(common.WWrap): r.append(":%s in file]"%self.master.server_playback.count()) else: r.append(":%s to go]"%self.master.server_playback.count()) - if self.master.get_ignore(): + if self.master.get_ignore_filter(): r.append("[") r.append(("heading_key", "I")) - r.append("gnore:%d]"%len(self.master.get_ignore())) + r.append("gnore:%d]" % len(self.master.get_ignore_filter())) + if self.master.get_tcp_filter(): + r.append("[") + r.append(("heading_key", "T")) + r.append("CP:%d]" % len(self.master.get_tcp_filter())) if self.master.state.intercept_txt: r.append("[") r.append(("heading_key", "i")) @@ -512,7 +516,8 @@ class ConsoleMaster(flow.FlowMaster): self.start_server_playback( ret, self.killextra, self.rheaders, - False, self.nopop + False, self.nopop, + self.options.replay_ignore_params, self.options.replay_ignore_content ) def spawn_editor(self, data): @@ -798,9 +803,13 @@ class ConsoleMaster(flow.FlowMaster): for command in commands: self.load_script(command) - def edit_ignore(self, ignore): + def edit_ignore_filter(self, ignore): patterns = (x[0] for x in ignore) - self.set_ignore(patterns) + self.set_ignore_filter(patterns) + + def edit_tcp_filter(self, tcp): + patterns = (x[0] for x in tcp) + self.set_tcp_filter(patterns) def loop(self): changed = True @@ -811,7 +820,7 @@ class ConsoleMaster(flow.FlowMaster): self.statusbar.redraw() size = self.drawscreen() changed = self.tick(self.masterq, 0.01) - self.ui.set_input_timeouts(max_wait=0.1) + self.ui.set_input_timeouts(max_wait=0.01) keys = self.ui.get_input() if keys: changed = True @@ -860,10 +869,18 @@ class ConsoleMaster(flow.FlowMaster): ) elif k == "I": self.view_grideditor( - grideditor.IgnoreEditor( + grideditor.HostPatternEditor( + self, + [[x] for x in self.get_ignore_filter()], + self.edit_ignore_filter + ) + ) + elif k == "T": + self.view_grideditor( + grideditor.HostPatternEditor( self, - [[x] for x in self.get_ignore()], - self.edit_ignore + [[x] for x in self.get_tcp_filter()], + self.edit_tcp_filter ) ) elif k == "i": @@ -1033,7 +1050,7 @@ class ConsoleMaster(flow.FlowMaster): self.eventlist[:] = [] def add_event(self, e, level="info"): - needed = dict(error=1, info=1, debug=2).get(level, 1) + needed = dict(error=0, info=1, debug=2).get(level, 1) if self.options.verbosity < needed: return diff --git a/libmproxy/console/flowlist.py b/libmproxy/console/flowlist.py index e0330171..3eb4eb1a 100644 --- a/libmproxy/console/flowlist.py +++ b/libmproxy/console/flowlist.py @@ -120,13 +120,15 @@ class ConnectionItem(common.WWrap): self.master.start_server_playback( [i.copy() for i in self.master.state.view], self.master.killextra, self.master.rheaders, - False, self.master.nopop + False, self.master.nopop, + self.master.options.replay_ignore_params, self.master.options.replay_ignore_content ) elif k == "t": self.master.start_server_playback( [self.flow.copy()], self.master.killextra, self.master.rheaders, - False, self.master.nopop + False, self.master.nopop, + self.master.options.replay_ignore_params, self.master.options.replay_ignore_content ) else: self.master.path_prompt( diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index b2c46147..1ec57a4e 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -574,9 +574,8 @@ class FlowView(common.WWrap): else: if not self.flow.response: self.flow.response = HTTPResponse( - self.flow.request, self.flow.request.httpversion, - 200, "OK", flow.ODictCaseless(), "", None + 200, "OK", flow.ODictCaseless(), "" ) self.flow.response.reply = controller.DummyReply() conn = self.flow.response @@ -749,7 +748,7 @@ class FlowView(common.WWrap): self.master.statusbar.message("") elif key == "m": p = list(contentview.view_prompts) - p.insert(0, ("clear", "c")) + p.insert(0, ("Clear", "C")) self.master.prompt_onekey( "Display mode", p, diff --git a/libmproxy/console/grideditor.py b/libmproxy/console/grideditor.py index d629ec82..438d0ad7 100644 --- a/libmproxy/console/grideditor.py +++ b/libmproxy/console/grideditor.py @@ -123,12 +123,13 @@ class GridWalker(urwid.ListWalker): except ValueError: self.editor.master.statusbar.message("Invalid Python-style string encoding.", 1000) return - errors = self.lst[self.focus][1] emsg = self.editor.is_error(self.focus_col, val) if emsg: self.editor.master.statusbar.message(emsg, 1000) errors.add(self.focus_col) + else: + errors.discard(self.focus_col) row = list(self.lst[self.focus][0]) row[self.focus_col] = val @@ -320,9 +321,11 @@ class GridEditor(common.WWrap): elif key == "d": self.walker.delete_focus() elif key == "r": - self.master.path_prompt("Read file: ", "", self.read_file) + if self.walker.get_current_value() is not None: + self.master.path_prompt("Read file: ", "", self.read_file) elif key == "R": - self.master.path_prompt("Read unescaped file: ", "", self.read_file, True) + if self.walker.get_current_value() is not None: + self.master.path_prompt("Read unescaped file: ", "", self.read_file, True) elif key == "e": o = self.walker.get_current_value() if o is not None: @@ -495,8 +498,8 @@ class ScriptEditor(GridEditor): return str(v) -class IgnoreEditor(GridEditor): - title = "Editing ignore patterns" +class HostPatternEditor(GridEditor): + title = "Editing host patterns" columns = 1 headings = ("Regex (matched on hostname:port / ip:port)",) diff --git a/libmproxy/console/help.py b/libmproxy/console/help.py index bdcf3fd9..27288a36 100644 --- a/libmproxy/console/help.py +++ b/libmproxy/console/help.py @@ -119,6 +119,7 @@ class HelpView(urwid.ListBox): ("s", "add/remove scripts"), ("S", "server replay"), ("t", "set sticky cookie expression"), + ("T", "set tcp proxying pattern"), ("u", "set sticky auth expression"), ] text.extend(common.format_keyvals(keys, key="key", val="text", indent=4)) diff --git a/libmproxy/dump.py b/libmproxy/dump.py index ccb2b5b5..0d9432c9 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -1,10 +1,13 @@ from __future__ import absolute_import -import sys, os +import sys +import os import netlib.utils from . import flow, filt, utils from .protocol import http -class DumpError(Exception): pass + +class DumpError(Exception): + pass class Options(object): @@ -37,6 +40,7 @@ class Options(object): "replay_ignore_content", "replay_ignore_params", ] + def __init__(self, **kwargs): for k, v in kwargs.items(): setattr(self, k, v) @@ -71,7 +75,7 @@ class DumpMaster(flow.FlowMaster): self.anticache = options.anticache self.anticomp = options.anticomp self.showhost = options.showhost - self.replay_ignore_params = options.replay_ignore_params + self.replay_ignore_params = options.replay_ignore_params self.replay_ignore_content = options.replay_ignore_content self.refresh_server_playback = options.refresh_server_playback @@ -88,7 +92,6 @@ class DumpMaster(flow.FlowMaster): if options.stickyauth: self.set_stickyauth(options.stickyauth) - if options.wfile: path = os.path.expanduser(options.wfile) try: @@ -152,7 +155,7 @@ class DumpMaster(flow.FlowMaster): return flows def add_event(self, e, level="info"): - needed = dict(error=1, info=1, debug=2).get(level, 1) + needed = dict(error=0, info=1, debug=2).get(level, 1) if self.o.verbosity >= needed: print >> self.outfile, e self.outfile.flush() @@ -202,7 +205,7 @@ class DumpMaster(flow.FlowMaster): elif self.o.flow_detail >= 3: print >> self.outfile, str_request(f, self.showhost) print >> self.outfile, self.indent(4, f.request.headers) - if utils.isBin(f.request.content): + if f.request.content != http.CONTENT_MISSING and utils.isBin(f.request.content): d = netlib.utils.hexdump(f.request.content) d = "\n".join("%s\t%s %s"%i for i in d) print >> self.outfile, self.indent(4, d) diff --git a/libmproxy/filt.py b/libmproxy/filt.py index 7d2bd737..5d259096 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -343,7 +343,9 @@ bnf = _make() def parse(s): try: - return bnf.parseString(s, parseAll=True)[0] + filt = bnf.parseString(s, parseAll=True)[0] + filt.pattern = s + return filt except pp.ParseException: return None except ValueError: diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 440798bc..a6bf17d8 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -11,7 +11,7 @@ import netlib.http from . import controller, protocol, tnetstring, filt, script, version from .onboarding import app from .protocol import http, handle -from .proxy.config import parse_host_pattern +from .proxy.config import HostMatcher import urlparse ODict = odict.ODict @@ -27,7 +27,12 @@ class AppRegistry: Add a WSGI app to the registry, to be served for requests to the specified domain, on the specified port. """ - self.apps[(domain, port)] = wsgi.WSGIAdaptor(app, domain, port, version.NAMEVERSION) + self.apps[(domain, port)] = wsgi.WSGIAdaptor( + app, + domain, + port, + version.NAMEVERSION + ) def get(self, request): """ @@ -72,7 +77,8 @@ class ReplaceHooks: def get_specs(self): """ - Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) tuples. + Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) + tuples. """ return [i[:3] for i in self.lst] @@ -119,7 +125,8 @@ class SetHeaders: def get_specs(self): """ - Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) tuples. + Retrieve the hook specifcations. Returns a list of (fpatt, rex, s) + tuples. """ return [i[:3] for i in self.lst] @@ -162,6 +169,7 @@ class ClientPlaybackState: def __init__(self, flows, exit): self.flows, self.exit = flows, exit self.current = None + self.testing = False # Disables actual replay for testing. def count(self): return len(self.flows) @@ -179,18 +187,16 @@ class ClientPlaybackState: if flow is self.current: self.current = None - def tick(self, master, testing=False): - """ - testing: Disables actual replay for testing. - """ + def tick(self, master): if self.flows and not self.current: - n = self.flows.pop(0) - n.reply = controller.DummyReply() - self.current = master.handle_request(n) - if not testing and not self.current.response: - master.replay_request(self.current) # pragma: no cover - elif self.current.response: - master.handle_response(self.current) + self.current = self.flows.pop(0).copy() + if not self.testing: + master.replay_request(self.current) + else: + self.current.reply = controller.DummyReply() + master.handle_request(self.current) + if self.current.response: + master.handle_response(self.current) class ServerPlaybackState: @@ -219,9 +225,10 @@ class ServerPlaybackState: queriesArray = urlparse.parse_qsl(query) filtered = [] + ignore_params = self.ignore_params or [] for p in queriesArray: - if p[0] not in self.ignore_params: - filtered.append(p) + if p[0] not in ignore_params: + filtered.append(p) key = [ str(r.host), @@ -339,11 +346,13 @@ class State(object): # These are compiled filt expressions: self._limit = None self.intercept = None - self._limit_txt = None @property def limit_txt(self): - return self._limit_txt + if self._limit: + return self._limit.pattern + else: + return None def flow_count(self): return len(self._flow_list) @@ -362,6 +371,8 @@ class State(object): """ Add a request to the state. Returns the matching flow. """ + if flow in self._flow_list: # catch flow replay + return flow self._flow_list.append(flow) if flow.match(self._limit): self.view.append(flow) @@ -398,10 +409,8 @@ class State(object): if not f: return "Invalid filter expression." self._limit = f - self._limit_txt = txt else: self._limit = None - self._limit_txt = None self.recalculate_view() def set_intercept(self, txt): @@ -465,7 +474,7 @@ class FlowMaster(controller.Master): self.refresh_server_playback = False self.replacehooks = ReplaceHooks() self.setheaders = SetHeaders() - self.replay_ignore_params = False + self.replay_ignore_params = False self.replay_ignore_content = None @@ -515,11 +524,17 @@ class FlowMaster(controller.Master): for script in self.scripts: self.run_single_script_hook(script, name, *args, **kwargs) - def get_ignore(self): - return [i.pattern for i in self.server.config.ignore] + def get_ignore_filter(self): + return self.server.config.check_ignore.patterns + + def set_ignore_filter(self, host_patterns): + self.server.config.check_ignore = HostMatcher(host_patterns) - def set_ignore(self, ignore): - self.server.config.ignore = parse_host_pattern(ignore) + def get_tcp_filter(self): + return self.server.config.check_tcp.patterns + + def set_tcp_filter(self, host_patterns): + self.server.config.check_tcp = HostMatcher(host_patterns) def set_stickycookie(self, txt): if txt: @@ -601,7 +616,7 @@ class FlowMaster(controller.Master): ] if all(e): self.shutdown() - self.client_playback.tick(self, timeout) + self.client_playback.tick(self) return controller.Master.tick(self, q, timeout) @@ -612,6 +627,11 @@ class FlowMaster(controller.Master): """ Loads a flow, and returns a new flow object. """ + + if self.server and self.server.config.mode == "reverse": + f.request.host, f.request.port = self.server.config.mode.dst[2:] + f.request.scheme = "https" if self.server.config.mode.dst[1] else "http" + f.reply = controller.DummyReply() if f.request: self.handle_request(f) @@ -656,6 +676,8 @@ class FlowMaster(controller.Master): """ Returns None if successful, or error message if not. """ + if f.live: + return "Can't replay request which is still live..." if f.intercepting: return "Can't replay while intercepting..." if f.request.content == http.CONTENT_MISSING: @@ -705,7 +727,11 @@ class FlowMaster(controller.Master): if f.live: app = self.apps.get(f.request) if app: - err = app.serve(f, f.client_conn.wfile, **{"mitmproxy.master": self}) + err = app.serve( + f, + f.client_conn.wfile, + **{"mitmproxy.master": self} + ) if err: self.add_event("Error in wsgi app. %s"%err, "error") f.reply(protocol.KILL) @@ -720,8 +746,12 @@ class FlowMaster(controller.Master): def handle_responseheaders(self, f): self.run_script_hook("responseheaders", f) - if self.stream_large_bodies: - self.stream_large_bodies.run(f, False) + try: + if self.stream_large_bodies: + self.stream_large_bodies.run(f, False) + except netlib.http.HttpError: + f.reply(protocol.KILL) + return f.reply() return f @@ -755,7 +785,6 @@ class FlowMaster(controller.Master): self.stream = None - class FlowWriter: def __init__(self, fo): self.fo = fo @@ -787,7 +816,7 @@ class FlowReader: v = ".".join(str(i) for i in data["version"]) raise FlowReadError("Incompatible serialized data version: %s"%v) off = self.fo.tell() - yield handle.protocols[data["conntype"]]["flow"].from_state(data) + yield handle.protocols[data["type"]]["flow"].from_state(data) except ValueError, v: # Error is due to EOF if self.fo.tell() == off and self.fo.read() == '': diff --git a/libmproxy/main.py b/libmproxy/main.py index 2d6a0119..e5b7f56b 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -1,5 +1,4 @@ from __future__ import print_function, absolute_import -import argparse import os import signal import sys @@ -9,27 +8,43 @@ from .proxy import process_proxy_options, ProxyServerError from .proxy.server import DummyServer, ProxyServer +# This file is not included in coverage analysis or tests - anything that can be +# tested should live elsewhere. + def check_versions(): """ - Having installed a wrong version of pyOpenSSL or netlib is unfortunately a very common source of error. - Check before every start that both versions are somewhat okay. + Having installed a wrong version of pyOpenSSL or netlib is unfortunately a + very common source of error. Check before every start that both versions are + somewhat okay. """ - # We don't introduce backward-incompatible changes in patch versions. Only consider major and minor version. + # We don't introduce backward-incompatible changes in patch versions. Only + # consider major and minor version. if netlib.version.IVERSION[:2] != version.IVERSION[:2]: print( "Warning: You are using mitmdump %s with netlib %s. " - "Most likely, that doesn't work - please upgrade!" % (version.VERSION, netlib.version.VERSION), - file=sys.stderr) - import OpenSSL, inspect - + "Most likely, that won't work - please upgrade!" % ( + version.VERSION, netlib.version.VERSION + ), + file=sys.stderr + ) + import OpenSSL + import inspect v = tuple([int(x) for x in OpenSSL.__version__.split(".")][:2]) if v < (0, 14): - print("You are using an outdated version of pyOpenSSL: mitmproxy requires pyOpenSSL 0.14 or greater.", - file=sys.stderr) - # Some users apparently have multiple versions of pyOpenSSL installed. Report which one we got. + print( + "You are using an outdated version of pyOpenSSL:" + " mitmproxy requires pyOpenSSL 0.14 or greater.", + file=sys.stderr + ) + # Some users apparently have multiple versions of pyOpenSSL installed. + # Report which one we got. pyopenssl_path = os.path.dirname(inspect.getfile(OpenSSL)) - print("Your pyOpenSSL %s installation is located at %s" % (OpenSSL.__version__, pyopenssl_path), - file=sys.stderr) + print( + "Your pyOpenSSL %s installation is located at %s" % ( + OpenSSL.__version__, pyopenssl_path + ), + file=sys.stderr + ) sys.exit(1) @@ -38,8 +53,14 @@ def assert_utf8_env(): for i in ["LANG", "LC_CTYPE", "LC_ALL"]: spec += os.environ.get(i, "").lower() if "utf" not in spec: - print("Error: mitmproxy requires a UTF console environment.", file=sys.stderr) - print("Set your LANG enviroment variable to something like en_US.UTF-8", file=sys.stderr) + print( + "Error: mitmproxy requires a UTF console environment.", + file=sys.stderr + ) + print( + "Set your LANG enviroment variable to something like en_US.UTF-8", + file=sys.stderr + ) sys.exit(1) @@ -54,34 +75,13 @@ def get_server(dummy_server, options): sys.exit(1) -def mitmproxy_cmdline(): - # Don't import libmproxy.console for mitmdump, urwid is not available on all platforms. +def mitmproxy(): # pragma: nocover from . import console - from .console import palettes - - parser = argparse.ArgumentParser(usage="%(prog)s [options]") - parser.add_argument('--version', action='version', version=version.NAMEVERSION) - cmdline.common_options(parser) - parser.add_argument( - "--palette", type=str, default="dark", - action="store", dest="palette", - help="Select color palette: " + ", ".join(palettes.palettes.keys()) - ) - parser.add_argument( - "-e", - action="store_true", dest="eventlog", - help="Show event log." - ) - group = parser.add_argument_group( - "Filters", - "See help in mitmproxy for filter expression syntax." - ) - group.add_argument( - "-i", "--intercept", action="store", - type=str, dest="intercept", default=None, - help="Intercept filter expression." - ) + check_versions() + assert_utf8_env() + + parser = cmdline.mitmproxy() options = parser.parse_args() if options.quiet: options.verbose = 0 @@ -92,15 +92,6 @@ def mitmproxy_cmdline(): console_options.eventlog = options.eventlog console_options.intercept = options.intercept - return console_options, proxy_config - - -def mitmproxy(): # pragma: nocover - from . import console - - check_versions() - assert_utf8_env() - console_options, proxy_config = mitmproxy_cmdline() server = get_server(console_options.no_server, proxy_config) m = console.ConsoleMaster(server, console_options) @@ -110,24 +101,12 @@ def mitmproxy(): # pragma: nocover pass -def mitmdump_cmdline(): +def mitmdump(): # pragma: nocover from . import dump - parser = argparse.ArgumentParser(usage="%(prog)s [options] [filter]") - parser.add_argument('--version', action='version', version="mitmdump" + " " + version.VERSION) - cmdline.common_options(parser) - parser.add_argument( - "--keepserving", - action="store_true", dest="keepserving", default=False, - help="Continue serving after client playback or file read. We exit by default." - ) - parser.add_argument( - "-d", - action="count", dest="flow_detail", default=1, - help="Increase flow detail display level. Can be passed multiple times." - ) - parser.add_argument('args', nargs=argparse.REMAINDER) + check_versions() + parser = cmdline.mitmdump() options = parser.parse_args() if options.quiet: options.verbose = 0 @@ -139,14 +118,6 @@ def mitmdump_cmdline(): dump_options.keepserving = options.keepserving dump_options.filtstr = " ".join(options.args) if options.args else None - return dump_options, proxy_config - - -def mitmdump(): # pragma: nocover - from . import dump - - check_versions() - dump_options, proxy_config = mitmdump_cmdline() server = get_server(dump_options.no_server, proxy_config) try: @@ -164,44 +135,11 @@ def mitmdump(): # pragma: nocover pass -def mitmweb_cmdline(): +def mitmweb(): # pragma: nocover from . import web - parser = argparse.ArgumentParser(usage="%(prog)s [options]") - parser.add_argument( - '--version', - action='version', - version="mitmweb" + " " + version.VERSION - ) - - group = parser.add_argument_group("Mitmweb") - group.add_argument( - "--wport", - action="store", type=int, dest="wport", default=8081, - metavar="PORT", - help="Mitmweb port." - ) - group.add_argument( - "--wiface", - action="store", dest="wiface", default="127.0.0.1", - metavar="IFACE", - help="Mitmweb interface." - ) - group.add_argument( - "--wdebug", - action="store_true", dest="wdebug", - help="Turn on mitmweb debugging" - ) - - cmdline.common_options(parser) - group = parser.add_argument_group( - "Filters", - "See help in mitmproxy for filter expression syntax." - ) - group.add_argument( - "-i", "--intercept", action="store", - type=str, dest="intercept", default=None, - help="Intercept filter expression." - ) + + check_versions() + parser = cmdline.mitmweb() options = parser.parse_args() if options.quiet: @@ -213,14 +151,7 @@ def mitmweb_cmdline(): web_options.wdebug = options.wdebug web_options.wiface = options.wiface web_options.wport = options.wport - return web_options, proxy_config - -def mitmweb(): # pragma: nocover - from . import web - - check_versions() - web_options, proxy_config = mitmweb_cmdline() server = get_server(web_options.no_server, proxy_config) m = web.WebMaster(server, web_options) diff --git a/libmproxy/onboarding/app.py b/libmproxy/onboarding/app.py index 9b5db38a..4023fae2 100644 --- a/libmproxy/onboarding/app.py +++ b/libmproxy/onboarding/app.py @@ -18,12 +18,12 @@ def index(): @mapp.route("/cert/pem") def certs_pem(): - p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.pem") + p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.pem") return flask.Response(open(p, "rb").read(), mimetype='application/x-x509-ca-cert') @mapp.route("/cert/p12") def certs_p12(): - p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.p12") + p = os.path.join(master().server.config.cadir, config.CONF_BASENAME + "-ca-cert.p12") return flask.Response(open(p, "rb").read(), mimetype='application/x-pkcs12') diff --git a/libmproxy/onboarding/templates/index.html b/libmproxy/onboarding/templates/index.html index 50cfd5db..65fda5d2 100644 --- a/libmproxy/onboarding/templates/index.html +++ b/libmproxy/onboarding/templates/index.html @@ -1,5 +1,5 @@ {% extends "frame.html" %} -{% block body %} +{% block body %} <center> <h2> Click to install the mitmproxy certificate: </h2> @@ -23,4 +23,13 @@ </div> </div> +<hr/> +<div class="text-center"> + Other mitmproxy users cannot intercept your connection. +</div> +<div class="text-center text-muted"> + This page is served by your local mitmproxy instance. The certificate you are about to install has been uniquely generated on mitmproxy's first run and is not shared + between mitmproxy installations. +</div> + {% endblock %} diff --git a/libmproxy/platform/windows.py b/libmproxy/platform/windows.py index ddbbed52..066a377d 100644 --- a/libmproxy/platform/windows.py +++ b/libmproxy/platform/windows.py @@ -1,4 +1,4 @@ -import argparse +import configargparse import cPickle as pickle from ctypes import byref, windll, Structure from ctypes.wintypes import DWORD @@ -361,7 +361,7 @@ class TransparentProxy(object): if __name__ == "__main__": - parser = argparse.ArgumentParser(description="Windows Transparent Proxy") + parser = configargparse.ArgumentParser(description="Windows Transparent Proxy") parser.add_argument('--mode', choices=['forward', 'local', 'both'], default="both", help='redirection operation mode: "forward" to only redirect forwarded packets, ' '"local" to only redirect packets originating from the local machine') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index de5f9950..49f5e8c0 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -18,12 +18,17 @@ HDR_FORM_URLENCODED = "application/x-www-form-urlencoded" CONTENT_MISSING = 0 +class KillSignal(Exception): + pass + + def get_line(fp): """ Get a line, possibly preceded by a blank. """ line = fp.readline() - if line == "\r\n" or line == "\n": # Possible leftover from previous message + if line == "\r\n" or line == "\n": + # Possible leftover from previous message line = fp.readline() if line == "": raise tcp.NetLibDisconnect() @@ -237,25 +242,47 @@ class HTTPRequest(HTTPMessage): is content associated, but not present. CONTENT_MISSING evaluates to False to make checking for the presence of content natural. - form_in: The request form which mitmproxy has received. The following values are possible: - - relative (GET /index.html, OPTIONS *) (covers origin form and asterisk form) - - absolute (GET http://example.com:80/index.html) - - authority-form (CONNECT example.com:443) - Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 + form_in: The request form which mitmproxy has received. The following + values are possible: + + - relative (GET /index.html, OPTIONS *) (covers origin form and + asterisk form) + - absolute (GET http://example.com:80/index.html) + - authority-form (CONNECT example.com:443) + Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 - form_out: The request form which mitmproxy has send out to the destination + form_out: The request form which mitmproxy will send out to the + destination timestamp_start: Timestamp indicating when request transmission started timestamp_end: Timestamp indicating when request transmission ended """ - def __init__(self, form_in, method, scheme, host, port, path, httpversion, headers, - content, timestamp_start=None, timestamp_end=None, form_out=None): + def __init__( + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + content, + timestamp_start=None, + timestamp_end=None, + form_out=None + ): assert isinstance(headers, ODictCaseless) or not headers - HTTPMessage.__init__(self, httpversion, headers, content, timestamp_start, - timestamp_end) - + HTTPMessage.__init__( + self, + httpversion, + headers, + content, + timestamp_start, + timestamp_end + ) self.form_in = form_in self.method = method self.scheme = scheme @@ -308,30 +335,43 @@ class HTTPRequest(HTTPMessage): request_line = get_line(rfile) - if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start + if hasattr(rfile, "first_byte_timestamp"): + # more accurate timestamp_start timestamp_start = rfile.first_byte_timestamp request_line_parts = http.parse_init(request_line) if not request_line_parts: - raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + raise http.HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) method, path, httpversion = request_line_parts if path == '*' or path.startswith("/"): form_in = "relative" if not netlib.utils.isascii(path): - raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + raise http.HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) elif method.upper() == 'CONNECT': form_in = "authority" r = http.parse_init_connect(request_line) if not r: - raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + raise http.HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) host, port, _ = r path = None else: form_in = "absolute" r = http.parse_init_proxy(request_line) if not r: - raise http.HttpError(400, "Bad HTTP request line: %s" % repr(request_line)) + raise http.HttpError( + 400, + "Bad HTTP request line: %s" % repr(request_line) + ) _, scheme, host, port, path, _ = r headers = http.read_headers(rfile) @@ -343,50 +383,69 @@ class HTTPRequest(HTTPMessage): method, None, True) timestamp_end = utils.timestamp() - return HTTPRequest(form_in, method, scheme, host, port, path, httpversion, headers, - content, timestamp_start, timestamp_end) + return HTTPRequest( + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + content, + timestamp_start, + timestamp_end + ) def _assemble_first_line(self, form=None): form = form or self.form_out if form == "relative": - path = self.path if self.method != "OPTIONS" else "*" - request_line = '%s %s HTTP/%s.%s' % \ - (self.method, path, self.httpversion[0], self.httpversion[1]) + request_line = '%s %s HTTP/%s.%s' % ( + self.method, self.path, self.httpversion[0], self.httpversion[1] + ) elif form == "authority": - request_line = '%s %s:%s HTTP/%s.%s' % (self.method, self.host, self.port, - self.httpversion[0], self.httpversion[1]) + request_line = '%s %s:%s HTTP/%s.%s' % ( + self.method, self.host, self.port, self.httpversion[0], + self.httpversion[1] + ) elif form == "absolute": - request_line = '%s %s://%s:%s%s HTTP/%s.%s' % \ - (self.method, self.scheme, self.host, self.port, self.path, - self.httpversion[0], self.httpversion[1]) + request_line = '%s %s://%s:%s%s HTTP/%s.%s' % ( + self.method, self.scheme, self.host, + self.port, self.path, self.httpversion[0], + self.httpversion[1] + ) else: raise http.HttpError(400, "Invalid request form") return request_line + # This list is adopted legacy code. + # We probably don't need to strip off keep-alive. + _headers_to_strip_off = ['Proxy-Connection', + 'Keep-Alive', + 'Connection', + 'Transfer-Encoding', + 'Upgrade'] + def _assemble_headers(self): headers = self.headers.copy() - for k in ['Proxy-Connection', - 'Keep-Alive', - 'Connection', - 'Transfer-Encoding']: + for k in self._headers_to_strip_off: del headers[k] - if headers["Upgrade"] == ["h2c"]: # Suppress HTTP2 https://http2.github.io/http2-spec/index.html#discover-http - del headers["Upgrade"] - if not 'host' in headers and self.scheme and self.host and self.port: + if 'host' not in headers and self.scheme and self.host and self.port: headers["Host"] = [utils.hostport(self.scheme, self.host, self.port)] - if self.content: + # If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header. + if self.content or self.content == "": headers["Content-Length"] = [str(len(self.content))] - elif 'Transfer-Encoding' in self.headers: # content-length for e.g. chuncked transfer-encoding with no content - headers["Content-Length"] = ["0"] return str(headers) def _assemble_head(self, form=None): - return "%s\r\n%s\r\n" % (self._assemble_first_line(form), self._assemble_headers()) + return "%s\r\n%s\r\n" % ( + self._assemble_first_line(form), self._assemble_headers() + ) def assemble(self, form=None): """ @@ -396,7 +455,10 @@ class HTTPRequest(HTTPMessage): Raises an Exception if the request cannot be assembled. """ if self.content == CONTENT_MISSING: - raise proxy.ProxyError(502, "Cannot assemble flow with CONTENT_MISSING") + raise proxy.ProxyError( + 502, + "Cannot assemble flow with CONTENT_MISSING" + ) head = self._assemble_head(form) if self.content: return head + self.content @@ -644,7 +706,9 @@ class HTTPResponse(HTTPMessage): return "<HTTPResponse: {code} {msg} ({contenttype}, {size})>".format( code=self.code, msg=self.msg, - contenttype=self.headers.get_first("content-type", "unknown content type"), + contenttype=self.headers.get_first( + "content-type", "unknown content type" + ), size=size ) @@ -665,7 +729,8 @@ class HTTPResponse(HTTPMessage): body_size_limit, include_body=include_body) - if hasattr(rfile, "first_byte_timestamp"): # more accurate timestamp_start + if hasattr(rfile, "first_byte_timestamp"): + # more accurate timestamp_start timestamp_start = rfile.first_byte_timestamp if include_body: @@ -687,26 +752,30 @@ class HTTPResponse(HTTPMessage): return 'HTTP/%s.%s %s %s' % \ (self.httpversion[0], self.httpversion[1], self.code, self.msg) + _headers_to_strip_off = ['Proxy-Connection', + 'Alternate-Protocol', + 'Alt-Svc'] + def _assemble_headers(self, preserve_transfer_encoding=False): headers = self.headers.copy() - for k in ['Proxy-Connection', - 'Alternate-Protocol', - 'Alt-Svc']: + for k in self._headers_to_strip_off: del headers[k] if not preserve_transfer_encoding: del headers['Transfer-Encoding'] - if self.content: + # If content is defined (i.e. not None or CONTENT_MISSING), we always add a content-length header. + if self.content or self.content == "": headers["Content-Length"] = [str(len(self.content))] - # add content-length for chuncked transfer-encoding with no content - elif not preserve_transfer_encoding and 'Transfer-Encoding' in self.headers: - headers["Content-Length"] = ["0"] return str(headers) def _assemble_head(self, preserve_transfer_encoding=False): return '%s\r\n%s\r\n' % ( - self._assemble_first_line(), self._assemble_headers(preserve_transfer_encoding=preserve_transfer_encoding)) + self._assemble_first_line(), + self._assemble_headers( + preserve_transfer_encoding=preserve_transfer_encoding + ) + ) def assemble(self): """ @@ -716,7 +785,10 @@ class HTTPResponse(HTTPMessage): Raises an Exception if the request cannot be assembled. """ if self.content == CONTENT_MISSING: - raise proxy.ProxyError(502, "Cannot assemble flow with CONTENT_MISSING") + raise proxy.ProxyError( + 502, + "Cannot assemble flow with CONTENT_MISSING" + ) head = self._assemble_head() if self.content: return head + self.content @@ -783,8 +855,9 @@ class HTTPResponse(HTTPMessage): pairs = [pair.partition("=") for pair in header.split(';')] cookie_name = pairs[0][0] # the key of the first key/value pairs cookie_value = pairs[0][2] # the value of the first key/value pairs - cookie_parameters = {key.strip().lower(): value.strip() for key, sep, value in - pairs[1:]} + cookie_parameters = { + key.strip().lower(): value.strip() for key, sep, value in pairs[1:] + } cookies.append((cookie_name, (cookie_value, cookie_parameters))) return dict(cookies) @@ -817,7 +890,8 @@ class HTTPFlow(Flow): self.response = None """@type: HTTPResponse""" - self.intercepting = False # FIXME: Should that rather be an attribute of Flow? + # FIXME: Should that rather be an attribute of Flow? + self.intercepting = False _stateobject_attributes = Flow._stateobject_attributes.copy() _stateobject_attributes.update( @@ -905,7 +979,9 @@ class HTTPFlow(Flow): class HttpAuthenticationError(Exception): def __init__(self, auth_headers=None): - super(HttpAuthenticationError, self).__init__("Proxy Authentication Required") + super(HttpAuthenticationError, self).__init__( + "Proxy Authentication Required" + ) self.headers = auth_headers self.code = 407 @@ -937,16 +1013,23 @@ class HTTPHandler(ProtocolHandler): try: self.c.server_conn.send(request_raw) # Only get the headers at first... - flow.response = HTTPResponse.from_stream(self.c.server_conn.rfile, flow.request.method, - body_size_limit=self.c.config.body_size_limit, - include_body=False) + flow.response = HTTPResponse.from_stream( + self.c.server_conn.rfile, flow.request.method, + body_size_limit=self.c.config.body_size_limit, + include_body=False + ) break except (tcp.NetLibDisconnect, http.HttpErrorConnClosed), v: - self.c.log("error in server communication: %s" % repr(v), level="debug") + self.c.log( + "error in server communication: %s" % repr(v), + level="debug" + ) if attempt == 0: - # In any case, we try to reconnect at least once. - # This is necessary because it might be possible that we already initiated an upstream connection - # after clientconnect that has already been expired, e.g consider the following event log: + # In any case, we try to reconnect at least once. This is + # necessary because it might be possible that we already + # initiated an upstream connection after clientconnect that + # has already been expired, e.g consider the following event + # log: # > clientconnect (transparent mode destination known) # > serverconnect # > read n% of large request @@ -959,19 +1042,21 @@ class HTTPHandler(ProtocolHandler): # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True - self.c.channel.ask("responseheaders", flow) - - # now get the rest of the request body, if body still needs to be read - # but not streaming this response - if flow.response.stream: - flow.response.content = CONTENT_MISSING + flow = self.c.channel.ask("responseheaders", flow) + if flow is None or flow == KILL: + raise KillSignal() else: - flow.response.content = http.read_http_body( - self.c.server_conn.rfile, flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, flow.response.code, False - ) - flow.response.timestamp_end = utils.timestamp() + # now get the rest of the request body, if body still needs to be + # read but not streaming this response + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = http.read_http_body( + self.c.server_conn.rfile, flow.response.headers, + self.c.config.body_size_limit, + flow.request.method, flow.response.code, False + ) + flow.response.timestamp_end = utils.timestamp() def handle_flow(self): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live) @@ -1001,10 +1086,10 @@ class HTTPHandler(ProtocolHandler): # sent through to the Master. flow.request = req request_reply = self.c.channel.ask("request", flow) - self.process_server_address(flow) # The inline script may have changed request.host - if request_reply is None or request_reply == KILL: - return False + raise KillSignal() + + self.process_server_address(flow) # The inline script may have changed request.host if isinstance(request_reply, HTTPResponse): flow.response = request_reply @@ -1018,7 +1103,7 @@ class HTTPHandler(ProtocolHandler): self.c.log("response", "debug", [flow.response._assemble_first_line()]) response_reply = self.c.channel.ask("response", flow) if response_reply is None or response_reply == KILL: - return False + raise KillSignal() self.send_response_to_client(flow) @@ -1050,15 +1135,27 @@ class HTTPHandler(ProtocolHandler): flow.live.restore_server() return True # Next flow please. - except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: + except ( + HttpAuthenticationError, + http.HttpError, + proxy.ProxyError, + tcp.NetLibError, + ), e: self.handle_error(e, flow) + except KillSignal: + self.c.log("Connection killed", "info") finally: flow.live = None # Connection is not live anymore. return False def handle_server_reconnect(self, state): if state["state"] == "connect": - send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False) + send_connect_request( + self.c.server_conn, + state["host"], + state["port"], + update_state=False + ) else: # pragma: nocover raise RuntimeError("Unknown State: %s" % state["state"]) @@ -1079,14 +1176,14 @@ class HTTPHandler(ProtocolHandler): if message: self.c.log(message, level="info") if message_debug: - self.c.log(message, level="debug") + self.c.log(message_debug, level="debug") if flow: - # TODO: no flows without request or with both request and response at the moment. + # TODO: no flows without request or with both request and response + # at the moment. if flow.request and not flow.response: flow.error = Error(message or message_debug) self.c.channel.ask("error", flow) - try: code = getattr(error, "code", 502) headers = getattr(error, "headers", None) @@ -1100,12 +1197,22 @@ class HTTPHandler(ProtocolHandler): def send_error(self, code, message, headers): response = http_status.RESPONSES.get(code, "Unknown") - html_content = '<html><head>\n<title>%d %s</title>\n</head>\n<body>\n%s\n</body>\n</html>' % \ - (code, response, message) + html_content = """ + <html> + <head> + <title>%d %s</title> + </head> + <body>%s</body> + </html> + """ % (code, response, message) self.c.client_conn.wfile.write("HTTP/1.1 %s %s\r\n" % (code, response)) - self.c.client_conn.wfile.write("Server: %s\r\n" % self.c.config.server_version) + self.c.client_conn.wfile.write( + "Server: %s\r\n" % self.c.config.server_version + ) self.c.client_conn.wfile.write("Content-type: text/html\r\n") - self.c.client_conn.wfile.write("Content-Length: %d\r\n" % len(html_content)) + self.c.client_conn.wfile.write( + "Content-Length: %d\r\n" % len(html_content) + ) if headers: for key, value in headers.items(): self.c.client_conn.wfile.write("%s: %s\r\n" % (key, value)) @@ -1145,11 +1252,15 @@ class HTTPHandler(ProtocolHandler): # Now we can process the request. if request.form_in == "authority": if self.c.client_conn.ssl_established: - raise http.HttpError(400, "Must not CONNECT on already encrypted connection") + raise http.HttpError( + 400, + "Must not CONNECT on already encrypted connection" + ) if self.c.config.mode == "regular": self.c.set_server_address((request.host, request.port)) - flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow + # Update server_conn attribute on the flow + flow.server_conn = self.c.server_conn self.c.establish_server_connection() self.c.client_conn.send( 'HTTP/1.1 200 Connection established\r\n' + @@ -1161,7 +1272,9 @@ class HTTPHandler(ProtocolHandler): elif self.c.config.mode == "upstream": return None else: - pass # CONNECT should never occur if we don't expect absolute-form requests + # CONNECT should never occur if we don't expect absolute-form + # requests + pass elif request.form_in == self.expected_form_in: @@ -1169,61 +1282,77 @@ class HTTPHandler(ProtocolHandler): if request.form_in == "absolute": if request.scheme != "http": - raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme) + raise http.HttpError( + 400, + "Invalid request scheme: %s" % request.scheme + ) if self.c.config.mode == "regular": - # Update info so that an inline script sees the correct value at flow.server_conn + # Update info so that an inline script sees the correct + # value at flow.server_conn self.c.set_server_address((request.host, request.port)) flow.server_conn = self.c.server_conn return None - - raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" % - (self.expected_form_in, request.form_in)) + raise http.HttpError( + 400, "Invalid HTTP request form (expected: %s, got: %s)" % ( + self.expected_form_in, request.form_in + ) + ) def process_server_address(self, flow): # Depending on the proxy mode, server handling is entirely different - # We provide a mostly unified API to the user, which needs to be unfiddled here + # We provide a mostly unified API to the user, which needs to be + # unfiddled here # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 ) address = netlib.tcp.Address((flow.request.host, flow.request.port)) ssl = (flow.request.scheme == "https") if self.c.config.mode == "upstream": - - # The connection to the upstream proxy may have a state we may need to take into account. + # The connection to the upstream proxy may have a state we may need + # to take into account. connected_to = None for s in flow.server_conn.state: if s[0] == "http" and s[1]["state"] == "connect": connected_to = tcp.Address((s[1]["host"], s[1]["port"])) - # We need to reconnect if the current flow either requires a (possibly impossible) - # change to the connection state, e.g. the host has changed but we already CONNECTed somewhere else. + # We need to reconnect if the current flow either requires a + # (possibly impossible) change to the connection state, e.g. the + # host has changed but we already CONNECTed somewhere else. needs_server_change = ( ssl != self.c.server_conn.ssl_established or - (connected_to and address != connected_to) # HTTP proxying is "stateless", CONNECT isn't. + # HTTP proxying is "stateless", CONNECT isn't. + (connected_to and address != connected_to) ) if needs_server_change: # force create new connection to the proxy server to reset state self.live.change_server(self.c.server_conn.address, force=True) if ssl: - send_connect_request(self.c.server_conn, address.host, address.port) + send_connect_request( + self.c.server_conn, + address.host, + address.port + ) self.c.establish_ssl(server=True) else: - # If we're not in upstream mode, we just want to update the host and possibly establish TLS. - self.live.change_server(address, ssl=ssl) # this is a no op if the addresses match. + # If we're not in upstream mode, we just want to update the host and + # possibly establish TLS. This is a no op if the addresses match. + self.live.change_server(address, ssl=ssl) flow.server_conn = self.c.server_conn def send_response_to_client(self, flow): if not flow.response.stream: # no streaming: - # we already received the full response from the server and can send it to the client straight away. + # we already received the full response from the server and can send + # it to the client straight away. self.c.client_conn.send(flow.response.assemble()) else: # streaming: - # First send the headers and then transfer the response incrementally: + # First send the headers and then transfer the response + # incrementally: h = flow.response._assemble_head(preserve_transfer_encoding=True) self.c.client_conn.send(h) for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, @@ -1237,7 +1366,8 @@ class HTTPHandler(ProtocolHandler): def check_close_connection(self, flow): """ - Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so. + Checks if the connection should be closed depending on the HTTP + semantics. Returns True, if so. """ close_connection = ( http.connection_close(flow.request.httpversion, flow.request.headers) or @@ -1260,20 +1390,39 @@ class HTTPHandler(ProtocolHandler): Returns False, if the connection should be closed immediately. """ address = tcp.Address.wrap(address) - if self.c.check_ignore_address(address): + if self.c.config.check_ignore(address): self.c.log("Ignore host: %s:%s" % address(), "info") - TCPHandler(self.c).handle_messages() + TCPHandler(self.c, log=False).handle_messages() return False else: self.expected_form_in = "relative" self.expected_form_out = "relative" self.skip_authentication = True - if address.port in self.c.config.ssl_ports: + # In practice, nobody issues a CONNECT request to send unencrypted HTTP requests afterwards. + # If we don't delegate to TCP mode, we should always negotiate a SSL connection. + # + # FIXME: + # Turns out the previous statement isn't entirely true. Chrome on Windows CONNECTs to :80 + # if an explicit proxy is configured and a websocket connection should be established. + # We don't support websocket at the moment, so it fails anyway, but we should come up with + # a better solution to this if we start to support WebSockets. + should_establish_ssl = ( + address.port in self.c.config.ssl_ports + or + not self.c.config.check_tcp(address) + ) + + if should_establish_ssl: self.c.log("Received CONNECT request to SSL port. Upgrading to SSL...", "debug") self.c.establish_ssl(server=True, client=True) self.c.log("Upgrade to SSL completed.", "debug") + if self.c.config.check_tcp(address): + self.c.log("Generic TCP mode for host: %s:%s" % address(), "info") + TCPHandler(self.c).handle_messages() + return False + return True def authenticate(self, request): @@ -1297,31 +1446,43 @@ class RequestReplayThread(threading.Thread): r = self.flow.request form_out_backup = r.form_out try: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - server_address = self.config.mode.get_upstream_server(self.flow.client_conn)[2:] - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - send_connect_request(server, r.host, r.port) - server.establish_ssl(self.config.clientcerts, sni=r.host) - r.form_out = "relative" - else: - r.form_out = "absolute" + self.flow.response = None + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == KILL: + raise KillSignal() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - server.establish_ssl(self.config.clientcerts, sni=r.host) - r.form_out = "relative" - - server.send(r.assemble()) - self.flow.response = HTTPResponse.from_stream(server.rfile, r.method, - body_size_limit=self.config.body_size_limit) - self.channel.ask("response", self.flow) - except (proxy.ProxyError, http.HttpError, tcp.NetLibError), v: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + server_address = self.config.mode.get_upstream_server(self.flow.client_conn)[2:] + server = ServerConnection(server_address) + server.connect() + if r.scheme == "https": + send_connect_request(server, r.host, r.port) + server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + if r.scheme == "https": + server.establish_ssl(self.config.clientcerts, sni=self.flow.server_conn.sni) + r.form_out = "relative" + + server.send(r.assemble()) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_stream(server.rfile, r.method, + body_size_limit=self.config.body_size_limit) + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == KILL: + raise KillSignal() + except (proxy.ProxyError, http.HttpError, tcp.NetLibError) as v: self.flow.error = Error(repr(v)) self.channel.ask("error", self.flow) + except KillSignal: + self.channel.tell("log", proxy.Log("Connection killed", "info")) finally: r.form_out = form_out_backup diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py index 519693db..3be1cc45 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/protocol/primitives.py @@ -59,8 +59,8 @@ class Flow(stateobject.StateObject): A Flow is a collection of objects representing a single transaction. This class is usually subclassed for each protocol, e.g. HTTPFlow. """ - def __init__(self, conntype, client_conn, server_conn, live=None): - self.conntype = conntype + def __init__(self, type, client_conn, server_conn, live=None): + self.type = type self.id = str(uuid.uuid4()) self.client_conn = client_conn """@type: ClientConnection""" @@ -78,7 +78,7 @@ class Flow(stateobject.StateObject): error=Error, client_conn=ClientConnection, server_conn=ServerConnection, - conntype=str + type=str ) def get_state(self, short=False): @@ -174,7 +174,7 @@ class LiveConnection(object): self._backup_server_conn = None """@type: libmproxy.proxy.connection.ServerConnection""" - def change_server(self, address, ssl=None, force=False, persistent_change=False): + def change_server(self, address, ssl=None, sni=None, force=False, persistent_change=False): """ Change the server connection to the specified address. @returns: @@ -183,7 +183,14 @@ class LiveConnection(object): """ address = netlib.tcp.Address.wrap(address) - ssl_mismatch = (ssl is not None and ssl != self.c.server_conn.ssl_established) + ssl_mismatch = ( + ssl is not None and + ( + ssl != self.c.server_conn.ssl_established + or + (sni is not None and sni != self.c.sni) + ) + ) address_mismatch = (address != self.c.server_conn.address) if persistent_change: @@ -212,6 +219,8 @@ class LiveConnection(object): self.c.set_server_address(address) self.c.establish_server_connection(ask=False) + if sni: + self.c.sni = sni if ssl: self.c.establish_ssl(server=True) return True diff --git a/libmproxy/protocol/tcp.py b/libmproxy/protocol/tcp.py index a56bf07b..da0c9087 100644 --- a/libmproxy/protocol/tcp.py +++ b/libmproxy/protocol/tcp.py @@ -13,6 +13,10 @@ class TCPHandler(ProtocolHandler): chunk_size = 4096 + def __init__(self, c, log=True): + super(TCPHandler, self).__init__(c) + self.log = log + def handle_messages(self): self.c.establish_server_connection() @@ -63,26 +67,25 @@ class TCPHandler(ProtocolHandler): # if one of the peers is over SSL, we need to send # bytes/strings if not src.ssl_established: - # only ssl to dst, i.e. we revc'd into buf but need - # bytes/string now. + # we revc'd into buf but need bytes/string now. contents = buf[:size].tobytes() - self.c.log( - "%s %s\r\n%s" % ( - direction, dst_str, cleanBin(contents) - ), - "debug" - ) + if self.log: + self.c.log( + "%s %s\r\n%s" % ( + direction, dst_str, cleanBin(contents) + ), + "info" + ) dst.connection.send(contents) else: # socket.socket.send supports raw bytearrays/memoryviews - self.c.log( - "%s %s\r\n%s" % ( - direction, - dst_str, - cleanBin(buf.tobytes()) - ), - "debug" - ) + if self.log: + self.c.log( + "%s %s\r\n%s" % ( + direction, dst_str, cleanBin(buf.tobytes()) + ), + "info" + ) dst.connection.send(buf[:size]) except socket.error as e: self.c.log("TCP connection closed unexpectedly.", "debug") diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 62104a24..3d373a28 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,26 +1,54 @@ from __future__ import absolute_import import os import re -from netlib import http_auth, certutils +from netlib import http_auth, certutils, tcp from .. import utils, platform, version -from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode +from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode TRANSPARENT_SSL_PORTS = [443, 8443] CONF_BASENAME = "mitmproxy" -CONF_DIR = "~/.mitmproxy" +CA_DIR = "~/.mitmproxy" -def parse_host_pattern(patterns): - return [re.compile(p, re.IGNORECASE) for p in patterns] +class HostMatcher(object): + def __init__(self, patterns=[]): + self.patterns = list(patterns) + self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] + + def __call__(self, address): + address = tcp.Address.wrap(address) + host = "%s:%s" % (address.host, address.port) + if any(rex.search(host) for rex in self.regexes): + return True + else: + return False + + def __nonzero__(self): + return bool(self.patterns) class ProxyConfig: - def __init__(self, host='', port=8080, server_version=version.NAMEVERSION, - confdir=CONF_DIR, ca_file=None, clientcerts=None, - no_upstream_cert=False, body_size_limit=None, - mode=None, upstream_server=None, http_form_in=None, http_form_out=None, - authenticator=None, ignore=[], - ciphers=None, certs=[], certforward=False, ssl_ports=TRANSPARENT_SSL_PORTS): + def __init__( + self, + host='', + port=8080, + server_version=version.NAMEVERSION, + cadir=CA_DIR, + clientcerts=None, + no_upstream_cert=False, + body_size_limit=None, + mode=None, + upstream_server=None, + http_form_in=None, + http_form_out=None, + authenticator=None, + ignore_hosts=[], + tcp_hosts=[], + ciphers=None, + certs=[], + certforward=False, + ssl_ports=TRANSPARENT_SSL_PORTS + ): self.host = host self.port = port self.server_version = server_version @@ -30,7 +58,9 @@ class ProxyConfig: self.body_size_limit = body_size_limit if mode == "transparent": - self.mode = TransparentProxyMode(platform.resolver(), TRANSPARENT_SSL_PORTS) + self.mode = TransparentProxyMode(platform.resolver(), ssl_ports) + elif mode == "socks5": + self.mode = Socks5ProxyMode(ssl_ports) elif mode == "reverse": self.mode = ReverseProxyMode(upstream_server) elif mode == "upstream": @@ -42,11 +72,11 @@ class ProxyConfig: self.mode.http_form_in = http_form_in or self.mode.http_form_in self.mode.http_form_out = http_form_out or self.mode.http_form_out - self.ignore = parse_host_pattern(ignore) + self.check_ignore = HostMatcher(ignore_hosts) + self.check_tcp = HostMatcher(tcp_hosts) self.authenticator = authenticator - self.confdir = os.path.expanduser(confdir) - self.ca_file = ca_file or os.path.join(self.confdir, CONF_BASENAME + "-ca.pem") - self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) + self.cadir = os.path.expanduser(cadir) + self.certstore = certutils.CertStore.from_store(self.cadir, CONF_BASENAME) for spec, cert in certs: self.certstore.add_cert_file(spec, cert) self.certforward = certforward @@ -63,6 +93,9 @@ def process_proxy_options(parser, options): if not platform.resolver: return parser.error("Transparent mode not supported on this platform.") mode = "transparent" + if options.socks_proxy: + c += 1 + mode = "socks5" if options.reverse_proxy: c += 1 mode = "reverse" @@ -72,7 +105,7 @@ def process_proxy_options(parser, options): mode = "upstream" upstream_server = options.upstream_proxy if c > 1: - return parser.error("Transparent mode, reverse mode and upstream proxy mode " + return parser.error("Transparent, SOCKS5, reverse and upstream proxy mode " "are mutually exclusive.") if options.clientcerts: @@ -109,10 +142,16 @@ def process_proxy_options(parser, options): parser.error("Certificate file does not exist: %s" % parts[1]) certs.append(parts) + ssl_ports = options.ssl_ports + if options.ssl_ports != TRANSPARENT_SSL_PORTS: + # arparse appends to default value by default, strip that off. + # see http://bugs.python.org/issue16399 + ssl_ports = ssl_ports[len(TRANSPARENT_SSL_PORTS):] + return ProxyConfig( host=options.addr, port=options.port, - confdir=options.confdir, + cadir=options.cadir, clientcerts=options.clientcerts, no_upstream_cert=options.no_upstream_cert, body_size_limit=body_size_limit, @@ -120,11 +159,13 @@ def process_proxy_options(parser, options): upstream_server=upstream_server, http_form_in=options.http_form_in, http_form_out=options.http_form_out, - ignore=options.ignore, + ignore_hosts=options.ignore_hosts, + tcp_hosts=options.tcp_hosts, authenticator=authenticator, ciphers=options.ciphers, certs=certs, certforward=options.certforward, + ssl_ports=ssl_ports ) @@ -133,10 +174,12 @@ def ssl_option_group(parser): group.add_argument( "--cert", dest='certs', default=[], type=str, metavar="SPEC", action="append", - help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' \ - 'The domain may include a wildcard, and is equal to "*" if not specified. ' \ - 'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' \ - 'it is used, else the default key in the conf dir is used. Can be passed multiple times.' + help='Add an SSL certificate. SPEC is of the form "[domain=]path". ' + 'The domain may include a wildcard, and is equal to "*" if not specified. ' + 'The file at path is a certificate in PEM format. If a private key is included in the PEM, ' + 'it is used, else the default key in the conf dir is used. ' + 'The PEM file should contain the full certificate chain, with the leaf certificate as the first entry. ' + 'Can be passed multiple times.' ) group.add_argument( "--client-certs", action="store", @@ -159,7 +202,7 @@ def ssl_option_group(parser): help="Don't connect to upstream server to look up certificate details." ) group.add_argument( - "--ssl-port", action="append", type=int, dest="ssl_ports", default=TRANSPARENT_SSL_PORTS, + "--ssl-port", action="append", type=int, dest="ssl_ports", default=list(TRANSPARENT_SSL_PORTS), metavar="PORT", help="Can be passed multiple times. Specify destination ports which are assumed to be SSL. " "Defaults to %s." % str(TRANSPARENT_SSL_PORTS) diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index 23d089d3..c0ae424d 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -1,5 +1,5 @@ from __future__ import absolute_import - +from netlib import socks class ProxyError(Exception): def __init__(self, code, message, headers=None): @@ -15,7 +15,7 @@ class ProxyMode(object): http_form_in = None http_form_out = None - def get_upstream_server(self, conn): + def get_upstream_server(self, client_conn): """ Returns the address of the server to connect to. Returns None if the address needs to be determined on the protocol level (regular proxy mode) @@ -46,7 +46,7 @@ class RegularProxyMode(ProxyMode): http_form_in = "absolute" http_form_out = "relative" - def get_upstream_server(self, conn): + def get_upstream_server(self, client_conn): return None @@ -58,9 +58,9 @@ class TransparentProxyMode(ProxyMode): self.resolver = resolver self.sslports = sslports - def get_upstream_server(self, conn): + def get_upstream_server(self, client_conn): try: - dst = self.resolver.original_addr(conn) + dst = self.resolver.original_addr(client_conn.connection) except Exception, e: raise ProxyError(502, "Transparent mode failure: %s" % str(e)) @@ -71,11 +71,80 @@ class TransparentProxyMode(ProxyMode): return [ssl, ssl] + list(dst) +class Socks5ProxyMode(ProxyMode): + http_form_in = "relative" + http_form_out = "relative" + + def __init__(self, sslports): + self.sslports = sslports + + @staticmethod + def _assert_socks5(msg): + if msg.ver != socks.VERSION.SOCKS5: + if msg.ver == ord("G") and len(msg.methods) == ord("E"): + guess = "Probably not a SOCKS request but a regular HTTP request. " + else: + guess = "" + raise socks.SocksError( + socks.REP.GENERAL_SOCKS_SERVER_FAILURE, + guess + "Invalid SOCKS version. Expected 0x05, got 0x%x" % msg.ver) + + def get_upstream_server(self, client_conn): + try: + # Parse Client Greeting + client_greet = socks.ClientGreeting.from_file(client_conn.rfile) + self._assert_socks5(client_greet) + if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: + raise socks.SocksError( + socks.METHOD.NO_ACCEPTABLE_METHODS, + "mitmproxy only supports SOCKS without authentication" + ) + + # Send Server Greeting + server_greet = socks.ServerGreeting( + socks.VERSION.SOCKS5, + socks.METHOD.NO_AUTHENTICATION_REQUIRED + ) + server_greet.to_file(client_conn.wfile) + client_conn.wfile.flush() + + # Parse Connect Request + connect_request = socks.Message.from_file(client_conn.rfile) + self._assert_socks5(connect_request) + if connect_request.msg != socks.CMD.CONNECT: + raise socks.SocksError( + socks.REP.COMMAND_NOT_SUPPORTED, + "mitmproxy only supports SOCKS5 CONNECT." + ) + + # We do not connect here yet, as the clientconnect event has not been handled yet. + + connect_reply = socks.Message( + socks.VERSION.SOCKS5, + socks.REP.SUCCEEDED, + socks.ATYP.DOMAINNAME, + client_conn.address # dummy value, we don't have an upstream connection yet. + ) + connect_reply.to_file(client_conn.wfile) + client_conn.wfile.flush() + + ssl = bool(connect_request.addr.port in self.sslports) + return ssl, ssl, connect_request.addr.host, connect_request.addr.port + + except socks.SocksError as e: + msg = socks.Message(5, e.code, socks.ATYP.DOMAINNAME, repr(e)) + try: + msg.to_file(client_conn.wfile) + except: + pass + raise ProxyError(502, "SOCKS5 mode failure: %s" % str(e)) + + class _ConstDestinationProxyMode(ProxyMode): def __init__(self, dst): self.dst = dst - def get_upstream_server(self, conn): + def get_upstream_server(self, client_conn): return self.dst diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 307a4bcd..55e2b30e 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -70,13 +70,15 @@ class ConnectionHandler: # Can we already identify the target server and connect to it? client_ssl, server_ssl = False, False - upstream_info = self.config.mode.get_upstream_server(self.client_conn.connection) + conn_kwargs = dict() + upstream_info = self.config.mode.get_upstream_server(self.client_conn) if upstream_info: self.set_server_address(upstream_info[2:]) client_ssl, server_ssl = upstream_info[:2] - if self.check_ignore_address(self.server_conn.address): + if self.config.check_ignore(self.server_conn.address): self.log("Ignore host: %s:%s" % self.server_conn.address(), "info") self.conntype = "tcp" + conn_kwargs["log"] = False client_ssl, server_ssl = False, False else: pass # No upstream info from the metadata: upstream info in the protocol (e.g. HTTP absolute-form) @@ -90,15 +92,18 @@ class ConnectionHandler: if client_ssl or server_ssl: self.establish_ssl(client=client_ssl, server=server_ssl) + if self.config.check_tcp(self.server_conn.address): + self.log("Generic TCP mode for host: %s:%s" % self.server_conn.address(), "info") + self.conntype = "tcp" + # Delegate handling to the protocol handler - protocol_handler(self.conntype)(self).handle_messages() + protocol_handler(self.conntype)(self, **conn_kwargs).handle_messages() - self.del_server_connection() self.log("clientdisconnect", "info") self.channel.tell("clientdisconnect", self) except ProxyError as e: - protocol_handler(self.conntype)(self).handle_error(e) + protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e) except Exception: import traceback, sys @@ -106,6 +111,10 @@ class ConnectionHandler: print >> sys.stderr, traceback.format_exc() print >> sys.stderr, "mitmproxy has crashed!" print >> sys.stderr, "Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy" + finally: + # Make sure that we close the server connection in any case. + # The client connection is closed by the ProxyServer and does not have be handled here. + self.del_server_connection() def del_server_connection(self): """ @@ -113,20 +122,13 @@ class ConnectionHandler: """ if self.server_conn and self.server_conn.connection: self.server_conn.finish() + self.server_conn.close() self.log("serverdisconnect", "debug", ["%s:%s" % (self.server_conn.address.host, self.server_conn.address.port)]) self.channel.tell("serverdisconnect", self) self.server_conn = None self.sni = None - def check_ignore_address(self, address): - address = tcp.Address.wrap(address) - host = "%s:%s" % (address.host, address.port) - if host and any(rex.search(host) for rex in self.config.ignore): - return True - else: - return False - def set_server_address(self, address): """ Sets a new server address with the given priority. @@ -190,14 +192,14 @@ class ConnectionHandler: if client: if self.client_conn.ssl_established: raise ProxyError(502, "SSL to Client already established.") - cert, key = self.find_cert() + cert, key, chain_file = self.find_cert() try: self.client_conn.convert_to_ssl( cert, key, handle_sni=self.handle_sni, cipher_list=self.config.ciphers, dhparams=self.config.certstore.dhparams, - ca_file=self.config.ca_file + chain_file=chain_file ) except tcp.NetLibError as v: raise ProxyError(400, repr(v)) @@ -234,7 +236,7 @@ class ConnectionHandler: def find_cert(self): if self.config.certforward and self.server_conn.ssl_established: - return self.server_conn.cert, self.config.certstore.gen_pkey(self.server_conn.cert) + return self.server_conn.cert, self.config.certstore.gen_pkey(self.server_conn.cert), None else: host = self.server_conn.address.host sans = [] @@ -264,17 +266,17 @@ class ConnectionHandler: self.log("SNI received: %s" % self.sni, "debug") self.server_reconnect() # reconnect to upstream server with SNI # Now, change client context to reflect changed certificate: - cert, key = self.find_cert() + cert, key, chain_file = self.find_cert() new_context = self.client_conn._create_ssl_context( cert, key, method=SSL.TLSv1_METHOD, cipher_list=self.config.ciphers, dhparams=self.config.certstore.dhparams, - ca_file=self.config.ca_file + chain_file=chain_file ) connection.set_context(new_context) # An unhandled exception in this method will core dump PyOpenSSL, so # make dang sure it doesn't happen. - except Exception: # pragma: no cover + except: # pragma: no cover import traceback - self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error")
\ No newline at end of file + self.log("Error in handle_sni:\r\n" + traceback.format_exc(), "error") diff --git a/libmproxy/version.py b/libmproxy/version.py index 3483625d..8dcaecc8 100644 --- a/libmproxy/version.py +++ b/libmproxy/version.py @@ -1,5 +1,9 @@ -IVERSION = (0, 11) +IVERSION = (0, 11, 1) VERSION = ".".join(str(i) for i in IVERSION) MINORVERSION = ".".join(str(i) for i in IVERSION[:2]) NAME = "mitmproxy" NAMEVERSION = NAME + " " + VERSION + +NEXT_MINORVERSION = list(IVERSION) +NEXT_MINORVERSION[1] += 1 +NEXT_MINORVERSION = ".".join(str(i) for i in NEXT_MINORVERSION[:2])
\ No newline at end of file diff --git a/libmproxy/web/static/flows.json b/libmproxy/web/static/flows.json index a0358db0..35accd38 100644 --- a/libmproxy/web/static/flows.json +++ b/libmproxy/web/static/flows.json @@ -93,7 +93,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -259,7 +259,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -425,7 +425,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -595,7 +595,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -765,7 +765,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -919,7 +919,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1057,7 +1057,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1195,7 +1195,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1329,7 +1329,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1483,7 +1483,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1633,7 +1633,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1767,7 +1767,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1901,7 +1901,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -2027,7 +2027,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 |