diff options
Diffstat (limited to 'libmproxy')
| -rw-r--r-- | libmproxy/cmdline.py | 9 | ||||
| -rw-r--r-- | libmproxy/console/__init__.py | 30 | ||||
| -rw-r--r-- | libmproxy/console/grideditor.py | 4 | ||||
| -rw-r--r-- | libmproxy/console/help.py | 1 | ||||
| -rw-r--r-- | libmproxy/flow.py | 18 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 9 | ||||
| -rw-r--r-- | libmproxy/protocol/primitives.py | 6 | ||||
| -rw-r--r-- | libmproxy/protocol/tcp.py | 35 | ||||
| -rw-r--r-- | libmproxy/proxy/config.py | 27 | ||||
| -rw-r--r-- | libmproxy/proxy/server.py | 20 | ||||
| -rw-r--r-- | libmproxy/web/static/flows.json | 28 |
11 files changed, 119 insertions, 68 deletions
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index c0eb57c9..83eab7ee 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -263,7 +263,7 @@ def common_options(parser): ) group.add_argument( "-I", "--ignore", - action="append", type=str, dest="ignore", default=[], + action="append", type=str, dest="ignore_hosts", default=[], metavar="HOST", help="Ignore host and forward all traffic without processing it. " "In transparent mode, it is recommended to use an IP address (range), not the hostname. " @@ -272,6 +272,13 @@ def common_options(parser): "Can be passed multiple times. " ) group.add_argument( + "--tcp", + action="append", type=str, dest="tcp_hosts", default=[], + metavar="HOST", + help="Generic TCP SSL proxy mode for all hosts that match the pattern. Similar to --ignore," + "but SSL connections are intercepted. The communication contents are printed to the event log in verbose mode." + ) + group.add_argument( "-n", action="store_true", dest="no_server", help="Don't start a proxy server." diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 9c4b4827..cb6a977f 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -129,10 +129,14 @@ class StatusBar(common.WWrap): r.append(":%s in file]"%self.master.server_playback.count()) else: r.append(":%s to go]"%self.master.server_playback.count()) - if self.master.get_ignore(): + if self.master.get_ignore_filter(): r.append("[") r.append(("heading_key", "I")) - r.append("gnore:%d]"%len(self.master.get_ignore())) + r.append("gnore:%d]" % len(self.master.get_ignore_filter())) + if self.master.get_tcp_filter(): + r.append("[") + r.append(("heading_key", "T")) + r.append("CP:%d]" % len(self.master.get_tcp_filter())) if self.master.state.intercept_txt: r.append("[") r.append(("heading_key", "i")) @@ -798,9 +802,13 @@ class ConsoleMaster(flow.FlowMaster): for command in commands: self.load_script(command) - def edit_ignore(self, ignore): + def edit_ignore_filter(self, ignore): patterns = (x[0] for x in ignore) - self.set_ignore(patterns) + self.set_ignore_filter(patterns) + + def edit_tcp_filter(self, tcp): + patterns = (x[0] for x in tcp) + self.set_tcp_filter(patterns) def loop(self): changed = True @@ -860,10 +868,18 @@ class ConsoleMaster(flow.FlowMaster): ) elif k == "I": self.view_grideditor( - grideditor.IgnoreEditor( + grideditor.HostPatternEditor( + self, + [[x] for x in self.get_ignore_filter()], + self.edit_ignore_filter + ) + ) + elif k == "T": + self.view_grideditor( + grideditor.HostPatternEditor( self, - [[x] for x in self.get_ignore()], - self.edit_ignore + [[x] for x in self.get_tcp_filter()], + self.edit_tcp_filter ) ) elif k == "i": diff --git a/libmproxy/console/grideditor.py b/libmproxy/console/grideditor.py index d629ec82..1673d536 100644 --- a/libmproxy/console/grideditor.py +++ b/libmproxy/console/grideditor.py @@ -495,8 +495,8 @@ class ScriptEditor(GridEditor): return str(v) -class IgnoreEditor(GridEditor): - title = "Editing ignore patterns" +class HostPatternEditor(GridEditor): + title = "Editing host patterns" columns = 1 headings = ("Regex (matched on hostname:port / ip:port)",) diff --git a/libmproxy/console/help.py b/libmproxy/console/help.py index bdcf3fd9..27288a36 100644 --- a/libmproxy/console/help.py +++ b/libmproxy/console/help.py @@ -119,6 +119,7 @@ class HelpView(urwid.ListBox): ("s", "add/remove scripts"), ("S", "server replay"), ("t", "set sticky cookie expression"), + ("T", "set tcp proxying pattern"), ("u", "set sticky auth expression"), ] text.extend(common.format_keyvals(keys, key="key", val="text", indent=4)) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 440798bc..5c3a0c7e 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -11,7 +11,7 @@ import netlib.http from . import controller, protocol, tnetstring, filt, script, version from .onboarding import app from .protocol import http, handle -from .proxy.config import parse_host_pattern +from .proxy.config import HostMatcher import urlparse ODict = odict.ODict @@ -515,11 +515,17 @@ class FlowMaster(controller.Master): for script in self.scripts: self.run_single_script_hook(script, name, *args, **kwargs) - def get_ignore(self): - return [i.pattern for i in self.server.config.ignore] + def get_ignore_filter(self): + return self.server.config.check_ignore.patterns - def set_ignore(self, ignore): - self.server.config.ignore = parse_host_pattern(ignore) + def set_ignore_filter(self, host_patterns): + self.server.config.check_ignore = HostMatcher(host_patterns) + + def get_tcp_filter(self): + return self.server.config.check_tcp.patterns + + def set_tcp_filter(self, host_patterns): + self.server.config.check_tcp = HostMatcher(host_patterns) def set_stickycookie(self, txt): if txt: @@ -787,7 +793,7 @@ class FlowReader: v = ".".join(str(i) for i in data["version"]) raise FlowReadError("Incompatible serialized data version: %s"%v) off = self.fo.tell() - yield handle.protocols[data["conntype"]]["flow"].from_state(data) + yield handle.protocols[data["type"]]["flow"].from_state(data) except ValueError, v: # Error is due to EOF if self.fo.tell() == off and self.fo.read() == '': diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 32a88b4b..33d860ca 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1260,9 +1260,9 @@ class HTTPHandler(ProtocolHandler): Returns False, if the connection should be closed immediately. """ address = tcp.Address.wrap(address) - if self.c.check_ignore_address(address): + if self.c.config.check_ignore(address): self.c.log("Ignore host: %s:%s" % address(), "info") - TCPHandler(self.c).handle_messages() + TCPHandler(self.c, log=False).handle_messages() return False else: self.expected_form_in = "relative" @@ -1274,6 +1274,11 @@ class HTTPHandler(ProtocolHandler): self.c.establish_ssl(server=True, client=True) self.c.log("Upgrade to SSL completed.", "debug") + if self.c.config.check_tcp(address): + self.c.log("Generic TCP mode for host: %s:%s" % address(), "info") + TCPHandler(self.c).handle_messages() + return False + return True def authenticate(self, request): diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py index 519693db..1bf7f832 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/protocol/primitives.py @@ -59,8 +59,8 @@ class Flow(stateobject.StateObject): A Flow is a collection of objects representing a single transaction. This class is usually subclassed for each protocol, e.g. HTTPFlow. """ - def __init__(self, conntype, client_conn, server_conn, live=None): - self.conntype = conntype + def __init__(self, type, client_conn, server_conn, live=None): + self.type = type self.id = str(uuid.uuid4()) self.client_conn = client_conn """@type: ClientConnection""" @@ -78,7 +78,7 @@ class Flow(stateobject.StateObject): error=Error, client_conn=ClientConnection, server_conn=ServerConnection, - conntype=str + type=str ) def get_state(self, short=False): diff --git a/libmproxy/protocol/tcp.py b/libmproxy/protocol/tcp.py index a56bf07b..da0c9087 100644 --- a/libmproxy/protocol/tcp.py +++ b/libmproxy/protocol/tcp.py @@ -13,6 +13,10 @@ class TCPHandler(ProtocolHandler): chunk_size = 4096 + def __init__(self, c, log=True): + super(TCPHandler, self).__init__(c) + self.log = log + def handle_messages(self): self.c.establish_server_connection() @@ -63,26 +67,25 @@ class TCPHandler(ProtocolHandler): # if one of the peers is over SSL, we need to send # bytes/strings if not src.ssl_established: - # only ssl to dst, i.e. we revc'd into buf but need - # bytes/string now. + # we revc'd into buf but need bytes/string now. contents = buf[:size].tobytes() - self.c.log( - "%s %s\r\n%s" % ( - direction, dst_str, cleanBin(contents) - ), - "debug" - ) + if self.log: + self.c.log( + "%s %s\r\n%s" % ( + direction, dst_str, cleanBin(contents) + ), + "info" + ) dst.connection.send(contents) else: # socket.socket.send supports raw bytearrays/memoryviews - self.c.log( - "%s %s\r\n%s" % ( - direction, - dst_str, - cleanBin(buf.tobytes()) - ), - "debug" - ) + if self.log: + self.c.log( + "%s %s\r\n%s" % ( + direction, dst_str, cleanBin(buf.tobytes()) + ), + "info" + ) dst.connection.send(buf[:size]) except socket.error as e: self.c.log("TCP connection closed unexpectedly.", "debug") diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index abdb7c41..948decc1 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import os import re -from netlib import http_auth, certutils +from netlib import http_auth, certutils, tcp from .. import utils, platform, version from .primitives import RegularProxyMode, TransparentProxyMode, UpstreamProxyMode, ReverseProxyMode, Socks5ProxyMode @@ -10,8 +10,21 @@ CONF_BASENAME = "mitmproxy" CONF_DIR = "~/.mitmproxy" -def parse_host_pattern(patterns): - return [re.compile(p, re.IGNORECASE) for p in patterns] +class HostMatcher(object): + def __init__(self, patterns=[]): + self.patterns = list(patterns) + self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns] + + def __call__(self, address): + address = tcp.Address.wrap(address) + host = "%s:%s" % (address.host, address.port) + if any(rex.search(host) for rex in self.regexes): + return True + else: + return False + + def __nonzero__(self): + return bool(self.patterns) class ProxyConfig: @@ -19,7 +32,7 @@ class ProxyConfig: confdir=CONF_DIR, clientcerts=None, no_upstream_cert=False, body_size_limit=None, mode=None, upstream_server=None, http_form_in=None, http_form_out=None, - authenticator=None, ignore=[], + authenticator=None, ignore_hosts=[], tcp_hosts=[], ciphers=None, certs=[], certforward=False, ssl_ports=TRANSPARENT_SSL_PORTS): self.host = host self.port = port @@ -44,7 +57,8 @@ class ProxyConfig: self.mode.http_form_in = http_form_in or self.mode.http_form_in self.mode.http_form_out = http_form_out or self.mode.http_form_out - self.ignore = parse_host_pattern(ignore) + self.check_ignore = HostMatcher(ignore_hosts) + self.check_tcp = HostMatcher(tcp_hosts) self.authenticator = authenticator self.confdir = os.path.expanduser(confdir) self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) @@ -124,7 +138,8 @@ def process_proxy_options(parser, options): upstream_server=upstream_server, http_form_in=options.http_form_in, http_form_out=options.http_form_out, - ignore=options.ignore, + ignore_hosts=options.ignore_hosts, + tcp_hosts=options.tcp_hosts, authenticator=authenticator, ciphers=options.ciphers, certs=certs, diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 4c7fbbf0..fdf6405a 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -70,13 +70,15 @@ class ConnectionHandler: # Can we already identify the target server and connect to it? client_ssl, server_ssl = False, False + conn_kwargs = dict() upstream_info = self.config.mode.get_upstream_server(self.client_conn) if upstream_info: self.set_server_address(upstream_info[2:]) client_ssl, server_ssl = upstream_info[:2] - if self.check_ignore_address(self.server_conn.address): + if self.config.check_ignore(self.server_conn.address): self.log("Ignore host: %s:%s" % self.server_conn.address(), "info") self.conntype = "tcp" + conn_kwargs["log"] = False client_ssl, server_ssl = False, False else: pass # No upstream info from the metadata: upstream info in the protocol (e.g. HTTP absolute-form) @@ -90,15 +92,19 @@ class ConnectionHandler: if client_ssl or server_ssl: self.establish_ssl(client=client_ssl, server=server_ssl) + if self.config.check_tcp(self.server_conn.address): + self.log("Generic TCP mode for host: %s:%s" % self.server_conn.address(), "info") + self.conntype = "tcp" + # Delegate handling to the protocol handler - protocol_handler(self.conntype)(self).handle_messages() + protocol_handler(self.conntype)(self, **conn_kwargs).handle_messages() self.del_server_connection() self.log("clientdisconnect", "info") self.channel.tell("clientdisconnect", self) except ProxyError as e: - protocol_handler(self.conntype)(self).handle_error(e) + protocol_handler(self.conntype)(self, **conn_kwargs).handle_error(e) except Exception: import traceback, sys @@ -119,14 +125,6 @@ class ConnectionHandler: self.server_conn = None self.sni = None - def check_ignore_address(self, address): - address = tcp.Address.wrap(address) - host = "%s:%s" % (address.host, address.port) - if host and any(rex.search(host) for rex in self.config.ignore): - return True - else: - return False - def set_server_address(self, address): """ Sets a new server address with the given priority. diff --git a/libmproxy/web/static/flows.json b/libmproxy/web/static/flows.json index a0358db0..35accd38 100644 --- a/libmproxy/web/static/flows.json +++ b/libmproxy/web/static/flows.json @@ -93,7 +93,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -259,7 +259,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -425,7 +425,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -595,7 +595,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -765,7 +765,7 @@ "clientcert": null, "ssl_established": true }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -919,7 +919,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1057,7 +1057,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1195,7 +1195,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1329,7 +1329,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1483,7 +1483,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1633,7 +1633,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1767,7 +1767,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -1901,7 +1901,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 @@ -2027,7 +2027,7 @@ "clientcert": null, "ssl_established": false }, - "conntype": "http", + "type": "http", "version": [ 0, 11 |