diff options
Diffstat (limited to 'libmproxy')
| -rw-r--r-- | libmproxy/cmdline.py | 5 | ||||
| -rw-r--r-- | libmproxy/console/__init__.py | 19 | ||||
| -rw-r--r-- | libmproxy/console/flowdetailview.py | 4 | ||||
| -rw-r--r-- | libmproxy/flow.py | 12 | ||||
| -rw-r--r-- | libmproxy/main.py | 1 | ||||
| -rw-r--r-- | libmproxy/models/connections.py | 4 | ||||
| -rw-r--r-- | libmproxy/protocol/__init__.py | 4 | ||||
| -rw-r--r-- | libmproxy/protocol/base.py | 5 | ||||
| -rw-r--r-- | libmproxy/protocol/http_replay.py | 4 | ||||
| -rw-r--r-- | libmproxy/protocol/rawtcp.py | 19 | ||||
| -rw-r--r-- | libmproxy/protocol/tls.py | 126 | ||||
| -rw-r--r-- | libmproxy/proxy/modes/socks_proxy.py | 3 | ||||
| -rw-r--r-- | libmproxy/proxy/root_context.py | 22 | ||||
| -rw-r--r-- | libmproxy/script/reloader.py | 26 |
14 files changed, 167 insertions, 87 deletions
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 99b76e68..cd1a8bba 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -669,6 +669,11 @@ def mitmproxy(): help="Show event log." ) parser.add_argument( + "-f", "--follow", + action="store_true", dest="follow", + help="Follow flow list." + ) + parser.add_argument( "--no-mouse", action="store_true", dest="no_mouse", help="Disable mouse interaction." diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 31edca81..0df10256 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -105,25 +105,25 @@ class ConsoleState(flow.State): for f in self.flows: if self.flow_marked(f): marked_flows.append(f) - + super(ConsoleState, self).clear() - + for f in marked_flows: self.add_flow(f) self.set_flow_marked(f, True) - + if len(self.flows.views) == 0: self.focus = None else: self.focus = 0 self.set_focus(self.focus) - + def flow_marked(self, flow): return self.get_flow_setting(flow, "marked", False) - + def set_flow_marked(self, flow, marked): self.add_flow_setting(flow, "marked", marked) - + class Options(object): attributes = [ @@ -134,6 +134,7 @@ class Options(object): "anticomp", "client_replay", "eventlog", + "follow", "keepserving", "kill", "intercept", @@ -212,6 +213,7 @@ class ConsoleMaster(flow.FlowMaster): self.eventlog = options.eventlog self.eventlist = urwid.SimpleListWalker([]) + self.follow = options.follow if options.client_replay: self.client_playback_path(options.client_replay) @@ -562,6 +564,9 @@ class ConsoleMaster(flow.FlowMaster): else: body = flowlist.FlowListBox(self) + if self.follow: + self.toggle_follow_flows() + signals.push_view_state.send( self, window = window.Window( @@ -604,7 +609,7 @@ class ConsoleMaster(flow.FlowMaster): def save_flows(self, path): return self._write_flows(path, self.state.view) - + def save_marked_flows(self, path): marked_flows = [] for f in self.state.view: diff --git a/libmproxy/console/flowdetailview.py b/libmproxy/console/flowdetailview.py index 40769c95..394ad217 100644 --- a/libmproxy/console/flowdetailview.py +++ b/libmproxy/console/flowdetailview.py @@ -20,7 +20,7 @@ def flowdetails(state, flow): req = flow.request resp = flow.response - if sc: + if sc is not None: text.append(urwid.Text([("head", "Server Connection:")])) parts = [ ["Address", "%s:%s" % sc.address()], @@ -76,7 +76,7 @@ def flowdetails(state, flow): common.format_keyvals(parts, key="key", val="text", indent=4) ) - if cc: + if cc is not None: text.append(urwid.Text([("head", "Client Connection:")])) parts = [ diff --git a/libmproxy/flow.py b/libmproxy/flow.py index f02b5767..1f28166f 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -666,7 +666,7 @@ class FlowMaster(controller.Master): script.reloader.unwatch(script_obj) self.scripts.remove(script_obj) - def load_script(self, command, use_reloader=False): + def load_script(self, command, use_reloader=True): """ Loads a script. Returns an error description if something went wrong. @@ -1040,16 +1040,20 @@ class FlowMaster(controller.Master): s.unload() except script.ScriptException as e: ok = False - self.add_event('Error reloading "{}": {}'.format(s.filename, str(e))) + self.add_event('Error reloading "{}": {}'.format(s.filename, str(e)), 'error') try: s.load() except script.ScriptException as e: ok = False - self.add_event('Error reloading "{}": {}'.format(s.filename, str(e))) + self.add_event('Error reloading "{}": {}'.format(s.filename, str(e)), 'error') else: - self.add_event('"{}" reloaded.'.format(s.filename)) + self.add_event('"{}" reloaded.'.format(s.filename), 'info') return ok + def handle_tcp_message(self, m): + self.run_script_hook("tcp_message", m) + m.reply() + def shutdown(self): self.unload_scripts() controller.Master.shutdown(self) diff --git a/libmproxy/main.py b/libmproxy/main.py index 3c908ed9..655d573d 100644 --- a/libmproxy/main.py +++ b/libmproxy/main.py @@ -54,6 +54,7 @@ def mitmproxy(args=None): # pragma: nocover console_options.palette = options.palette console_options.palette_transparent = options.palette_transparent console_options.eventlog = options.eventlog + console_options.follow = options.follow console_options.intercept = options.intercept console_options.limit = options.limit console_options.no_mouse = options.no_mouse diff --git a/libmproxy/models/connections.py b/libmproxy/models/connections.py index 0991955d..f5dabe4e 100644 --- a/libmproxy/models/connections.py +++ b/libmproxy/models/connections.py @@ -88,8 +88,8 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): class ServerConnection(tcp.TCPClient, stateobject.StateObject): - def __init__(self, address): - tcp.TCPClient.__init__(self, address) + def __init__(self, address, source_address=None): + tcp.TCPClient.__init__(self, address, source_address) self.via = None self.timestamp_start = None diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index 0d624fd7..d8ebd4f0 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -28,12 +28,12 @@ as late as possible; this makes server replay without any outgoing connections p from __future__ import (absolute_import, print_function, division) from .base import Layer, ServerConnectionMixin, Kill from .http import Http1Layer, UpstreamConnectLayer, Http2Layer -from .tls import TlsLayer, is_tls_record_magic +from .tls import TlsLayer, is_tls_record_magic, TlsClientHello from .rawtcp import RawTCPLayer __all__ = [ "Layer", "ServerConnectionMixin", "Kill", "Http1Layer", "UpstreamConnectLayer", "Http2Layer", - "TlsLayer", "is_tls_record_magic", + "TlsLayer", "is_tls_record_magic", "TlsClientHello" "RawTCPLayer" ] diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index af6b1c3b..d984cadb 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -111,7 +111,7 @@ class ServerConnectionMixin(object): def __init__(self, server_address=None): super(ServerConnectionMixin, self).__init__() - self.server_conn = ServerConnection(server_address) + self.server_conn = ServerConnection(server_address, (self.config.host, 0)) self.__check_self_connect() def __check_self_connect(self): @@ -157,10 +157,11 @@ class ServerConnectionMixin(object): """ self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) address = self.server_conn.address + source_address = self.server_conn.source_address self.server_conn.finish() self.server_conn.close() self.channel.tell("serverdisconnect", self.server_conn) - self.server_conn = ServerConnection(address) + self.server_conn = ServerConnection(address, source_address) def connect(self): """ diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py index b7faad07..63870dfb 100644 --- a/libmproxy/protocol/http_replay.py +++ b/libmproxy/protocol/http_replay.py @@ -46,7 +46,7 @@ class RequestReplayThread(threading.Thread): # In all modes, we directly connect to the server displayed if self.config.mode == "upstream": server_address = self.config.upstream_server.address - server = ServerConnection(server_address) + server = ServerConnection(server_address, (self.config.host, 0)) server.connect() if r.scheme == "https": connect_request = make_connect_request((r.host, r.port)) @@ -68,7 +68,7 @@ class RequestReplayThread(threading.Thread): r.form_out = "absolute" else: server_address = (r.host, r.port) - server = ServerConnection(server_address) + server = ServerConnection(server_address, (self.config.host, 0)) server.connect() if r.scheme == "https": server.establish_ssl( diff --git a/libmproxy/protocol/rawtcp.py b/libmproxy/protocol/rawtcp.py index 5f08fd17..ccd3c7ec 100644 --- a/libmproxy/protocol/rawtcp.py +++ b/libmproxy/protocol/rawtcp.py @@ -13,6 +13,15 @@ from ..exceptions import ProtocolException from .base import Layer +class TcpMessage(object): + def __init__(self, client_conn, server_conn, sender, receiver, message): + self.client_conn = client_conn + self.server_conn = server_conn + self.sender = sender + self.receiver = receiver + self.message = message + + class RawTCPLayer(Layer): chunk_size = 4096 @@ -50,7 +59,13 @@ class RawTCPLayer(Layer): return continue - dst.sendall(buf[:size]) + tcp_message = TcpMessage( + self.client_conn, self.server_conn, + self.client_conn if dst == server else self.server_conn, + self.server_conn if dst == server else self.client_conn, + buf[:size].tobytes()) + self.channel.ask("tcp_message", tcp_message) + dst.sendall(tcp_message.message) if self.logging: # log messages are prepended with the client address, @@ -59,7 +74,7 @@ class RawTCPLayer(Layer): direction = "-> tcp -> {}".format(repr(self.server_conn.address)) else: direction = "<- tcp <- {}".format(repr(self.server_conn.address)) - data = clean_bin(buf[:size].tobytes()) + data = clean_bin(tcp_message.message) self.log( "{}\r\n{}".format(direction, data), "info" diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index ed747643..6d4cac85 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -221,6 +221,80 @@ def is_tls_record_magic(d): d[2] in ('\x00', '\x01', '\x02', '\x03') ) +def get_client_hello(client_conn): + """ + Peek into the socket and read all records that contain the initial client hello message. + + client_conn: + The :py:class:`client connection <libmproxy.models.ClientConnection>`. + + Returns: + The raw handshake packet bytes, without TLS record header(s). + """ + client_hello = "" + client_hello_size = 1 + offset = 0 + while len(client_hello) < client_hello_size: + record_header = client_conn.rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) != 5: + raise TlsProtocolException('Expected TLS record, got "%s" instead.' % record_header) + record_size = struct.unpack("!H", record_header[3:])[0] + 5 + record_body = client_conn.rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise TlsProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) + client_hello += record_body + offset += record_size + client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 + return client_hello + +class TlsClientHello(object): + def __init__(self, raw_client_hello): + self._client_hello = ClientHello.parse(raw_client_hello) + + def raw(self): + return self._client_hello + + @property + def client_cipher_suites(self): + return self._client_hello.cipher_suites.cipher_suites + + @property + def client_sni(self): + for extension in self._client_hello.extensions: + if (extension.type == 0x00 and len(extension.server_names) == 1 + and extension.server_names[0].type == 0): + return extension.server_names[0].name + + @property + def client_alpn_protocols(self): + for extension in self._client_hello.extensions: + if extension.type == 0x10: + return list(extension.alpn_protocols) + + @classmethod + def from_client_conn(cls, client_conn): + """ + Peek into the connection, read the initial client hello and parse it to obtain ALPN values. + client_conn: + The :py:class:`client connection <libmproxy.models.ClientConnection>`. + Returns: + :py:class:`client hello <libmproxy.protocol.tls.TlsClientHello>`. + """ + try: + raw_client_hello = get_client_hello(client_conn)[4:] # exclude handshake header. + except ProtocolException as e: + raise TlsProtocolException('Cannot read raw Client Hello: %s' % repr(e)) + + try: + return cls(raw_client_hello) + except ConstructError as e: + raise TlsProtocolException('Cannot parse Client Hello: %s, Raw Client Hello: %s' % \ + (repr(e), raw_client_hello.encode("hex"))) + + def __repr__(self): + return "TlsClientHello( sni: %s alpn_protocols: %s, cipher_suites: %s)" % \ + (self.client_sni, self.client_alpn_protocols, self.client_cipher_suites) + class TlsLayer(Layer): def __init__(self, ctx, client_tls, server_tls): @@ -281,60 +355,18 @@ class TlsLayer(Layer): else: return "TlsLayer(inactive)" - def _get_client_hello(self): - """ - Peek into the socket and read all records that contain the initial client hello message. - - Returns: - The raw handshake packet bytes, without TLS record header(s). - """ - client_hello = "" - client_hello_size = 1 - offset = 0 - while len(client_hello) < client_hello_size: - record_header = self.client_conn.rfile.peek(offset + 5)[offset:] - if not is_tls_record_magic(record_header) or len(record_header) != 5: - raise TlsProtocolException('Expected TLS record, got "%s" instead.' % record_header) - record_size = struct.unpack("!H", record_header[3:])[0] + 5 - record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] - if len(record_body) != record_size - 5: - raise TlsProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) - client_hello += record_body - offset += record_size - client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 - return client_hello def _parse_client_hello(self): """ Peek into the connection, read the initial client hello and parse it to obtain ALPN values. """ try: - raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. - except ProtocolException as e: - self.log("Cannot parse Client Hello: %s" % repr(e), "error") - return - - try: - client_hello = ClientHello.parse(raw_client_hello) - except ConstructError as e: + parsed = TlsClientHello.from_client_conn(self.client_conn) + self.client_sni = parsed.client_sni + self.client_alpn_protocols = parsed.client_alpn_protocols + self.client_ciphers = parsed.client_cipher_suites + except TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") - self.log("Raw Client Hello: %s" % raw_client_hello.encode("hex"), "debug") - return - - self.client_ciphers = client_hello.cipher_suites.cipher_suites - - for extension in client_hello.extensions: - if extension.type == 0x00: - if len(extension.server_names) != 1 or extension.server_names[0].type != 0: - self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") - self.client_sni = extension.server_names[0].name - elif extension.type == 0x10: - self.client_alpn_protocols = list(extension.alpn_protocols) - - self.log( - "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), - "debug" - ) def connect(self): if not self.server_conn: diff --git a/libmproxy/proxy/modes/socks_proxy.py b/libmproxy/proxy/modes/socks_proxy.py index 264c734a..90788e37 100644 --- a/libmproxy/proxy/modes/socks_proxy.py +++ b/libmproxy/proxy/modes/socks_proxy.py @@ -8,6 +8,9 @@ from ...protocol import Layer, ServerConnectionMixin class Socks5Proxy(Layer, ServerConnectionMixin): + def __init__(self, ctx): + super(Socks5Proxy, self).__init__(ctx) + def __call__(self): try: # Parse Client Greeting diff --git a/libmproxy/proxy/root_context.py b/libmproxy/proxy/root_context.py index f62b0c8e..d70fc299 100644 --- a/libmproxy/proxy/root_context.py +++ b/libmproxy/proxy/root_context.py @@ -4,15 +4,14 @@ import sys import six -from libmproxy.exceptions import ProtocolException +from libmproxy.exceptions import ProtocolException, TlsProtocolException from netlib.exceptions import TcpException from ..protocol import ( RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin, - UpstreamConnectLayer + UpstreamConnectLayer, TlsClientHello ) from .modes import HttpProxy, HttpUpstreamProxy, ReverseProxy - class RootContext(object): """ The outermost context provided to the root layer. @@ -48,16 +47,25 @@ class RootContext(object): return self.channel.ask("next_layer", layer) def _next_layer(self, top_layer): - # 1. Check for --ignore. - if self.config.check_ignore(top_layer.server_conn.address): - return RawTCPLayer(top_layer, logging=False) - try: d = top_layer.client_conn.rfile.peek(3) except TcpException as e: six.reraise(ProtocolException, ProtocolException(str(e)), sys.exc_info()[2]) client_tls = is_tls_record_magic(d) + # 1. check for --ignore + if self.config.check_ignore: + ignore = self.config.check_ignore(top_layer.server_conn.address) + if not ignore and client_tls: + try: + client_hello = TlsClientHello.from_client_conn(self.client_conn) + except TlsProtocolException as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + else: + ignore = self.config.check_ignore((client_hello.client_sni, 443)) + if ignore: + return RawTCPLayer(top_layer, logging=False) + # 2. Always insert a TLS layer, even if there's neither client nor server tls. # An inline script may upgrade from http to https, # in which case we need some form of TLS layer. diff --git a/libmproxy/script/reloader.py b/libmproxy/script/reloader.py index 26691fa3..e81bdef6 100644 --- a/libmproxy/script/reloader.py +++ b/libmproxy/script/reloader.py @@ -1,6 +1,12 @@ import os -from watchdog.events import PatternMatchingEventHandler -from watchdog.observers import Observer +import sys +from watchdog.events import RegexMatchingEventHandler +if sys.platform == 'darwin': + from watchdog.observers.polling import PollingObserver as Observer +else: + from watchdog.observers import Observer +# The OSX reloader in watchdog 0.8.3 breaks when unobserving paths. +# We use the PollingObserver instead. _observers = {} @@ -9,7 +15,8 @@ def watch(script, callback): if script in _observers: raise RuntimeError("Script already observed") script_dir = os.path.dirname(os.path.abspath(script.args[0])) - event_handler = _ScriptModificationHandler(callback) + script_name = os.path.basename(script.args[0]) + event_handler = _ScriptModificationHandler(callback, filename=script_name) observer = Observer() observer.schedule(event_handler, script_dir) observer.start() @@ -23,18 +30,17 @@ def unwatch(script): observer.join() -class _ScriptModificationHandler(PatternMatchingEventHandler): - def __init__(self, callback): - # We could enumerate all relevant *.py files (as werkzeug does it), - # but our case looks like it isn't as simple as enumerating sys.modules. - # This should be good enough for now. +class _ScriptModificationHandler(RegexMatchingEventHandler): + def __init__(self, callback, filename='.*'): + super(_ScriptModificationHandler, self).__init__( ignore_directories=True, - patterns=["*.py"] + regexes=['.*'+filename] ) self.callback = callback def on_modified(self, event): self.callback() -__all__ = ["watch", "unwatch"]
\ No newline at end of file +__all__ = ["watch", "unwatch"] + |