From e766838a1a6bbb0b2290a3986ba8ae71c736f6f6 Mon Sep 17 00:00:00 2001 From: "Michael J. Bazzinotti" Date: Sat, 2 Jan 2016 10:00:27 -0500 Subject: Add Inline Script Hooks to TCP mode --- docs/scripting/inlinescripts.rst | 63 +++++++++++++++++++++++++-------------- examples/tcp_message.py | 24 +++++++++++++++ libmproxy/flow.py | 4 +++ libmproxy/protocol/rawtcp.py | 19 ++++++++++-- test/scripts/tcp_stream_modify.py | 3 ++ test/test_server.py | 12 ++++++++ 6 files changed, 100 insertions(+), 25 deletions(-) create mode 100644 examples/tcp_message.py create mode 100644 test/scripts/tcp_stream_modify.py diff --git a/docs/scripting/inlinescripts.rst b/docs/scripting/inlinescripts.rst index 19e17582..27e4abef 100644 --- a/docs/scripting/inlinescripts.rst +++ b/docs/scripting/inlinescripts.rst @@ -36,14 +36,13 @@ We encourage you to either browse them locally or on `GitHub`_. Events ------ -.. TODO: Split this into Connection, HTTP and TCP events once we have TCP events. - The ``context`` argument passed to each event method is always a :py:class:`~libmproxy.script.ScriptContext` instance. It is guaranteed to be the same object for the scripts lifetime and is not shared between multiple inline scripts. You can safely use it to store any form of state you require. -Events are listed in the order they usually occur. +Script Lifecycle Events +^^^^^^^^^^^^^^^^^^^^^^^ .. py:function:: start(context, argv) @@ -52,6 +51,13 @@ Events are listed in the order they usually occur. :param List[str] argv: The inline scripts' arguments. For example, ``mitmproxy -s 'example.py --foo 42'`` sets argv to ``["--foo", "42"]``. +.. py:function:: done(context) + + Called once on script shutdown, after any other events. + +Connection Events +^^^^^^^^^^^^^^^^^ + .. py:function:: clientconnect(context, root_layer) Called when a client initiates a connection to the proxy. Note that @@ -64,14 +70,13 @@ Events are listed in the order they usually occur. :py:class:`~libmproxy.proxy.RootContext`. For example, ``root_layer.client_conn.address`` gives the remote address of the connecting client. +.. py:function:: clientdisconnect(context, root_layer) -.. py:function:: request(context, flow) + Called when a client disconnects from the proxy. - Called when a client request has been received. The ``flow`` object is - guaranteed to have a non-None ``request`` attribute. + .. versionchanged:: 0.14 - :param HTTPFlow flow: The flow containing the request which has been received. - The object is guaranteed to have a non-None ``request`` attribute. + :param Layer root_layer: see :py:func:`clientconnect` .. py:function:: serverconnect(context, server_conn) @@ -81,6 +86,25 @@ Events are listed in the order they usually occur. :param ServerConnection server_conn: The server connection object. It is guaranteed to have a non-None ``address`` attribute. +.. py:function:: serverdisconnect(context, server_conn) + + Called when the proxy has closed the server connection. + + .. versionadded:: 0.14 + + :param ServerConnection server_conn: see :py:func:`serverconnect` + +HTTP Events +^^^^^^^^^^^ + +.. py:function:: request(context, flow) + + Called when a client request has been received. The ``flow`` object is + guaranteed to have a non-None ``request`` attribute. + + :param HTTPFlow flow: The flow containing the request which has been received. + The object is guaranteed to have a non-None ``request`` attribute. + .. py:function:: responseheaders(context, flow) Called when the headers of a server response have been received. @@ -109,26 +133,19 @@ Events are listed in the order they usually occur. :param HTTPFlow flow: The flow containing the error. It is guaranteed to have non-None ``error`` attribute. -.. py:function:: serverdisconnect(context, server_conn) +TCP Events +^^^^^^^^^^ - Called when the proxy has closed the server connection. +.. py:function:: tcp_message(context, tcp_msg) - .. versionadded:: 0.14 + .. warning:: API is subject to change - :param ServerConnection server_conn: see :py:func:`serverconnect` + If the proxy is in :ref:`TCP mode `, this event is called when it + receives a TCP payload from the client or server. -.. py:function:: clientdisconnect(context, root_layer) - - Called when a client disconnects from the proxy. - - .. versionchanged:: 0.14 - - :param Layer root_layer: see :py:func:`clientconnect` - -.. py:function:: done(context) - - Called once on script shutdown, after any other events. + The sender and receiver are identifiable. The message is user-modifiable. + :param TcpMessage tcp_msg: see *examples/tcp_message.py* API --- diff --git a/examples/tcp_message.py b/examples/tcp_message.py new file mode 100644 index 00000000..c63368e4 --- /dev/null +++ b/examples/tcp_message.py @@ -0,0 +1,24 @@ +''' +tcp_message Inline Script Hook API Demonstration +------------------------------------------------ + +* modifies packets containing "foo" to "bar" +* prints various details for each packet. + +example cmdline invocation: +mitmdump -T --host --tcp ".*" -q -s examples/tcp_message.py +''' +from netlib.utils import clean_bin + +def tcp_message(ctx, tcp_msg): + modified_msg = tcp_msg.message.replace("foo", "bar") + + is_modified = False if modified_msg == tcp_msg.message else True + tcp_msg.message = modified_msg + + print("[tcp_message{}] from {} {} to {} {}:\r\n{}".format( + " (modified)" if is_modified else "", + "client" if tcp_msg.sender == tcp_msg.client_conn else "server", + tcp_msg.sender.address, + "server" if tcp_msg.receiver == tcp_msg.server_conn else "client", + tcp_msg.receiver.address, clean_bin(tcp_msg.message))) diff --git a/libmproxy/flow.py b/libmproxy/flow.py index a2b069ed..9815a298 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -1050,6 +1050,10 @@ class FlowMaster(controller.Master): self.add_event('"{}" reloaded.'.format(s.filename)) return ok + def handle_tcp_message(self, m): + self.run_script_hook("tcp_message", m) + m.reply() + def shutdown(self): self.unload_scripts() controller.Master.shutdown(self) diff --git a/libmproxy/protocol/rawtcp.py b/libmproxy/protocol/rawtcp.py index 5f08fd17..ccd3c7ec 100644 --- a/libmproxy/protocol/rawtcp.py +++ b/libmproxy/protocol/rawtcp.py @@ -13,6 +13,15 @@ from ..exceptions import ProtocolException from .base import Layer +class TcpMessage(object): + def __init__(self, client_conn, server_conn, sender, receiver, message): + self.client_conn = client_conn + self.server_conn = server_conn + self.sender = sender + self.receiver = receiver + self.message = message + + class RawTCPLayer(Layer): chunk_size = 4096 @@ -50,7 +59,13 @@ class RawTCPLayer(Layer): return continue - dst.sendall(buf[:size]) + tcp_message = TcpMessage( + self.client_conn, self.server_conn, + self.client_conn if dst == server else self.server_conn, + self.server_conn if dst == server else self.client_conn, + buf[:size].tobytes()) + self.channel.ask("tcp_message", tcp_message) + dst.sendall(tcp_message.message) if self.logging: # log messages are prepended with the client address, @@ -59,7 +74,7 @@ class RawTCPLayer(Layer): direction = "-> tcp -> {}".format(repr(self.server_conn.address)) else: direction = "<- tcp <- {}".format(repr(self.server_conn.address)) - data = clean_bin(buf[:size].tobytes()) + data = clean_bin(tcp_message.message) self.log( "{}\r\n{}".format(direction, data), "info" diff --git a/test/scripts/tcp_stream_modify.py b/test/scripts/tcp_stream_modify.py new file mode 100644 index 00000000..9870dddf --- /dev/null +++ b/test/scripts/tcp_stream_modify.py @@ -0,0 +1,3 @@ +def tcp_message(ctx,tm): + if tm.sender == tm.server_conn: + tm.message = tm.message.replace("foo", "bar") diff --git a/test/test_server.py b/test/test_server.py index 5f348121..2e21fce7 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -502,6 +502,18 @@ class TestHttps2Http(tservers.ReverseProxTest): class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin): ssl = False + def test_tcp_stream_modify(self): + self.master.load_script( + tutils.test_data.path("scripts/tcp_stream_modify.py")) + + self._tcpproxy_on() + d = self.pathod('200:b"foo"') + self._tcpproxy_off() + + assert d.content == "bar" + + self.master.unload_scripts() + class TestTransparentSSL(tservers.TransparentProxTest, CommonMixin, TcpMixin): ssl = True -- cgit v1.2.3 From 25705af76d72899140ec43e224aa2636e6a2d8f1 Mon Sep 17 00:00:00 2001 From: Niko Kommenda Date: Tue, 12 Jan 2016 16:41:41 +0000 Subject: added sslstrip to inline script examples --- examples/sslstrip.py | 41 +++++++++++++++++++++++++++++++++++++++++ 1 file changed, 41 insertions(+) create mode 100644 examples/sslstrip.py diff --git a/examples/sslstrip.py b/examples/sslstrip.py new file mode 100644 index 00000000..263ac1c1 --- /dev/null +++ b/examples/sslstrip.py @@ -0,0 +1,41 @@ +from netlib.http import decoded +import re +from six.moves import urllib + +def start(context, argv) : + + #set of SSL/TLS capable hosts + context.secure_hosts = set() + +def request(context, flow) : + + flow.request.headers.pop('Accept-Encoding', None) + flow.request.headers.pop('If-Modified-Since', None) + flow.request.headers.pop('Cache-Control', None) + + #proxy connections to SSL-enabled hosts + if flow.request.pretty_host in context.secure_hosts : + flow.request.scheme = 'https' + flow.request.port = 443 + +def response(context, flow) : + + with decoded(flow.response) : + flow.request.headers.pop('Strict-Transport-Security', None) + flow.request.headers.pop('Public-Key-Pins', None) + + #strip links in response body + flow.response.content = flow.response.content.replace('https://', 'http://') + + #strip links in 'Location' header + if flow.response.headers.get('Location','').startswith('https://'): + location = flow.response.headers['Location'] + hostname = urllib.parse.urlparse(location).hostname + if hostname: + context.secure_hosts.add(hostname) + flow.response.headers['Location'] = location.replace('https://', 'http://', 1) + + #strip secure flag from 'Set-Cookie' headers + cookies = flow.response.headers.get_all('Set-Cookie') + cookies = [re.sub(r';\s*secure\s*', '', s) for s in cookies] + flow.response.headers.set_all('Set-Cookie', cookies) -- cgit v1.2.3 From 55e89865ff4f87f4c8ad16070a6c2177fcf6fac9 Mon Sep 17 00:00:00 2001 From: Niko Kommenda Date: Tue, 12 Jan 2016 22:25:42 +0000 Subject: no longer strips Accept-Encoding as mitmproxy can handle compression --- examples/sslstrip.py | 1 - 1 file changed, 1 deletion(-) diff --git a/examples/sslstrip.py b/examples/sslstrip.py index 263ac1c1..369427a2 100644 --- a/examples/sslstrip.py +++ b/examples/sslstrip.py @@ -9,7 +9,6 @@ def start(context, argv) : def request(context, flow) : - flow.request.headers.pop('Accept-Encoding', None) flow.request.headers.pop('If-Modified-Since', None) flow.request.headers.pop('Cache-Control', None) -- cgit v1.2.3 From cbf94180722daf52794eb185c86d78ac2da9d864 Mon Sep 17 00:00:00 2001 From: Felix Yan Date: Thu, 14 Jan 2016 14:59:51 +0800 Subject: Allow Pillow 3.1 Tested with Pillow 3.1.0, all tests passed and the functionality is working correctly here. --- setup.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/setup.py b/setup.py index a5963807..29ac4753 100644 --- a/setup.py +++ b/setup.py @@ -26,7 +26,7 @@ deps = { "construct>=2.5.2, <2.6", "six>=1.10.0, <1.11", "lxml==3.4.4", # there are no Windows wheels for newer versions, so we pin this. - "Pillow>=3.0.0, <3.1", + "Pillow>=3.0.0, <3.2", "watchdog>=0.8.3, <0.9", } # A script -> additional dependencies dict. -- cgit v1.2.3