combine projects

5 files changed, 522 insertions, 0 deletions

diff --git a/mitmproxy/platform/__init__.py b/mitmproxy/platform/__init__.py
new file mode 100644
index 00000000..e1ff7c47
--- /dev/null
+++ b/mitmproxy/platform/__init__.py
@@ -0,0 +1,16 @@
+import sys
+
+resolver = None
+
+if sys.platform == "linux2":
+    from . import linux
+    resolver = linux.Resolver
+elif sys.platform == "darwin":
+    from . import osx
+    resolver = osx.Resolver
+elif sys.platform.startswith("freebsd"):
+    from . import osx
+    resolver = osx.Resolver
+elif sys.platform == "win32":
+    from . import windows
+    resolver = windows.Resolver
diff --git a/mitmproxy/platform/linux.py b/mitmproxy/platform/linux.py
new file mode 100644
index 00000000..38bfbe42
--- /dev/null
+++ b/mitmproxy/platform/linux.py
@@ -0,0 +1,14 @@
+import socket
+import struct
+
+# Python socket module does not have this constant
+SO_ORIGINAL_DST = 80
+
+
+class Resolver(object):
+
+    def original_addr(self, csock):
+        odestdata = csock.getsockopt(socket.SOL_IP, SO_ORIGINAL_DST, 16)
+        _, port, a1, a2, a3, a4 = struct.unpack("!HHBBBBxxxxxxxx", odestdata)
+        address = "%d.%d.%d.%d" % (a1, a2, a3, a4)
+        return address, port
diff --git a/mitmproxy/platform/osx.py b/mitmproxy/platform/osx.py
new file mode 100644
index 00000000..afbc919b
--- /dev/null
+++ b/mitmproxy/platform/osx.py
@@ -0,0 +1,36 @@
+import subprocess
+import pf
+
+"""
+    Doing this the "right" way by using DIOCNATLOOK on the pf device turns out
+    to be a pain. Apple has made a number of modifications to the data
+    structures returned, and compiling userspace tools to test and work with
+    this turns out to be a pain in the ass. Parsing pfctl output is short,
+    simple, and works.
+
+    Note: Also Tested with FreeBSD 10 pkgng Python 2.7.x.
+    Should work almost exactly as on Mac OS X and except with some changes to
+    the output processing of pfctl (see pf.py).
+"""
+
+
+class Resolver(object):
+    STATECMD = ("sudo", "-n", "/sbin/pfctl", "-s", "state")
+
+    def original_addr(self, csock):
+        peer = csock.getpeername()
+        try:
+            stxt = subprocess.check_output(self.STATECMD, stderr=subprocess.STDOUT)
+        except subprocess.CalledProcessError as e:
+            if "sudo: a password is required" in e.output:
+                insufficient_priv = True
+            else:
+                raise RuntimeError("Error getting pfctl state: " + repr(e))
+        else:
+            insufficient_priv = "sudo: a password is required" in stxt
+
+        if insufficient_priv:
+            raise RuntimeError(
+                "Insufficient privileges to access pfctl. "
+                "See http://mitmproxy.org/doc/transparent/osx.html for details.")
+        return pf.lookup(peer[0], peer[1], stxt)
diff --git a/mitmproxy/platform/pf.py b/mitmproxy/platform/pf.py
new file mode 100644
index 00000000..97a4c192
--- /dev/null
+++ b/mitmproxy/platform/pf.py
@@ -0,0 +1,24 @@
+import sys
+
+
+def lookup(address, port, s):
+    """
+        Parse the pfctl state output s, to look up the destination host
+        matching the client (address, port).
+
+        Returns an (address, port) tuple, or None.
+    """
+    spec = "%s:%s" % (address, port)
+    for i in s.split("\n"):
+        if "ESTABLISHED:ESTABLISHED" in i and spec in i:
+            s = i.split()
+            if len(s) > 4:
+                if sys.platform.startswith("freebsd"):
+                    # strip parentheses for FreeBSD pfctl
+                    s = s[3][1:-1].split(":")
+                else:
+                    s = s[4].split(":")
+
+                if len(s) == 2:
+                    return s[0], int(s[1])
+    raise RuntimeError("Could not resolve original destination.")
diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py
new file mode 100644
index 00000000..9fe04cfa
--- /dev/null
+++ b/mitmproxy/platform/windows.py
@@ -0,0 +1,432 @@
+import configargparse
+import cPickle as pickle
+from ctypes import byref, windll, Structure
+from ctypes.wintypes import DWORD
+import os
+import socket
+import SocketServer
+import struct
+import threading
+import time
+from collections import OrderedDict
+
+from pydivert.windivert import WinDivert
+from pydivert.enum import Direction, Layer, Flag
+
+
+PROXY_API_PORT = 8085
+
+
+class Resolver(object):
+
+    def __init__(self):
+        TransparentProxy.setup()
+        self.socket = None
+        self.lock = threading.RLock()
+        self._connect()
+
+    def _connect(self):
+        if self.socket:
+            self.socket.close()
+        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.socket.connect(("127.0.0.1", PROXY_API_PORT))
+
+        self.wfile = self.socket.makefile('wb')
+        self.rfile = self.socket.makefile('rb')
+        pickle.dump(os.getpid(), self.wfile)
+
+    def original_addr(self, csock):
+        client = csock.getpeername()[:2]
+        with self.lock:
+            try:
+                pickle.dump(client, self.wfile)
+                self.wfile.flush()
+                addr = pickle.load(self.rfile)
+                if addr is None:
+                    raise RuntimeError("Cannot resolve original destination.")
+                addr = list(addr)
+                addr[0] = str(addr[0])
+                addr = tuple(addr)
+                return addr
+            except (EOFError, socket.error):
+                self._connect()
+                return self.original_addr(csock)
+
+
+class APIRequestHandler(SocketServer.StreamRequestHandler):
+
+    """
+    TransparentProxy API: Returns the pickled server address, port tuple
+    for each received pickled client address, port tuple.
+    """
+
+    def handle(self):
+        proxifier = self.server.proxifier
+        pid = None
+        try:
+            pid = pickle.load(self.rfile)
+            if pid is not None:
+                proxifier.trusted_pids.add(pid)
+
+            while True:
+                client = pickle.load(self.rfile)
+                server = proxifier.client_server_map.get(client, None)
+                pickle.dump(server, self.wfile)
+                self.wfile.flush()
+
+        except (EOFError, socket.error):
+            proxifier.trusted_pids.discard(pid)
+
+
+class APIServer(SocketServer.ThreadingMixIn, SocketServer.TCPServer):
+
+    def __init__(self, proxifier, *args, **kwargs):
+        SocketServer.TCPServer.__init__(self, *args, **kwargs)
+        self.proxifier = proxifier
+        self.daemon_threads = True
+
+
+# Windows error.h
+ERROR_INSUFFICIENT_BUFFER = 0x7A
+
+
+# http://msdn.microsoft.com/en-us/library/windows/desktop/bb485761(v=vs.85).aspx
+class MIB_TCPROW2(Structure):
+    _fields_ = [
+        ('dwState', DWORD),
+        ('dwLocalAddr', DWORD),
+        ('dwLocalPort', DWORD),
+        ('dwRemoteAddr', DWORD),
+        ('dwRemotePort', DWORD),
+        ('dwOwningPid', DWORD),
+        ('dwOffloadState', DWORD)
+    ]
+
+
+# http://msdn.microsoft.com/en-us/library/windows/desktop/bb485772(v=vs.85).aspx
+def MIB_TCPTABLE2(size):
+    class _MIB_TCPTABLE2(Structure):
+        _fields_ = [('dwNumEntries', DWORD),
+                    ('table', MIB_TCPROW2 * size)]
+
+    return _MIB_TCPTABLE2()
+
+
+class TransparentProxy(object):
+
+    """
+    Transparent Windows Proxy for mitmproxy based on WinDivert/PyDivert.
+
+    Requires elevated (admin) privileges. Can be started separately by manually running the file.
+
+    This module can be used to intercept and redirect all traffic that is forwarded by the user's machine and
+    traffic sent from the machine itself.
+
+    How it works:
+
+    (1) First, we intercept all packages that match our filter (destination port 80 and 443 by default).
+    We both consider traffic that is forwarded by the OS (WinDivert's NETWORK_FORWARD layer) as well as traffic
+    sent from the local machine (WinDivert's NETWORK layer). In the case of traffic from the local machine, we need to
+    distinguish between traffc sent from applications and traffic sent from the proxy. To accomplish this, we use
+    Windows' GetTcpTable2 syscall to determine the source application's PID.
+
+    For each intercepted package, we
+        1. Store the source -> destination mapping (address and port)
+        2. Remove the package from the network (by not reinjecting it).
+        3. Re-inject the package into the local network stack, but with the destination address changed to the proxy.
+
+    (2) Next, the proxy receives the forwarded packet, but does not know the real destination yet (which we overwrote
+    with the proxy's address). On Linux, we would now call getsockopt(SO_ORIGINAL_DST), but that unfortunately doesn't
+    work on Windows. However, we still have the correct source information. As a workaround, we now access the forward
+    module's API (see APIRequestHandler), submit the source information and get the actual destination back (which the
+    forward module stored in (1.3)).
+
+    (3) The proxy now establish the upstream connection as usual.
+
+    (4) Finally, the proxy sends the response back to the client. To make it work, we need to change the packet's source
+    address back to the original destination (using the mapping from (1.3)), to which the client believes he is talking
+    to.
+
+    Limitations:
+
+    - No IPv6 support. (Pull Requests welcome)
+    - TCP ports do not get re-used simulateously on the client, i.e. the proxy will fail if application X
+      connects to example.com and example.org from 192.168.0.42:4242 simultaneously. This could be mitigated by
+      introducing unique "meta-addresses" which mitmproxy sees, but this would remove the correct client info from
+      mitmproxy.
+
+    """
+
+    def __init__(self,
+                 mode="both",
+                 redirect_ports=(80, 443), custom_filter=None,
+                 proxy_addr=False, proxy_port=8080,
+                 api_host="localhost", api_port=PROXY_API_PORT,
+                 cache_size=65536):
+        """
+        :param mode: Redirection operation mode: "forward" to only redirect forwarded packets, "local" to only redirect
+        packets originating from the local machine, "both" to redirect both.
+        :param redirect_ports: if the destination port is in this tuple, the requests are redirected to the proxy.
+        :param custom_filter: specify a custom WinDivert filter to select packets that should be intercepted. Overrides
+        redirect_ports setting.
+        :param proxy_addr: IP address of the proxy (IP within a network, 127.0.0.1 does not work). By default,
+        this is detected automatically.
+        :param proxy_port: Port the proxy is listenting on.
+        :param api_host: Host the forward module API is listening on.
+        :param api_port: Port the forward module API is listening on.
+        :param cache_size: Maximum number of connection tuples that are stored. Only relevant in very high
+        load scenarios.
+        """
+        if proxy_port in redirect_ports:
+            raise ValueError("The proxy port must not be a redirect port.")
+
+        if not proxy_addr:
+            # Auto-Detect local IP.
+            # https://stackoverflow.com/questions/166506/finding-local-ip-addresses-using-pythons-stdlib
+            s = socket.socket(socket.AF_INET, socket.SOCK_DGRAM)
+            s.connect(("8.8.8.8", 80))
+            proxy_addr = s.getsockname()[0]
+            s.close()
+
+        self.mode = mode
+        self.proxy_addr, self.proxy_port = proxy_addr, proxy_port
+        self.connection_cache_size = cache_size
+
+        self.client_server_map = OrderedDict()
+
+        self.api = APIServer(self, (api_host, api_port), APIRequestHandler)
+        self.api_thread = threading.Thread(target=self.api.serve_forever)
+        self.api_thread.daemon = True
+
+        self.driver = WinDivert()
+        self.driver.register()
+
+        self.request_filter = custom_filter or " or ".join(
+            ("tcp.DstPort == %d" %
+             p) for p in redirect_ports)
+        self.request_forward_handle = None
+        self.request_forward_thread = threading.Thread(
+            target=self.request_forward)
+        self.request_forward_thread.daemon = True
+
+        self.addr_pid_map = dict()
+        self.trusted_pids = set()
+        self.tcptable2 = MIB_TCPTABLE2(0)
+        self.tcptable2_size = DWORD(0)
+        self.request_local_handle = None
+        self.request_local_thread = threading.Thread(target=self.request_local)
+        self.request_local_thread.daemon = True
+
+        # The proxy server responds to the client. To the client,
+        # this response should look like it has been sent by the real target
+        self.response_filter = "outbound and tcp.SrcPort == %d" % proxy_port
+        self.response_handle = None
+        self.response_thread = threading.Thread(target=self.response)
+        self.response_thread.daemon = True
+
+        self.icmp_handle = None
+
+    @classmethod
+    def setup(cls):
+        # TODO: Make sure that server can be killed cleanly. That's a bit difficult as we don't have access to
+        # controller.should_exit when this is called.
+        s = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        server_unavailable = s.connect_ex(("127.0.0.1", PROXY_API_PORT))
+        if server_unavailable:
+            proxifier = TransparentProxy()
+            proxifier.start()
+
+    def start(self):
+        self.api_thread.start()
+
+        # Block all ICMP requests (which are sent on Windows by default).
+        # In layman's terms: If we don't do this, our proxy machine tells the client that it can directly connect to the
+        # real gateway if they are on the same network.
+        self.icmp_handle = self.driver.open_handle(
+            filter="icmp",
+            layer=Layer.NETWORK,
+            flags=Flag.DROP)
+
+        self.response_handle = self.driver.open_handle(
+            filter=self.response_filter,
+            layer=Layer.NETWORK)
+        self.response_thread.start()
+
+        if self.mode == "forward" or self.mode == "both":
+            self.request_forward_handle = self.driver.open_handle(
+                filter=self.request_filter,
+                layer=Layer.NETWORK_FORWARD)
+            self.request_forward_thread.start()
+        if self.mode == "local" or self.mode == "both":
+            self.request_local_handle = self.driver.open_handle(
+                filter=self.request_filter,
+                layer=Layer.NETWORK)
+            self.request_local_thread.start()
+
+    def shutdown(self):
+        if self.mode == "local" or self.mode == "both":
+            self.request_local_handle.close()
+        if self.mode == "forward" or self.mode == "both":
+            self.request_forward_handle.close()
+
+        self.response_handle.close()
+        self.icmp_handle.close()
+        self.api.shutdown()
+
+    def recv(self, handle):
+        """
+        Convenience function that receives a packet from the passed handler and handles error codes.
+        If the process has been shut down, (None, None) is returned.
+        """
+        try:
+            raw_packet, metadata = handle.recv()
+            return self.driver.parse_packet(raw_packet), metadata
+        except WindowsError as e:
+            if e.winerror == 995:
+                return None, None
+            else:
+                raise
+
+    def fetch_pids(self):
+        ret = windll.iphlpapi.GetTcpTable2(
+            byref(
+                self.tcptable2), byref(
+                self.tcptable2_size), 0)
+        if ret == ERROR_INSUFFICIENT_BUFFER:
+            self.tcptable2 = MIB_TCPTABLE2(self.tcptable2_size.value)
+            self.fetch_pids()
+        elif ret == 0:
+            for row in self.tcptable2.table[:self.tcptable2.dwNumEntries]:
+                local = (
+                    socket.inet_ntoa(struct.pack('L', row.dwLocalAddr)),
+                    socket.htons(row.dwLocalPort)
+                )
+                self.addr_pid_map[local] = row.dwOwningPid
+        else:
+            raise RuntimeError("Unknown GetTcpTable2 return code: %s" % ret)
+
+    def request_local(self):
+        while True:
+            packet, metadata = self.recv(self.request_local_handle)
+            if not packet:
+                return
+
+            client = (packet.src_addr, packet.src_port)
+
+            if client not in self.addr_pid_map:
+                self.fetch_pids()
+
+            # If this fails, we most likely have a connection from an external client to
+            # a local server on 80/443. In this, case we always want to proxy
+            # the request.
+            pid = self.addr_pid_map.get(client, None)
+
+            if pid not in self.trusted_pids:
+                self._request(packet, metadata)
+            else:
+                self.request_local_handle.send((packet.raw, metadata))
+
+    def request_forward(self):
+        """
+        Redirect packages to the proxy
+        """
+        while True:
+            packet, metadata = self.recv(self.request_forward_handle)
+            if not packet:
+                return
+
+            self._request(packet, metadata)
+
+    def _request(self, packet, metadata):
+        # print(" * Redirect client -> server to proxy")
+        # print("%s:%s -> %s:%s" % (packet.src_addr, packet.src_port, packet.dst_addr, packet.dst_port))
+        client = (packet.src_addr, packet.src_port)
+        server = (packet.dst_addr, packet.dst_port)
+
+        if client in self.client_server_map:
+            # Force re-add to mark as "newest" entry in the dict.
+            del self.client_server_map[client]
+        while len(self.client_server_map) > self.connection_cache_size:
+            self.client_server_map.popitem(False)
+
+        self.client_server_map[client] = server
+
+        packet.dst_addr, packet.dst_port = self.proxy_addr, self.proxy_port
+        metadata.direction = Direction.INBOUND
+
+        packet = self.driver.update_packet_checksums(packet)
+        # Use any handle thats on the NETWORK layer - request_local may be
+        # unavailable.
+        self.response_handle.send((packet.raw, metadata))
+
+    def response(self):
+        """
+        Spoof source address of packets send from the proxy to the client
+        """
+        while True:
+            packet, metadata = self.recv(self.response_handle)
+            if not packet:
+                return
+
+            # If the proxy responds to the client, let the client believe the target server sent the packets.
+            # print(" * Adjust proxy -> client")
+            client = (packet.dst_addr, packet.dst_port)
+            server = self.client_server_map.get(client, None)
+            if server:
+                packet.src_addr, packet.src_port = server
+            else:
+                print("Warning: Previously unseen connection from proxy to %s:%s." % client)
+
+            packet = self.driver.update_packet_checksums(packet)
+            self.response_handle.send((packet.raw, metadata))
+
+
+if __name__ == "__main__":
+    parser = configargparse.ArgumentParser(
+        description="Windows Transparent Proxy")
+    parser.add_argument(
+        '--mode',
+        choices=[
+            'forward',
+            'local',
+            'both'],
+        default="both",
+        help='redirection operation mode: "forward" to only redirect forwarded packets, '
+        '"local" to only redirect packets originating from the local machine')
+    group = parser.add_mutually_exclusive_group()
+    group.add_argument(
+        "--redirect-ports",
+        nargs="+",
+        type=int,
+        default=[
+            80,
+            443],
+        metavar="80",
+        help="ports that should be forwarded to the proxy")
+    group.add_argument(
+        "--custom-filter",
+        default=None,
+        metavar="WINDIVERT_FILTER",
+        help="Custom WinDivert interception rule.")
+    parser.add_argument("--proxy-addr", default=False,
+                        help="Proxy Server Address")
+    parser.add_argument("--proxy-port", type=int, default=8080,
+                        help="Proxy Server Port")
+    parser.add_argument("--api-host", default="localhost",
+                        help="API hostname to bind to")
+    parser.add_argument("--api-port", type=int, default=PROXY_API_PORT,
+                        help="API port")
+    parser.add_argument("--cache-size", type=int, default=65536,
+                        help="Maximum connection cache size")
+    options = parser.parse_args()
+    proxy = TransparentProxy(**vars(options))
+    proxy.start()
+    print(" * Transparent proxy active.")
+    print("   Filter: {0}".format(proxy.request_filter))
+    try:
+        while True:
+            time.sleep(1)
+    except KeyboardInterrupt:
+        print(" * Shutting down...")
+        proxy.shutdown()
+        print(" * Shut down.")