From 78a44c5199d81bc46038b5d680638124f613b871 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 7 Sep 2014 00:42:25 +0200 Subject: add docs on proxy modes --- doc-src/_nav.html | 2 + doc-src/custom-routing.txt | 24 +++ doc-src/dev/architecture.html | 8 + doc-src/dev/index.py | 1 + doc-src/features/upstreamproxy.html | 2 +- doc-src/index.py | 1 + doc-src/modes.html | 210 +++++++++++++++++++++ doc-src/schematics/architecture.png | Bin 0 -> 87365 bytes doc-src/schematics/architecture.vsdx | Bin 60572 -> 60922 bytes .../schematics/proxy-flowchart/proxy-flowchart.pdf | Bin 292625 -> 0 bytes .../proxy-flowchart/proxy-flowchart.vsdx | Bin 218325 -> 0 bytes doc-src/schematics/proxy-modes-flowchart.png | Bin 0 -> 34635 bytes doc-src/schematics/proxy-modes-regular.png | Bin 0 -> 18283 bytes doc-src/schematics/proxy-modes-reverse.png | Bin 0 -> 16719 bytes doc-src/schematics/proxy-modes-transparent-1.png | Bin 0 -> 16305 bytes doc-src/schematics/proxy-modes-transparent-2.png | Bin 0 -> 23041 bytes doc-src/schematics/proxy-modes-transparent-3.png | Bin 0 -> 23855 bytes .../schematics/proxy-modes-transparent-wrong.png | Bin 0 -> 17568 bytes doc-src/schematics/proxy-modes-upstream.png | Bin 0 -> 14781 bytes doc-src/schematics/proxy-modes.pdf | Bin 265000 -> 335485 bytes doc-src/schematics/proxy-modes.vsdx | Bin 160439 -> 195186 bytes doc-src/screenshots/ios-gateway.png | Bin 0 -> 154469 bytes doc-src/screenshots/ios-manual.png | Bin 0 -> 196431 bytes doc-src/screenshots/ios-reverse.png | Bin 0 -> 66150 bytes 24 files changed, 247 insertions(+), 1 deletion(-) create mode 100644 doc-src/custom-routing.txt create mode 100644 doc-src/dev/architecture.html create mode 100644 doc-src/modes.html create mode 100644 doc-src/schematics/architecture.png delete mode 100644 doc-src/schematics/proxy-flowchart/proxy-flowchart.pdf delete mode 100644 doc-src/schematics/proxy-flowchart/proxy-flowchart.vsdx create mode 100644 doc-src/schematics/proxy-modes-flowchart.png create mode 100644 doc-src/schematics/proxy-modes-regular.png create mode 100644 doc-src/schematics/proxy-modes-reverse.png create mode 100644 doc-src/schematics/proxy-modes-transparent-1.png create mode 100644 doc-src/schematics/proxy-modes-transparent-2.png create mode 100644 doc-src/schematics/proxy-modes-transparent-3.png create mode 100644 doc-src/schematics/proxy-modes-transparent-wrong.png create mode 100644 doc-src/schematics/proxy-modes-upstream.png create mode 100644 doc-src/screenshots/ios-gateway.png create mode 100644 doc-src/screenshots/ios-manual.png create mode 100644 doc-src/screenshots/ios-reverse.png (limited to 'doc-src') diff --git a/doc-src/_nav.html b/doc-src/_nav.html index cd70cc25..02ea68e4 100644 --- a/doc-src/_nav.html +++ b/doc-src/_nav.html @@ -2,6 +2,7 @@ $!nav(idxpath, this, state)!$ $!nav("install.html", this, state)!$ $!nav("howmitmproxy.html", this, state)!$ + $!nav("modes.html", this, state)!$
  • Tools
    • $!nav("mitmproxy.html", this, state)!$ @@ -47,5 +48,6 @@ $!nav("tutorials/transparent-dhcp.html", this, state)!$
  • Hacking
    • + $!nav("dev/architecture.html", this, state)!$ $!nav("dev/testing.html", this, state)!$ diff --git a/doc-src/custom-routing.txt b/doc-src/custom-routing.txt new file mode 100644 index 00000000..2ba2281f --- /dev/null +++ b/doc-src/custom-routing.txt @@ -0,0 +1,24 @@ +# Adapted from http://tldp.org/HOWTO/TransparentProxy-6.html (6.2 Second method) +# Note that the choice of firewall mark (3) and routing table (2) was fairly arbitrary. +# If you are already using policy routing or firewall marking for some other purpose, +# make sure you choose unique numbers here. Otherwise, don't worry about it. + + + +# On the router, run + +PROXY_IP=192.168.1.100 +TARGET_IP=192.168.1.110 + +iptables -t mangle -A PREROUTING -j ACCEPT -p tcp -m multiport --dports 80,443 -s ! $TARGET_IP +# Alternative to MITM the whole network: +# iptables -t mangle -A PREROUTING -j ACCEPT -p tcp -m multiport --dports 80,443 -s $PROXY_IP +iptables -t mangle -A PREROUTING -j MARK --set-mark 3 -p tcp -m multiport --dports 80,443 +ip rule add fwmark 3 table 2 +ip route add default via $PROXY_IP dev br0 table 2 + + + +# On the proxy machine, run + +iptables -A PREROUTING -t nat -i eth0 -p tcp -m multiport --dports 80,443 -j REDIRECT --to-port 8080 diff --git a/doc-src/dev/architecture.html b/doc-src/dev/architecture.html new file mode 100644 index 00000000..8ab8821f --- /dev/null +++ b/doc-src/dev/architecture.html @@ -0,0 +1,8 @@ +To give you a better understanding of how mitmproxy works, mitmproxy's high-level architecture is detailed +in the following graphic: + + + +(architecture.pdf) +

    Please don't refrain from asking any further +questions on the mailing list, the IRC channel or the GitHub issue tracker.

    \ No newline at end of file diff --git a/doc-src/dev/index.py b/doc-src/dev/index.py index 001c2b89..bb7872c7 100644 --- a/doc-src/dev/index.py +++ b/doc-src/dev/index.py @@ -2,5 +2,6 @@ from countershape import Page pages = [ Page("testing.html", "Testing"), + Page("architecture.html", "Architecture"), # Page("addingviews.html", "Writing Content Views"), ] diff --git a/doc-src/features/upstreamproxy.html b/doc-src/features/upstreamproxy.html index 5b9c2289..6039f4df 100644 --- a/doc-src/features/upstreamproxy.html +++ b/doc-src/features/upstreamproxy.html @@ -7,7 +7,7 @@ mitmproxy forwards ordinary HTTP requests to an upstream server. - + diff --git a/doc-src/index.py b/doc-src/index.py index f222de14..b7ab9995 100644 --- a/doc-src/index.py +++ b/doc-src/index.py @@ -79,6 +79,7 @@ pages = [ Page("mitmproxy.html", "mitmproxy"), Page("mitmdump.html", "mitmdump"), Page("howmitmproxy.html", "How mitmproxy works"), + Page("modes.html", "Modes of Operation"), Page("ssl.html", "Overview"), Directory("certinstall"), diff --git a/doc-src/modes.html b/doc-src/modes.html new file mode 100644 index 00000000..77bd1b05 --- /dev/null +++ b/doc-src/modes.html @@ -0,0 +1,210 @@ +Mitmproxy comes with several modes of operation, which allow you to use mitmproxy in a variety of scenarios. +This documents briefly explains each mode and possible setups. +
    +Mitmproxy has four modes of operation: + + +

    Now, which one should you pick? Use this flow chart: +

    + +

    + +
    +

    Regular Proxy

    +
    + +Mitmproxy's regular mode it the most simple one and the easiest to set up. + +
      +
    1. Start mitmproxy.
      2. +
    2. Configure your client to use mitmproxy. This means that you either adjust the proxy setting of your local browser + or point an external device to your proxy (which should look like + this).
      3. +
    3. Quick Check: You can already visit an unencrypted HTTP site over the proxy.
      4. +
    4. Open the magic domain mitm.it and install the certificate for your device.
      5. +
    + +
    + Heads Up: Unfortunately, some applications prefer to bypass the HTTP proxy settings of the system - + Android applications are a common example. In these cases, you need to use mitmproxy's transparent mode. +
    + +

    If you are proxying an external device, your network will probably look like this:

    + +

    +

    The square brackets signify the source and destination IP addresses. Your client explicitly connects + to mitmproxy and mitmproxy explicitly connects to the target server. +

    + +
    +

    Transparent Proxy

    +
    + +When a transparent proxy is used, traffic is redirected into a proxy at the network layer, without any client +configuration being required. This makes transparent proxying ideal for those situations where you can't change client +behaviour. The basic principle is that mitmproxy sits somewhere on the line from the client to the internet and +transparently intercepts the request. In the graphic below, a machine running mitmproxy has been inserted between +the router and the internet: + + + +

    The square brackets signify the source and destination IP addresses. Round brackets mark the next + hop on the Ethernet/data link layer. This distinction is important to make: When the packet arrives + at the mitmproxy machine, it must still be addressed to the target server. In other words: A simple IP redirect on + the router does not work - this would remove the target information, leaving mitmproxy unable to + determine the real destination. +

    + + + +

    Common Configurations

    + +The first graphic is a little bit idealistic: Usually, you'll have your local wireless lan network and no +machines between your router and the internet. Fortunately, there are other ways to configure your network: +(a) Configuring the client to use a custom gateway/router/"next hop", (b) Implementing custom routing on the router +or (c) setting up a separate wireless network router which gets proxied. +There are of course other options, but we'll look at these three. In most cases, setting (a) is recommended due to its +ease of use. + +

    (a) Custom Gateway

    + +

    Looking at your local home network, it's clear what happens if you enter "example.com" into your address bar: After you +press enter, your OS sends a packet to your router, which then sends this to your ISP, which then sends it to some +Tier-1 carrier, which then sends it... I think you get the idea. The important part for us is the first step here: +Your machine is configured to use your router as the next hop. Your router certainly doesn't host example.com, but your +machine knows that your router will forward it upstream. On the technical level, your router probably provides a DHCP +server, which instructs all clients to use his address as the Default Gateway for connections that leave the +current subnet (your local network).

    +

    +How does this help us? Here comes our trick: By configuring the client to use our machine as its Gateway, all traffic +will be sent to our machine, which then forwards it to the router. This provides us with the scenario we'd like to have, +namely packets on our doorstep that are addressed for someone else: +

    + + + +Given this concept, we can set up mitmproxy: +
      +
    1. Configure your proxy machine for transparent mode.
      You can find instructions + in the Transparent Proxying section of the mitmproxy docs.
      2. +
    2. Configure your client to use your proxy machine's IP as the default gateway. This setting is usually called + Standard Gateway, Router or something along these lines + (iOS screenshot).
      3. +
    3. Quick Check: You can already visit an unencrypted HTTP site over the proxy.
      4. +
    4. Open the magic domain mitm.it and install the certificate for your device.
      5. +
    + +
    + Troubleshooting Transparent Mode +

    Wrong transparent mode configurations are a frequent source of + error. If it doesn't work for you, try the following things:

    + + If you encounter any other pitfalls that should be listed here, please let us know! +
    + +

    (b) Custom Routing

    + +Custom routing is a fairly advanced setup which we'll only document briefly here. +First and foremost, it usually requires root on your router. The basic idea is to teach your router a custom routing +table that says "for requests from ip X, the proxy machine is the next gateway". + + + + +For this setup, we expect you to have a basic understanding of networking in general. In short, you should get started +with these routing commands. The Troubleshooting part directly above this +section might be helpful for you as well. + +

    (c) Separate Network

    + +Setting up a separate network using a cheap router might be a viable option, too. Such a configuration mostly resembles +the idealistic graphic from the beginning (Variant 1). Take a look at the +Transparently proxify virtual machines tutorial to see how +such a network could be implemented. The troubleshooting section for custom gateways may be helpful for you, too. + + +
    +

    Reverse Proxy

    +
    + +Mitmproxy is usually used with a client that uses the proxy to access the Internet. Using reverse proxy mode, you can +use mitmproxy to represent a server: + + + + +There are various use-cases: + + +

    +Please note that cloning Google by using mitmproxy -R http://google.com/ does not really work +(as in this screenshot). +This may work for the first request, but the HTML remains unchanged: As soon as the user clicks on an non-relative URL +(or downloads a non-relative image resource), they speak with Google directly again. +

    +

    + On another note, mitmproxy either supports an HTTP or an HTTPS upstream server, not both at the same time. You can + simply work around this by spawning a second mitmproxy instance. Each instance listens to one port and talks to one + port. +

    + +
    +

    Upstream Proxy

    +
    + +

    +If you want to add mitmproxy in front of a different proxy appliance, you can use mitmproxy's upstream mode. +In upstream mode, all requests are unconditionally transferred to an upstream proxy or your choice. +

    + + + + +

    +mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy mode. You could in theory chain multiple +mitmproxy instances in a row, but that doesn't make any sense in practice (i.e. outside of our tests). +

    \ No newline at end of file diff --git a/doc-src/schematics/architecture.png b/doc-src/schematics/architecture.png new file mode 100644 index 00000000..67d6c718 Binary files /dev/null and b/doc-src/schematics/architecture.png differ diff --git a/doc-src/schematics/architecture.vsdx b/doc-src/schematics/architecture.vsdx index 9820434a..c4ff13d2 100644 Binary files a/doc-src/schematics/architecture.vsdx and b/doc-src/schematics/architecture.vsdx differ diff --git a/doc-src/schematics/proxy-flowchart/proxy-flowchart.pdf b/doc-src/schematics/proxy-flowchart/proxy-flowchart.pdf deleted file mode 100644 index ae98eea3..00000000 Binary files a/doc-src/schematics/proxy-flowchart/proxy-flowchart.pdf and /dev/null differ diff --git a/doc-src/schematics/proxy-flowchart/proxy-flowchart.vsdx b/doc-src/schematics/proxy-flowchart/proxy-flowchart.vsdx deleted file mode 100644 index 4d75f49f..00000000 Binary files a/doc-src/schematics/proxy-flowchart/proxy-flowchart.vsdx and /dev/null differ diff --git a/doc-src/schematics/proxy-modes-flowchart.png b/doc-src/schematics/proxy-modes-flowchart.png new file mode 100644 index 00000000..716b5ee2 Binary files /dev/null and b/doc-src/schematics/proxy-modes-flowchart.png differ diff --git a/doc-src/schematics/proxy-modes-regular.png b/doc-src/schematics/proxy-modes-regular.png new file mode 100644 index 00000000..95bada08 Binary files /dev/null and b/doc-src/schematics/proxy-modes-regular.png differ diff --git a/doc-src/schematics/proxy-modes-reverse.png b/doc-src/schematics/proxy-modes-reverse.png new file mode 100644 index 00000000..071d3fc8 Binary files /dev/null and b/doc-src/schematics/proxy-modes-reverse.png differ diff --git a/doc-src/schematics/proxy-modes-transparent-1.png b/doc-src/schematics/proxy-modes-transparent-1.png new file mode 100644 index 00000000..c2027432 Binary files /dev/null and b/doc-src/schematics/proxy-modes-transparent-1.png differ diff --git a/doc-src/schematics/proxy-modes-transparent-2.png b/doc-src/schematics/proxy-modes-transparent-2.png new file mode 100644 index 00000000..1129e343 Binary files /dev/null and b/doc-src/schematics/proxy-modes-transparent-2.png differ diff --git a/doc-src/schematics/proxy-modes-transparent-3.png b/doc-src/schematics/proxy-modes-transparent-3.png new file mode 100644 index 00000000..ee26cb4f Binary files /dev/null and b/doc-src/schematics/proxy-modes-transparent-3.png differ diff --git a/doc-src/schematics/proxy-modes-transparent-wrong.png b/doc-src/schematics/proxy-modes-transparent-wrong.png new file mode 100644 index 00000000..6bac491f Binary files /dev/null and b/doc-src/schematics/proxy-modes-transparent-wrong.png differ diff --git a/doc-src/schematics/proxy-modes-upstream.png b/doc-src/schematics/proxy-modes-upstream.png new file mode 100644 index 00000000..d40a6494 Binary files /dev/null and b/doc-src/schematics/proxy-modes-upstream.png differ diff --git a/doc-src/schematics/proxy-modes.pdf b/doc-src/schematics/proxy-modes.pdf index d1cf97eb..f07ea05e 100644 Binary files a/doc-src/schematics/proxy-modes.pdf and b/doc-src/schematics/proxy-modes.pdf differ diff --git a/doc-src/schematics/proxy-modes.vsdx b/doc-src/schematics/proxy-modes.vsdx index ae76f80b..74d425fc 100644 Binary files a/doc-src/schematics/proxy-modes.vsdx and b/doc-src/schematics/proxy-modes.vsdx differ diff --git a/doc-src/screenshots/ios-gateway.png b/doc-src/screenshots/ios-gateway.png new file mode 100644 index 00000000..2489cba3 Binary files /dev/null and b/doc-src/screenshots/ios-gateway.png differ diff --git a/doc-src/screenshots/ios-manual.png b/doc-src/screenshots/ios-manual.png new file mode 100644 index 00000000..3977acfe Binary files /dev/null and b/doc-src/screenshots/ios-manual.png differ diff --git a/doc-src/screenshots/ios-reverse.png b/doc-src/screenshots/ios-reverse.png new file mode 100644 index 00000000..6ab5b7c0 Binary files /dev/null and b/doc-src/screenshots/ios-reverse.png differ -- cgit v1.2.3
    command-line -U http[s]://hostname[:port]command-line -U http://hostname[:port]
    mitmproxy shortcut U