Docs, minor cert tweaks.

16 files changed, 167 insertions, 42 deletions

diff --git a/README.mkd b/README.mkd
index abd52eed..f7138a5d 100644
--- a/README.mkd
+++ b/README.mkd
@@ -1,9 +1,14 @@
-__mitmproxy__ and __mitmdump__ are SSL-capable, intercepting HTTP proxies.
+__mitmproxy__ is an SSL-capable, intercepting HTTP proxy.
 
-__mitmproxy__ provides a curses interface that allows traffic to be inspected
-and edited on the fly.
+__mitmproxy__ is a console application that allows traffic flows to be
+inspected and edited on the fly.
 
-__mitmdump__ is the command-line companion of mitmproxy - like tcpdump for HTTP.
+__mitmdump__ is the command-line version of mitmproxy, with the same
+functionality but without the frills. Think tcpdump for HTTP.
+
+Both tools are fully documentented in the commandline __--help__ flag, and, in
+the case of __mitmproxy__, a built-in help page accessible through the __?__
+keyboard shortcut.
 
 
 Capabilities
@@ -11,8 +16,8 @@ Capabilities
 
 - Intercept HTTP requests and responses and modify them on the fly.
 - Save complete HTTP conversations for later replay and analysis.
-- Client replay to replay the client-side of an HTTP conversation.
-- Server replay replays the HTTP responses of a previously recorded server.
+- Replay the client-side of an HTTP conversations.
+- Replays HTTP responses of a previously recorded server.
 - Make scripted changes to HTTP traffic using a simple Python API. 
 - Dummy SSL certificate authority creates interception certificates on the fly.
 
diff --git a/doc-src/_layout.html b/doc-src/_layout.html
index 11d456a0..6010c95c 100644
--- a/doc-src/_layout.html
+++ b/doc-src/_layout.html
@@ -1,4 +1,4 @@
-<div id="doc3">
+<div id="doc">
     <div style="" id="hd">
 
         <div id="nav">
diff --git a/doc-src/anticache.html b/doc-src/anticache.html
new file mode 100644
index 00000000..ceeffe2e
--- /dev/null
+++ b/doc-src/anticache.html
@@ -0,0 +1,7 @@
+
+When the __anticache__ option is passed to mitmproxy, it removes headers
+(__if-none-match__ and __if-modified-since__) that might elicit a
+304-not-modified response from the server. This is useful when you want to make
+sure you capture an HTTP extchange in its totality, and during [client
+replay](@!urlTo("clientreplay.html")!@).
+
diff --git a/doc-src/certinstall/chrome.html b/doc-src/certinstall/chrome.html
deleted file mode 100644
index e69de29b..00000000
--- a/doc-src/certinstall/chrome.html
+++ /dev/null
diff --git a/doc-src/certinstall/index.py b/doc-src/certinstall/index.py
index 3dee24ec..bf115ff8 100644
--- a/doc-src/certinstall/index.py
+++ b/doc-src/certinstall/index.py
@@ -2,9 +2,9 @@ from countershape import Page
 
 pages = [
     Page("firefox.html", "Firefox"),
-    Page("chrome.html", "Chrome"),
     Page("safari.html", "Safari"),
     Page("ie.html", "Internet Explorer"),
+    Page("osx.html", "OSX"),
     Page("windows7.html", "Windows 7"),
     Page("ios.html", "IOS"),
 ]
diff --git a/doc-src/certinstall/osx.html b/doc-src/certinstall/osx.html
new file mode 100644
index 00000000..bcb72f50
--- /dev/null
+++ b/doc-src/certinstall/osx.html
@@ -0,0 +1,15 @@
+
+### 1: Open Finder, and double-click on the mitmproxy ca.pem file.
+
+
+### 2: You will be prompted to add the certificate. Click "Add": 
+
+<img src="@!urlTo('osx-addcert.png')!@"/>
+
+### 3: Click "Always Trust": 
+
+<img src="@!urlTo('osx-addcert-alwaystrust.png')!@"/>
+
+You may be prompted for your password. You should now see the mitmproxy cert
+listed under "Certificates".
+
diff --git a/doc-src/certinstall/windows7.html b/doc-src/certinstall/windows7.html
index 16d3f611..b8632d56 100644
--- a/doc-src/certinstall/windows7.html
+++ b/doc-src/certinstall/windows7.html
@@ -1,21 +1,40 @@
 
-<p> Here's a quick and easy procedure you can use for Windows 7, as long as
-the app in question uses the global Windows certificate repository. </p>
+The Windows certificate manager expects a different certificate format from the
+one used by mitmproxy. The easiest way to convert the cert to the appropriate
+format is to use the Firefox web browser.
+
+
+### 1: Make sure Firefox is installed on the system.
+
+### 2: Fire up mitmproxy on the interception host. 
+
+### 3: Configure Firefox to use the mitmproxy interceptor.
+
+### 4: Using Firefox, browse to an SSL-protected domain. You will see a warning: 
+
+
+### 5: Click "I understand the risks" and "Add Exception":
+
+
+### 6: Click "Get certificate", "View", and switch to the "Details" tab:
+
+
+### 7: Click "Export", and save the certificate in "X.509 Certificate (PEM)" format:
+
+
+### 8: Next, start a command prompt, and type "certmgr" to start the Certificate Manager:
+
+
+### 9: From the top menu, select "Action", "All tasks", and then "Import":
+
+
+### 10: Click "Next", and browse to select the cert we just exported from Firefox:
+
+
+### 11: Click "Next", and "Finish" to complete the import. Accept all warning prompts.
+
 
-<ul>
 
-    <li> Fire up MITMProxy, and configure Firefox on the Windows box to use
-    it. Browse to the target domain, and you should see a big warning about
-    an untrusted certificate. Use Firefox to export the certificate ("Add
-    Exception", "Get Certificate", then "View", tab to "Details" and click
-    "Export").  </li>
 
-    <li> From the command console, fire up <b>certmgr</b>. Select "Trusted
-    Root Certification Authorities", then on the top menu, "Action", "All
-    Tasks", and "Import". When prompted, select the certificate file you've
-    just saved from Firefox.</li>
 
-    <li> And that's it - your certificate should now be trusted for that
-    domain. Happy pentesting.</li>
 
-</ul>
diff --git a/doc-src/filters.html b/doc-src/filters.html
new file mode 100644
index 00000000..b71207fc
--- /dev/null
+++ b/doc-src/filters.html
@@ -0,0 +1,34 @@
+
+Many commands in __mitmproxy__ and __mitmdump__ take a filter expression.
+Filter expressions consist of the following operators:
+
+<table>
+    <!--(for i in filt_help)-->
+        <tr>
+            <td class="filt_cmd">@!i[0]!@</td>
+            <td class="filt_help">@!i[1]!@</td>
+        </tr>
+    <!--(end)-->
+</table>
+
+- Regexes are Python-style
+- Regexes can be specified as quoted strings
+- Header matching (~h, ~hq, ~hs) is against a string of the form "name: value".
+- Strings with no operators are matched against the request URL.
+- The default binary operator is &amp;.
+
+Examples
+========
+
+Url containing "google.com":
+
+    google\.com
+
+Requests whose body contains the string "test":
+
+    ~r ~b test
+
+Anything but requests with a text/html content type:
+
+    !(~r & ~t \"text/html\")
+
diff --git a/doc-src/index.html b/doc-src/index.html
index c2a6a1c2..049528a6 100644
--- a/doc-src/index.html
+++ b/doc-src/index.html
@@ -5,19 +5,19 @@
     * [Client-side replay](@!urlTo("clientreplay.html")!@)
     * [Server-side replay](@!urlTo("serverreplay.html")!@)
     * [Sticky cookies](@!urlTo("stickycookies.html")!@)
+    * [Anticache](@!urlTo("anticache.html")!@)
+    * [Filter expressions](@!urlTo("filters.html")!@)
     * [Scripting API](@!urlTo("scripts.html")!@)
-    * [Anticache](@!urlTo("scripts.html")!@)
-    * [Filter expressions](@!urlTo("scripts.html")!@)
 * SSL
     * [Overview](@!urlTo("/ssl.html")!@)
     * Browser certificate installation: 
-        * [Firefox](@!urlTo("certinstall/firefox.html")!@#firefox)
-        * [Chrome](@!urlTo("certinstall/chrome.html")!@#chrome)
-        * [Safari](@!urlTo("certinstall/safari.html")!@#safari)
-        * [Internet Explorer](@!urlTo("certinstall/ie.html")!@#ie)
+        * [Firefox](@!urlTo("certinstall/firefox.html")!@)
+        * [Safari](@!urlTo("certinstall/safari.html")!@)
+        * [Internet Explorer](@!urlTo("certinstall/ie.html")!@)
     * System-wide certificate installation:
-        * [Windows 7](@!urlTo("certinstall/windows7.html")!@#windows7)
-        * [iPhone/iPad](@!urlTo("certinstall/ios.html")!@#ios)
+        * [OSX](@!urlTo("certinstall/osx.html")!@)
+        * [Windows 7](@!urlTo("certinstall/windows7.html")!@)
+        * [iPhone/iPad](@!urlTo("certinstall/ios.html")!@)
 * [libmproxy](@!urlTo("library.html")!@)
 * Tutorials
     * Client replay: a 30 second example
diff --git a/doc-src/index.py b/doc-src/index.py
index b51a2a3c..c2d995a5 100644
--- a/doc-src/index.py
+++ b/doc-src/index.py
@@ -1,7 +1,9 @@
-import os
+import os, sys
 import countershape
 from countershape import Page, Directory, PythonModule, markup
 import countershape.grok, countershape.template    
+sys.path.insert(0, "..")
+from libmproxy import filt
 
 this.layout = countershape.Layout("_layout.html")
 ns.docTitle = "mitmproxy"
@@ -25,6 +27,30 @@ def example(s):
 
 ns.example = example
 
+filt_help = []
+for i in filt.filt_unary:
+    filt_help.append(
+        ("~%s"%i.code, i.help)
+    )
+for i in filt.filt_rex:
+    filt_help.append(
+        ("~%s regex"%i.code, i.help)
+    )
+for i in filt.filt_int:
+    filt_help.append(
+        ("~%s int"%i.code, i.help)
+    )
+filt_help.sort()
+filt_help.extend(
+    [
+        ("!", "unary not"),
+        ("&", "and"),
+        ("|", "or"),
+        ("(...)", "grouping"),
+    ]
+)
+ns.filt_help = filt_help
+ 
 
 
 pages = [
@@ -33,6 +59,8 @@ pages = [
     Page("clientreplay.html", "Client-side replay"),
     Page("serverreplay.html", "Server-side replay"),
     Page("stickycookies.html", "Sticky cookies"),
+    Page("anticache.html", "Anticache"),
+    Page("filters.html", "Filter expressions"),
     Page("scripts.html", "External scripts"),
     Page("library.html", "libmproxy: mitmproxy as a library"),
     Page("ssl.html", "SSL"),
diff --git a/doc-src/screenshots/osx-addcert-alwaystrust.png b/doc-src/screenshots/osx-addcert-alwaystrust.png
new file mode 100644
index 00000000..ef0155e8
--- /dev/null
+++ b/doc-src/screenshots/osx-addcert-alwaystrust.png
diff --git a/doc-src/screenshots/osx-addcert.png b/doc-src/screenshots/osx-addcert.png
new file mode 100644
index 00000000..16dd0284
--- /dev/null
+++ b/doc-src/screenshots/osx-addcert.png
diff --git a/doc-src/ssl.html b/doc-src/ssl.html
index 06d5a9d6..eb68dc95 100644
--- a/doc-src/ssl.html
+++ b/doc-src/ssl.html
@@ -23,12 +23,12 @@ exceptions.
 Installing the mitmproxy CA
 ===========================
 
+
 * Browser certificate installation: 
-    * [Firefox](@!urlTo("certinstall/firefox.html")!@#firefox)
-    * [Chrome](@!urlTo("certinstall/chrome.html")!@#chrome)
-    * [Safari](@!urlTo("certinstall/safari.html")!@#safari)
-    * [Internet Explorer](@!urlTo("certinstall/ie.html")!@#ie)
+    * [Firefox](@!urlTo("certinstall/firefox.html")!@)
+    * [Safari](@!urlTo("certinstall/safari.html")!@)
+    * [Internet Explorer](@!urlTo("certinstall/ie.html")!@)
 * System-wide certificate installation:
-    * [Windows 7](@!urlTo("certinstall/windows7.html")!@#windows7)
-    * [iPhone/iPad](@!urlTo("certinstall/ios.html")!@#ios)
-
+    * [OSX](@!urlTo("certinstall/osx.html")!@)
+    * [Windows 7](@!urlTo("certinstall/windows7.html")!@)
+    * [iPhone/iPad](@!urlTo("certinstall/ios.html")!@)
diff --git a/doc-src/stickycookies.html b/doc-src/stickycookies.html
index b28b04f6..5aa5045c 100644
--- a/doc-src/stickycookies.html
+++ b/doc-src/stickycookies.html
@@ -1,3 +1,17 @@
 
+When the __stickycookie__ option is set, __mitmproxy__ will add the cookie most
+recently set by the server to any cookie-less request. Consider a service that
+sets a cookie to track the session after authentication. Using sticky cookies,
+you can fire up mitmproxy, and authenticate to a service as you usually would
+using a browser. After authentication, you can request authenticated resources
+through mitmproxy as if they were unauthenticated, because mitmproxy will
+automatically add the session tracking cookie to requests. Among other things,
+this lets you script interactions with authenticated resources (using tools
+like wget or curl) without having to worry about authentication. 
+
+Sticky cookies are especially powerful when used in conjunction with [client
+replay](@!urlTo("clientreplay.html")!@) - you can record the authentication
+process once, and simply replay it on startup every time you need to interact
+with the secured resources.
 
 
diff --git a/libmproxy/resources/ca.cnf b/libmproxy/resources/ca.cnf
index e46bb08f..c65c66c8 100644
--- a/libmproxy/resources/ca.cnf
+++ b/libmproxy/resources/ca.cnf
@@ -5,24 +5,27 @@ x509_extensions = v3_ca
 req_extensions = v3_ca_req
 
 [ req_distinguished_name ]
-organizationName               = mitmproxy
-commonName                     = Dummy CA
+organizationName                = mitmproxy
+commonName                      = mitmproxy
 
 [ v3_ca ]
 subjectKeyIdentifier=hash
 authorityKeyIdentifier=keyid:always,issuer
 basicConstraints = critical,CA:true
 keyUsage = cRLSign, keyCertSign
+extendedKeyUsage=serverAuth,clientAuth,emailProtection,timeStamping,msCodeInd,msCodeCom,msCTLSign,msSGC,msEFS,nsSGC
 nsCertType = sslCA
 
 [ v3_ca_req ]
 basicConstraints = critical,CA:true
 keyUsage = cRLSign, keyCertSign
+extendedKeyUsage=serverAuth,clientAuth,emailProtection,timeStamping,msCodeInd,msCodeCom,msCTLSign,msSGC,msEFS,nsSGC
 nsCertType = sslCA
 
 [ v3_cert ]
 basicConstraints = CA:false
 keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage=serverAuth,clientAuth,emailProtection,timeStamping,msCodeInd,msCodeCom,msCTLSign,msSGC,msEFS,nsSGC
 nsCertType = server
 subjectKeyIdentifier=hash
 authorityKeyIdentifier=keyid:always,issuer
@@ -30,4 +33,5 @@ authorityKeyIdentifier=keyid:always,issuer
 [ v3_cert_req ]
 basicConstraints = CA:false
 keyUsage = nonRepudiation, digitalSignature, keyEncipherment
+extendedKeyUsage=serverAuth,clientAuth,emailProtection,timeStamping,msCodeInd,msCodeCom,msCTLSign,msSGC,msEFS,nsSGC
 nsCertType = server
diff --git a/libmproxy/utils.py b/libmproxy/utils.py
index 34c49e14..f4df813f 100644
--- a/libmproxy/utils.py
+++ b/libmproxy/utils.py
@@ -411,7 +411,6 @@ def dummy_cert(certdir, ca, commonname):
             "-days", "9999",
             "-out", certpath,
             "-CA", ca,
-            "-CAcreateserial",
             "-extfile", confpath,
             "-extensions", "v3_cert",
         ]