diff options
70 files changed, 1992 insertions, 1 deletions
diff --git a/docs/.gitignore b/docs/.gitignore new file mode 100644 index 00000000..69fa449d --- /dev/null +++ b/docs/.gitignore @@ -0,0 +1 @@ +_build/ diff --git a/docs/Makefile b/docs/Makefile new file mode 100644 index 00000000..a22bc8a2 --- /dev/null +++ b/docs/Makefile @@ -0,0 +1,195 @@ +# Makefile for Sphinx documentation +# + +# You can set these variables from the command line. +SPHINXOPTS = +SPHINXBUILD = sphinx-build +PAPER = +BUILDDIR = _build + +# User-friendly check for sphinx-build +ifeq ($(shell which $(SPHINXBUILD) >/dev/null 2>&1; echo $$?), 1) +$(error The '$(SPHINXBUILD)' command was not found. Make sure you have Sphinx installed, then set the SPHINXBUILD environment variable to point to the full path of the '$(SPHINXBUILD)' executable. Alternatively you can add the directory with the executable to your PATH. If you don't have Sphinx installed, grab it from http://sphinx-doc.org/) +endif + +# Internal variables. +PAPEROPT_a4 = -D latex_paper_size=a4 +PAPEROPT_letter = -D latex_paper_size=letter +ALLSPHINXOPTS = -d $(BUILDDIR)/doctrees $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . +# the i18n builder cannot share the environment and doctrees with the others +I18NSPHINXOPTS = $(PAPEROPT_$(PAPER)) $(SPHINXOPTS) . + +.PHONY: help clean html dirhtml singlehtml pickle json htmlhelp qthelp devhelp epub latex latexpdf text man changes linkcheck doctest coverage gettext + +help: + @echo "Please use \`make <target>' where <target> is one of" + @echo " html to make standalone HTML files" + @echo " dirhtml to make HTML files named index.html in directories" + @echo " singlehtml to make a single large HTML file" + @echo " pickle to make pickle files" + @echo " json to make JSON files" + @echo " htmlhelp to make HTML files and a HTML help project" + @echo " qthelp to make HTML files and a qthelp project" + @echo " applehelp to make an Apple Help Book" + @echo " devhelp to make HTML files and a Devhelp project" + @echo " epub to make an epub" + @echo " latex to make LaTeX files, you can set PAPER=a4 or PAPER=letter" + @echo " latexpdf to make LaTeX files and run them through pdflatex" + @echo " latexpdfja to make LaTeX files and run them through platex/dvipdfmx" + @echo " text to make text files" + @echo " man to make manual pages" + @echo " texinfo to make Texinfo files" + @echo " info to make Texinfo files and run them through makeinfo" + @echo " gettext to make PO message catalogs" + @echo " changes to make an overview of all changed/added/deprecated items" + @echo " xml to make Docutils-native XML files" + @echo " pseudoxml to make pseudoxml-XML files for display purposes" + @echo " linkcheck to check all external links for integrity" + @echo " doctest to run all doctests embedded in the documentation (if enabled)" + @echo " coverage to run coverage check of the documentation (if enabled)" + +clean: + rm -rf $(BUILDDIR)/* + +html: + $(SPHINXBUILD) -b html $(ALLSPHINXOPTS) $(BUILDDIR)/html + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/html." + +dirhtml: + $(SPHINXBUILD) -b dirhtml $(ALLSPHINXOPTS) $(BUILDDIR)/dirhtml + @echo + @echo "Build finished. The HTML pages are in $(BUILDDIR)/dirhtml." + +singlehtml: + $(SPHINXBUILD) -b singlehtml $(ALLSPHINXOPTS) $(BUILDDIR)/singlehtml + @echo + @echo "Build finished. The HTML page is in $(BUILDDIR)/singlehtml." + +pickle: + $(SPHINXBUILD) -b pickle $(ALLSPHINXOPTS) $(BUILDDIR)/pickle + @echo + @echo "Build finished; now you can process the pickle files." + +json: + $(SPHINXBUILD) -b json $(ALLSPHINXOPTS) $(BUILDDIR)/json + @echo + @echo "Build finished; now you can process the JSON files." + +htmlhelp: + $(SPHINXBUILD) -b htmlhelp $(ALLSPHINXOPTS) $(BUILDDIR)/htmlhelp + @echo + @echo "Build finished; now you can run HTML Help Workshop with the" \ + ".hhp project file in $(BUILDDIR)/htmlhelp." + +qthelp: + $(SPHINXBUILD) -b qthelp $(ALLSPHINXOPTS) $(BUILDDIR)/qthelp + @echo + @echo "Build finished; now you can run "qcollectiongenerator" with the" \ + ".qhcp project file in $(BUILDDIR)/qthelp, like this:" + @echo "# qcollectiongenerator $(BUILDDIR)/qthelp/mitmproxy.qhcp" + @echo "To view the help file:" + @echo "# assistant -collectionFile $(BUILDDIR)/qthelp/mitmproxy.qhc" + +applehelp: + $(SPHINXBUILD) -b applehelp $(ALLSPHINXOPTS) $(BUILDDIR)/applehelp + @echo + @echo "Build finished. The help book is in $(BUILDDIR)/applehelp." + @echo "N.B. You won't be able to view it unless you put it in" \ + "~/Library/Documentation/Help or install it in your application" \ + "bundle." + +devhelp: + $(SPHINXBUILD) -b devhelp $(ALLSPHINXOPTS) $(BUILDDIR)/devhelp + @echo + @echo "Build finished." + @echo "To view the help file:" + @echo "# mkdir -p $$HOME/.local/share/devhelp/mitmproxy" + @echo "# ln -s $(BUILDDIR)/devhelp $$HOME/.local/share/devhelp/mitmproxy" + @echo "# devhelp" + +epub: + $(SPHINXBUILD) -b epub $(ALLSPHINXOPTS) $(BUILDDIR)/epub + @echo + @echo "Build finished. The epub file is in $(BUILDDIR)/epub." + +latex: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo + @echo "Build finished; the LaTeX files are in $(BUILDDIR)/latex." + @echo "Run \`make' in that directory to run these through (pdf)latex" \ + "(use \`make latexpdf' here to do that automatically)." + +latexpdf: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through pdflatex..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +latexpdfja: + $(SPHINXBUILD) -b latex $(ALLSPHINXOPTS) $(BUILDDIR)/latex + @echo "Running LaTeX files through platex and dvipdfmx..." + $(MAKE) -C $(BUILDDIR)/latex all-pdf-ja + @echo "pdflatex finished; the PDF files are in $(BUILDDIR)/latex." + +text: + $(SPHINXBUILD) -b text $(ALLSPHINXOPTS) $(BUILDDIR)/text + @echo + @echo "Build finished. The text files are in $(BUILDDIR)/text." + +man: + $(SPHINXBUILD) -b man $(ALLSPHINXOPTS) $(BUILDDIR)/man + @echo + @echo "Build finished. The manual pages are in $(BUILDDIR)/man." + +texinfo: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo + @echo "Build finished. The Texinfo files are in $(BUILDDIR)/texinfo." + @echo "Run \`make' in that directory to run these through makeinfo" \ + "(use \`make info' here to do that automatically)." + +info: + $(SPHINXBUILD) -b texinfo $(ALLSPHINXOPTS) $(BUILDDIR)/texinfo + @echo "Running Texinfo files through makeinfo..." + make -C $(BUILDDIR)/texinfo info + @echo "makeinfo finished; the Info files are in $(BUILDDIR)/texinfo." + +gettext: + $(SPHINXBUILD) -b gettext $(I18NSPHINXOPTS) $(BUILDDIR)/locale + @echo + @echo "Build finished. The message catalogs are in $(BUILDDIR)/locale." + +changes: + $(SPHINXBUILD) -b changes $(ALLSPHINXOPTS) $(BUILDDIR)/changes + @echo + @echo "The overview file is in $(BUILDDIR)/changes." + +linkcheck: + $(SPHINXBUILD) -b linkcheck $(ALLSPHINXOPTS) $(BUILDDIR)/linkcheck + @echo + @echo "Link check complete; look for any errors in the above output " \ + "or in $(BUILDDIR)/linkcheck/output.txt." + +doctest: + $(SPHINXBUILD) -b doctest $(ALLSPHINXOPTS) $(BUILDDIR)/doctest + @echo "Testing of doctests in the sources finished, look at the " \ + "results in $(BUILDDIR)/doctest/output.txt." + +coverage: + $(SPHINXBUILD) -b coverage $(ALLSPHINXOPTS) $(BUILDDIR)/coverage + @echo "Testing of coverage in the sources finished, look at the " \ + "results in $(BUILDDIR)/coverage/python.txt." + +xml: + $(SPHINXBUILD) -b xml $(ALLSPHINXOPTS) $(BUILDDIR)/xml + @echo + @echo "Build finished. The XML files are in $(BUILDDIR)/xml." + +pseudoxml: + $(SPHINXBUILD) -b pseudoxml $(ALLSPHINXOPTS) $(BUILDDIR)/pseudoxml + @echo + @echo "Build finished. The pseudo-XML files are in $(BUILDDIR)/pseudoxml." + +livehtml: + sphinx-autobuild -b html -z '../libmproxy' -r '___jb_(old|bak)___$$' $(ALLSPHINXOPTS) $(BUILDDIR)/html
\ No newline at end of file diff --git a/docs/certinstall-webapp.png b/docs/certinstall-webapp.png Binary files differnew file mode 100644 index 00000000..10e795cd --- /dev/null +++ b/docs/certinstall-webapp.png diff --git a/docs/certinstall.rst b/docs/certinstall.rst new file mode 100644 index 00000000..f0e71223 --- /dev/null +++ b/docs/certinstall.rst @@ -0,0 +1,173 @@ +.. _certinstall: + +About Certificates +================== + +Introduction +------------ + +Mitmproxy can decrypt encrypted traffic on the fly, as long as the client +trusts its built-in certificate authority. Usually this means that the +mitmproxy CA certificates have to be installed on the client device. + +Quick Setup +----------- + +By far the easiest way to install the mitmproxy certificates is to use the +built-in certificate installation app. To do this, just start mitmproxy and +configure your target device with the correct proxy settings. Now start a +browser on the device, and visit the magic domain **mitm.it**. You should see +something like this: + +.. image:: certinstall-webapp.png + +Click on the relevant icon, follow the setup instructions for the platform +you're on and you are good to go. + + +Installing the mitmproxy CA certificate manually +------------------------------------------------ + +Sometimes using the quick install app is not an option - Java or the iOS +Simulator spring to mind - or you just need to do it manually for some other +reason. Below is a list of pointers to manual certificate installation +documentation for some common platforms. + +The mitmproxy CA cert is located in ``~/.mitmproxy`` after it has been generated at the first +start of mitmproxy. + + +iOS +^^^ + +http://kb.mit.edu/confluence/pages/viewpage.action?pageId=152600377 + +iOS Simulator +^^^^^^^^^^^^^ + +See https://github.com/ADVTOOLS/ADVTrustStore#how-to-use-advtruststore + +Java +^^^^ + +See http://docs.oracle.com/cd/E19906-01/820-4916/geygn/index.html + +Android/Android Simulator +^^^^^^^^^^^^^^^^^^^^^^^^^ + +See http://wiki.cacert.org/FAQ/ImportRootCert#Android_Phones_.26_Tablets + +Windows +^^^^^^^ + +See http://windows.microsoft.com/en-ca/windows/import-export-certificates-private-keys#1TC=windows-7 + +Windows (automated) +^^^^^^^^^^^^^^^^^^^ + +>>> certutil.exe -importpfx mitmproxy-ca-cert.p12 + +See also: https://technet.microsoft.com/en-us/library/cc732443.aspx + +Mac OS X +^^^^^^^^ + +See https://support.apple.com/kb/PH7297?locale=en_US + +Ubuntu/Debian +^^^^^^^^^^^^^ + +See http://askubuntu.com/questions/73287/how-do-i-install-a-root-certificate/94861#94861 + +Mozilla Firefox +^^^^^^^^^^^^^^^ + +See https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox + +Chrome on Linux +^^^^^^^^^^^^^^^ + +See https://code.google.com/p/chromium/wiki/LinuxCertManagement + + +More on mitmproxy certificates +------------------------------ + +The first time **mitmproxy** or **mitmdump** is run, the mitmproxy Certificate +Authority (CA) is created in the config directory (``~/.mitmproxy`` by default). +This CA is used for on-the-fly generation of dummy certificates for each of the +SSL sites that your client visits. Since your browser won't trust the +mitmproxy CA out of the box , you will see an SSL certificate warning every +time you visit a new SSL domain through mitmproxy. When you are testing a +single site through a browser, just accepting the bogus SSL cert manually is +not too much trouble, but there are a many circumstances where you will want to +configure your testing system or browser to trust the mitmproxy CA as a +signing root authority. For security reasons, the mitmproxy CA is generated uniquely on the first +start and is not shared between mitmproxy installations on different devices. + + +CA and cert files +----------------- + +The files created by mitmproxy in the .mitmproxy directory are as follows: + +===================== ==================================================================================== +mitmproxy-ca.pem The certificate **and the private key** in PEM format. +mitmproxy-ca-cert.pem The certificate in PEM format. Use this to distribute on most non-Windows platforms. +mitmproxy-ca-cert.p12 The certificate in PKCS12 format. For use on Windows. +mitmproxy-ca-cert.cer Same file as .pem, but with an extension expected by some Android devices. +===================== ==================================================================================== + +Using a custom certificate +-------------------------- + +You can use your own certificate by passing the ``--cert`` option to +mitmproxy. Mitmproxy then uses the provided certificate for interception of the +specified domains instead of generating a certificate signed by its own CA. + +The certificate file is expected to be in the PEM format. You can include +intermediary certificates right below your leaf certificate, so that you PEM +file roughly looks like this: + +.. code-block:: none + + -----BEGIN PRIVATE KEY----- + <private key> + -----END PRIVATE KEY----- + -----BEGIN CERTIFICATE----- + <cert> + -----END CERTIFICATE----- + -----BEGIN CERTIFICATE----- + <intermediary cert (optional)> + -----END CERTIFICATE----- + + +For example, you can generate a certificate in this format using these instructions: + + +>>> openssl genrsa -out cert.key 2048 +>>> openssl req -new -x509 -key cert.key -out cert.crt + (Specify the mitm domain as Common Name, e.g. *.google.com) +>>> cat cert.key cert.crt > cert.pem +>>> mitmproxy --cert=cert.pem + + +Using a custom certificate authority +------------------------------------ + +By default, mitmproxy will use ``~/.mitmproxy/mitmproxy-ca.pem`` as +the certificate authority to generate certificates for all domains for which no +custom certificate is provided (see above). You can use your own certificate +authority by passing the ``--confdir`` option to mitmproxy. Mitmproxy +will then look for ``mitmproxy-ca.pem`` in the specified directory. If +no such file exists, it will be generated automatically. + + +Using a client side certificate +------------------------------- + +You can use a client certificate by passing the ``--client-certs DIRECTORY`` +option to mitmproxy. If you visit example.org, mitmproxy looks for a file named ``example.org.pem`` +in the specified directory and uses this as the client cert. The certificate file needs to be in +the PEM format and should contain both the unencrypted private key and the certificate. + diff --git a/docs/conf.py b/docs/conf.py new file mode 100644 index 00000000..1e686007 --- /dev/null +++ b/docs/conf.py @@ -0,0 +1,216 @@ +# -*- coding: utf-8 -*- +# +# mitmproxy documentation build configuration file, created by +# sphinx-quickstart on Thu Sep 03 14:04:13 2015. +# +# This file is execfile()d with the current directory set to its +# containing dir. +# +# Note that not all possible configuration values are present in this +# autogenerated file. +# +# All configuration values have a default; values that are commented out +# serve to show the default. + +import sys +import os +import shlex + +# If extensions (or modules to document with autodoc) are in another directory, +# add these directories to sys.path here. If the directory is relative to the +# documentation root, use os.path.abspath to make it absolute, like shown here. +sys.path.insert(0, os.path.abspath('..')) + +import libmproxy.version + +# -- General configuration ------------------------------------------------ + +# If your documentation needs a minimal Sphinx version, state it here. +#needs_sphinx = '1.0' + +# Add any Sphinx extension module names here, as strings. They can be +# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom +# ones. + +extensions = [ + 'sphinx.ext.autodoc', + 'sphinx.ext.doctest', + 'sphinx.ext.viewcode', + 'sphinx.ext.napoleon', + 'sphinxcontrib.documentedlist' +] + +autodoc_member_order = "bysource" + +# Add any paths that contain templates here, relative to this directory. +#templates_path = ['_templates'] + +# The suffix(es) of source filenames. +# You can specify multiple suffix as a list of string: +# source_suffix = ['.rst', '.md'] +source_suffix = '.rst' + +# The encoding of source files. +#source_encoding = 'utf-8-sig' + +# The master toctree document. +master_doc = 'index' + +# General information about the project. +project = u'mitmproxy docs' +copyright = u'2015, the mitmproxy project' +author = u'The mitmproxy project' + +# The version info for the project you're documenting, acts as replacement for +# |version| and |release|, also used in various other places throughout the +# built documents. +# +# The short X.Y version. +version = libmproxy.version.VERSION +# The full version, including alpha/beta/rc tags. +release = libmproxy.version.VERSION + +# The language for content autogenerated by Sphinx. Refer to documentation +# for a list of supported languages. +# +# This is also used if you do content translation via gettext catalogs. +# Usually you set "language" from the command line for these cases. +language = None + +# There are two options for replacing |today|: either, you set today to some +# non-false value, then it is used: +#today = '' +# Else, today_fmt is used as the format for a strftime call. +#today_fmt = '%B %d, %Y' + +# List of patterns, relative to source directory, that match files and +# directories to ignore when looking for source files. +exclude_patterns = ['_build'] + +# The reST default role (used for this markup: `text`) to use for all +# documents. +#default_role = None + +# If true, '()' will be appended to :func: etc. cross-reference text. +#add_function_parentheses = True + +# If true, the current module name will be prepended to all description +# unit titles (such as .. function::). +#add_module_names = True + +# If true, sectionauthor and moduleauthor directives will be shown in the +# output. They are ignored by default. +#show_authors = False + +# The name of the Pygments (syntax highlighting) style to use. +pygments_style = 'sphinx' + +# A list of ignored prefixes for module index sorting. +modindex_common_prefix = ['libmproxy.'] + +# If true, keep warnings as "system message" paragraphs in the built documents. +#keep_warnings = False + +# If true, `todo` and `todoList` produce output, else they produce nothing. +todo_include_todos = False + + +# -- Options for HTML output ---------------------------------------------- + +# The theme to use for HTML and HTML Help pages. See the documentation for +# a list of builtin themes. +html_theme = 'sphinx_rtd_theme' + +# Theme options are theme-specific and customize the look and feel of a theme +# further. For a list of options available for each theme, see the +# documentation. +html_theme_options = { + # 'logo_only': True, +} + +# Add any paths that contain custom themes here, relative to this directory. +#html_theme_path = [] + +# The name for this set of Sphinx documents. If None, it defaults to +# "<project> v<release> documentation". +html_title = "mitmproxy %s documentation" % version + +# A shorter title for the navigation bar. Default is the same as html_title. +#html_short_title = None + +# The name of an image file (relative to this directory) to place at the top +# of the sidebar. +html_logo = "mitmproxy-long.png" + +# The name of an image file (within the static path) to use as favicon of the +# docs. This file should be a Windows icon file (.ico) being 16x16 or 32x32 +# pixels large. +html_favicon = "favicon.ico" + +# Add any paths that contain custom static files (such as style sheets) here, +# relative to this directory. They are copied after the builtin static files, +# so a file named "default.css" will overwrite the builtin "default.css". +# html_static_path = ['_static'] + +# Add any extra paths that contain custom files (such as robots.txt or +# .htaccess) here, relative to this directory. These files are copied +# directly to the root of the documentation. +#html_extra_path = [] + +# If not '', a 'Last updated on:' timestamp is inserted at every page bottom, +# using the given strftime format. +#html_last_updated_fmt = '%b %d, %Y' + +# If true, SmartyPants will be used to convert quotes and dashes to +# typographically correct entities. +#html_use_smartypants = True + +# Custom sidebar templates, maps document names to template names. +#html_sidebars = {} + +# Additional templates that should be rendered to pages, maps page names to +# template names. +#html_additional_pages = {} + +# If false, no module index is generated. +#html_domain_indices = True + +# If false, no index is generated. +#html_use_index = True + +# If true, the index is split into individual pages for each letter. +#html_split_index = False + +# If true, links to the reST sources are added to the pages. +#html_show_sourcelink = True + +# If true, "Created using Sphinx" is shown in the HTML footer. Default is True. +#html_show_sphinx = True + +# If true, "(C) Copyright ..." is shown in the HTML footer. Default is True. +#html_show_copyright = True + +# If true, an OpenSearch description file will be output, and all pages will +# contain a <link> tag referring to it. The value of this option must be the +# base URL from which the finished HTML is served. +#html_use_opensearch = '' + +# This is the file name suffix for HTML files (e.g. ".xhtml"). +#html_file_suffix = None + +# Language to be used for generating the HTML full-text search index. +# Sphinx supports the following languages: +# 'da', 'de', 'en', 'es', 'fi', 'fr', 'hu', 'it', 'ja' +# 'nl', 'no', 'pt', 'ro', 'ru', 'sv', 'tr' +#html_search_language = 'en' + +# A dictionary with options for the search language support, empty by default. +# Now only 'ja' uses this config value +#html_search_options = {'type': 'default'} + +# The name of a javascript file (relative to the configuration directory) that +# implements a search results scorer. If empty, the default will be used. +#html_search_scorer = 'scorer.js' + +# Output file base name for HTML help builder. +htmlhelp_basename = 'mitmproxydoc'
\ No newline at end of file diff --git a/docs/config.rst b/docs/config.rst new file mode 100644 index 00000000..345207ab --- /dev/null +++ b/docs/config.rst @@ -0,0 +1,85 @@ +.. _config: + +Configuration +============= + +Mitmproxy is configured through a set of files in the users ~/.mitmproxy +directory. + +mitmproxy.conf + Settings for the :program:`mitmproxy`. This file can contain any options supported by mitmproxy. + +mitmdump.conf + Settings for the :program:`mitmdump`. This file can contain any options supported by mitmdump. + +common.conf + Settings shared between all command-line tools. Settings in this file are over-ridden by those + in the tool-specific files. Only options shared by mitmproxy and mitmdump should be used in this + file. + +Syntax +------ + +Comments +^^^^^^^^ + +.. code-block:: none + + # this is a comment + ; this is also a comment (.ini style) + --- and this is a comment too (yaml style) + +Key/Value pairs +^^^^^^^^^^^^^^^ + +- Keys and values are case-sensitive +- Whitespace is ignored +- Lists are comma-delimited, and enclosed in square brackets + +.. code-block:: none + + name = value # (.ini style) + name: value # (yaml style) + --name value # (command-line option style) + + fruit = [apple, orange, lemon] + indexes = [1, 12, 35 , 40] + +Flags +^^^^^ + +These are boolean options that take no value but true/false. + +.. code-block:: none + + name = true # (.ini style) + name + --name # (command-line option style) + +Options +------- + +The options available in the config files are precisely those available as +command-line flags, with the key being the option's long name. To get a +complete list of these, use the :option:`--help` option on each of the tools. Be +careful to only specify common options in the **common.conf** file - +unsupported options in this file will be detected as an error on startup. + +Examples +-------- + +common.conf +^^^^^^^^^^^ + +Note that :option:`--port` is an option supported by all tools. + +.. code-block:: none + + port = 8080 + +mitmproxy.conf +^^^^^^^^^^^^^^ + +.. code-block:: none + + palette = light diff --git a/docs/dev/exceptions.rst b/docs/dev/exceptions.rst new file mode 100644 index 00000000..d1e4bfe5 --- /dev/null +++ b/docs/dev/exceptions.rst @@ -0,0 +1,9 @@ +.. _exceptions: + +Exceptions +========== + +.. automodule:: libmproxy.exceptions + :show-inheritance: + :members: + :undoc-members:
\ No newline at end of file diff --git a/docs/dev/models.rst b/docs/dev/models.rst new file mode 100644 index 00000000..850d89f5 --- /dev/null +++ b/docs/dev/models.rst @@ -0,0 +1,25 @@ +.. _models: + +Models +====== + +.. warning:: + The documentation for models has not been converted to rst yet and **many attributes/features + are missing**. + Please read the source code instead. + +.. automodule:: libmproxy.models + :show-inheritance: + :members: + :undoc-members: + + +.. automodule:: netlib.http.semantics + :members: Request, Response + :undoc-members: + + .. autoclass:: Headers + :show-inheritance: + :members: + :special-members: + :no-undoc-members:
\ No newline at end of file diff --git a/docs/dev/protocols.rst b/docs/dev/protocols.rst new file mode 100644 index 00000000..498f634d --- /dev/null +++ b/docs/dev/protocols.rst @@ -0,0 +1,15 @@ +.. _protocols: + +Protocols +========= + +.. automodule:: libmproxy.protocol + + .. autoclass:: Layer + :members: + :special-members: + + .. autoclass:: ServerConnectionMixin + :members: + + .. autoexception:: Kill
\ No newline at end of file diff --git a/docs/dev/proxy.rst b/docs/dev/proxy.rst new file mode 100644 index 00000000..c0cdb259 --- /dev/null +++ b/docs/dev/proxy.rst @@ -0,0 +1,12 @@ +.. _proxy: + +Proxy Server +============ + +.. automodule:: libmproxy.proxy + + .. autoclass:: ProxyServer + .. autoclass:: DummyServer + .. autoclass:: ProxyConfig + .. autoclass:: RootContext + :members:
\ No newline at end of file diff --git a/docs/favicon.ico b/docs/favicon.ico Binary files differnew file mode 100644 index 00000000..3c3b891c --- /dev/null +++ b/docs/favicon.ico diff --git a/docs/features/anticache.rst b/docs/features/anticache.rst new file mode 100644 index 00000000..5244587a --- /dev/null +++ b/docs/features/anticache.rst @@ -0,0 +1,15 @@ +.. _anticache: + +Anticache +========= +When the :option:`--anticache` option is passed to mitmproxy, it removes headers +(``if-none-match`` and ``if-modified-since``) that might elicit a +``304 not modified`` response from the server. This is useful when you want to make +sure you capture an HTTP exchange in its totality. It's also often used during +:ref:`clientreplay`, when you want to make sure the server responds with complete data. + + +================== ====================== +command-line :option:`--anticache` +mitmproxy shortcut :kbd:`o` then :kbd:`a` +================== ======================
\ No newline at end of file diff --git a/docs/features/clientreplay.rst b/docs/features/clientreplay.rst new file mode 100644 index 00000000..b8ca989e --- /dev/null +++ b/docs/features/clientreplay.rst @@ -0,0 +1,18 @@ +.. _clientreplay: + +Client-side replay +================== + +Client-side replay does what it says on the tin: you provide a previously saved +HTTP conversation, and mitmproxy replays the client requests one by one. Note +that mitmproxy serializes the requests, waiting for a response from the server +before starting the next request. This might differ from the recorded +conversation, where requests may have been made concurrently. + +You may want to use client-side replay in conjunction with the +:ref:`anticache` option, to make sure the server responds with complete data. + +================== ================= +command-line :option:`-c path` +mitmproxy shortcut :kbd:`c` +================== =================
\ No newline at end of file diff --git a/docs/features/filters.rst b/docs/features/filters.rst new file mode 100644 index 00000000..5b22376c --- /dev/null +++ b/docs/features/filters.rst @@ -0,0 +1,38 @@ +.. _filters: + +Filter expressions +================== + +Many commands in :program:`mitmproxy` and :program:`mitmdump` take a filter expression. +Filter expressions consist of the following operators: + +.. documentedlist:: + :listobject: libmproxy.filt.help + +- Regexes are Python-style +- Regexes can be specified as quoted strings +- Header matching (~h, ~hq, ~hs) is against a string of the form "name: value". +- Strings with no operators are matched against the request URL. +- The default binary operator is &. + +Examples +-------- + +URL containing "google.com": + +.. code-block:: none + + google\.com + +Requests whose body contains the string "test": + +.. code-block:: none + + ~q ~b test + +Anything but requests with a text/html content type: + +.. code-block:: none + + !(~q & ~t "text/html") + diff --git a/docs/features/replacements.rst b/docs/features/replacements.rst new file mode 100644 index 00000000..8f760866 --- /dev/null +++ b/docs/features/replacements.rst @@ -0,0 +1,72 @@ +.. _replacements: + +Replacements +============ + +Mitmproxy lets you specify an arbitrary number of patterns that define text +replacements within flows. Each pattern has 3 components: a filter that defines +which flows a replacement applies to, a regular expression that defines what +gets replaced, and a target value that defines what is substituted in. + +Replace hooks fire when either a client request or a server response is +received. Only the matching flow component is affected: so, for example, if a +replace hook is triggered on server response, the replacement is only run on +the Response object leaving the Request intact. You control whether the hook +triggers on the request, response or both using the filter pattern. If you need +finer-grained control than this, it's simple to create a script using the +replacement API on Flow components. + +Replacement hooks are extremely handy in interactive testing of applications. +For instance you can use a replace hook to replace the text "XSS" with a +complicated XSS exploit, and then "inject" the exploit simply by interacting +with the application through the browser. When used with tools like Firebug and +mitmproxy's own interception abilities, replacement hooks can be an amazingly +flexible and powerful feature. + + +On the command-line +------------------- + +The replacement hook command-line options use a compact syntax to make it easy +to specify all three components at once. The general form is as follows: + +.. code-block:: none + + /patt/regex/replacement + +Here, **patt** is a mitmproxy filter expression, **regex** is a valid Python +regular expression, and **replacement** is a string literal. The first +character in the expression (``/`` in this case) defines what the separation +character is. Here's an example of a valid expression that replaces "foo" with +"bar" in all requests: + +.. code-block:: none + + :~q:foo:bar + +In practice, it's pretty common for the replacement literal to be long and +complex. For instance, it might be an XSS exploit that weighs in at hundreds or +thousands of characters. To cope with this, there's a variation of the +replacement hook specifier that lets you load the replacement text from a file. +So, you might start **mitmdump** as follows: + +>>> mitmdump --replace-from-file :~q:foo:~/xss-exploit + +This will load the replacement text from the file ``~/xss-exploit``. + +Both the :option:`--replace` and :option:`--replace-from-file` flags can be passed multiple +times. + + +Interactively +------------- + +The :kbd:`R` shortcut key in the mitmproxy options menu (:kbd:`o`) lets you add and edit +replacement hooks using a built-in editor. The context-sensitive help (:kbd:`?`) has +complete usage information. + +================== ============================= +command-line :option:`--replace`, + :option:`--replace-from-file` +mitmproxy shortcut :kbd:`o` then :kbd:`R` +================== ============================= diff --git a/docs/features/serverreplay.rst b/docs/features/serverreplay.rst new file mode 100644 index 00000000..3b4af4e8 --- /dev/null +++ b/docs/features/serverreplay.rst @@ -0,0 +1,39 @@ +.. _serverreplay: + +Server-side replay +================== + +Server-side replay lets us replay server responses from a saved HTTP +conversation. + +Matching requests with responses +-------------------------------- + +By default, :program:`mitmproxy` excludes request headers when matching incoming +requests with responses from the replay file. This works in most circumstances, +and makes it possible to replay server responses in situations where request +headers would naturally vary, e.g. using a different user agent. +The :option:`--rheader headername` command-line option allows you to override +this behaviour by specifying individual headers that should be included in matching. + + +Response refreshing +------------------- + +Simply replaying server responses without modification will often result in +unexpected behaviour. For example cookie timeouts that were in the future at +the time a conversation was recorded might be in the past at the time it is +replayed. By default, :program:`mitmproxy` refreshes server responses before sending +them to the client. The **date**, **expires** and **last-modified** headers are +all updated to have the same relative time offset as they had at the time of +recording. So, if they were in the past at the time of recording, they will be +in the past at the time of replay, and vice versa. Cookie expiry times are +updated in a similar way. + +You can turn off response refreshing using the :option:`--norefresh` argument, or using +the :kbd:`o` options shortcut within :program:`mitmproxy`. + +================== ================= +command-line :option:`-S path` +mitmproxy shortcut :kbd:`S` +================== =================
\ No newline at end of file diff --git a/docs/features/setheaders.rst b/docs/features/setheaders.rst new file mode 100644 index 00000000..0a6c2296 --- /dev/null +++ b/docs/features/setheaders.rst @@ -0,0 +1,18 @@ +.. _setheaders: + +Set Headers +=========== + +This feature lets you specify a set of headers to be added to requests or +responses, based on a filter pattern. You can specify these either on the +command-line, or through an interactive editor in mitmproxy. + +Example: +.. code-block:: none + + mitmdump -R http://example.com --setheader :~q:Host:example.com + +================== ============================= +command-line :option:`--setheader PATTERN` +mitmproxy shortcut :kbd:`o` then :kbd:`H` +================== =============================
\ No newline at end of file diff --git a/docs/features/upstreamcerts.rst b/docs/features/upstreamcerts.rst new file mode 100644 index 00000000..a287daef --- /dev/null +++ b/docs/features/upstreamcerts.rst @@ -0,0 +1,4 @@ +.. _upstreamcerts: + +Upstream Certificates +=====================
\ No newline at end of file diff --git a/docs/howmitmproxy.rst b/docs/howmitmproxy.rst new file mode 100644 index 00000000..8bc20792 --- /dev/null +++ b/docs/howmitmproxy.rst @@ -0,0 +1,238 @@ +How mitmproxy works +=================== + +Mitmproxy is an enormously flexible tool. Knowing exactly how the proxying +process works will help you deploy it creatively, and take into account its +fundamental assumptions and how to work around them. This document explains +mitmproxy's proxy mechanism in detail, starting with the simplest unencrypted +explicit proxying, and working up to the most complicated interaction - +transparent proxying of SSL-protected traffic [#ssl]_ in the presence of `Server Name Indication`_. + +Explicit HTTP +------------- + +Configuring the client to use mitmproxy as an explicit proxy is the simplest +and most reliable way to intercept traffic. The proxy protocol is codified in the +`HTTP RFC`_, so the behaviour of both +the client and the server is well defined, and usually reliable. In the +simplest possible interaction with mitmproxy, a client connects directly to the +proxy, and makes a request that looks like this: + +.. code-block:: http + + GET http://example.com/index.html HTTP/1.1 + +This is a proxy GET request - an extended form of the vanilla HTTP GET request +that includes a schema and host specification, and it includes all the +information mitmproxy needs to proceed. + +.. image:: schematics/how-mitmproxy-works-explicit.png + :align: center + +1. The client connects to the proxy and makes a request. +2. Mitmproxy connects to the upstream server and simply forwards the request on. + + +Explicit HTTPS +-------------- + +The process for an explicitly proxied HTTPS connection is quite different. The +client connects to the proxy and makes a request that looks like this: + +.. code-block:: http + + CONNECT example.com:443 HTTP/1.1 + +A conventional proxy can neither view nor manipulate an SSL-encrypted data +stream, so a CONNECT request simply asks the proxy to open a pipe between the +client and server. The proxy here is just a facilitator - it blindly forwards +data in both directions without knowing anything about the contents. The +negotiation of the SSL connection happens over this pipe, and the subsequent +flow of requests and responses are completely opaque to the proxy. + +The MITM in mitmproxy +^^^^^^^^^^^^^^^^^^^^^ + +This is where mitmproxy's fundamental trick comes into play. The MITM in its +name stands for Man-In-The-Middle - a reference to the process we use to +intercept and interfere with these theoretically opaque data streams. The basic +idea is to pretend to be the server to the client, and pretend to be the client +to the server, while we sit in the middle decoding traffic from both sides. The +tricky part is that the `Certificate Authority`_ system is +designed to prevent exactly this attack, by allowing a trusted third-party to +cryptographically sign a server's SSL certificates to verify that they are +legit. If this signature doesn't match or is from a non-trusted party, a secure +client will simply drop the connection and refuse to proceed. Despite the many +shortcomings of the CA system as it exists today, this is usually fatal to +attempts to MITM an SSL connection for analysis. Our answer to this conundrum +is to become a trusted Certificate Authority ourselves. Mitmproxy includes a +full CA implementation that generates interception certificates on the fly. To +get the client to trust these certificates, we :ref:`register mitmproxy as a trusted +CA with the device manually <certinstall>`. + +Complication 1: What's the remote hostname? +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +To proceed with this plan, we need to know the domain name to use in the +interception certificate - the client will verify that the certificate is for +the domain it's connecting to, and abort if this is not the case. At first +blush, it seems that the CONNECT request above gives us all we need - in this +example, both of these values are "example.com". But what if the client had +initiated the connection as follows: + +.. code-block:: http + + CONNECT 10.1.1.1:443 HTTP/1.1 + +Using the IP address is perfectly legitimate because it gives us enough +information to initiate the pipe, even though it doesn't reveal the remote +hostname. + +Mitmproxy has a cunning mechanism that smooths this over - :ref:`upstream +certificate sniffing <upstreamcerts>`. As soon as we +see the CONNECT request, we pause the client part of the conversation, and +initiate a simultaneous connection to the server. We complete the SSL handshake +with the server, and inspect the certificates it used. Now, we use the Common +Name in the upstream SSL certificates to generate the dummy certificate for the +client. Voila, we have the correct hostname to present to the client, even if +it was never specified. + + +Complication 2: Subject Alternative Name +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +Enter the next complication. Sometimes, the certificate Common Name is not, in +fact, the hostname that the client is connecting to. This is because of the +optional `Subject Alternative Name`_ field in the SSL certificate +that allows an arbitrary number of alternative domains to be specified. If the +expected domain matches any of these, the client will proceed, even though the +domain doesn't match the certificate Common Name. The answer here is simple: +when we extract the CN from the upstream cert, we also extract the SANs, and +add them to the generated dummy certificate. + + +Complication 3: Server Name Indication +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +One of the big limitations of vanilla SSL is that each certificate requires its +own IP address. This means that you couldn't do virtual hosting where multiple +domains with independent certificates share the same IP address. In a world +with a rapidly shrinking IPv4 address pool this is a problem, and we have a +solution in the form of the `Server Name Indication`_ extension to +the SSL and TLS protocols. This lets the client specify the remote server name +at the start of the SSL handshake, which then lets the server select the right +certificate to complete the process. + +SNI breaks our upstream certificate sniffing process, because when we connect +without using SNI, we get served a default certificate that may have nothing to +do with the certificate expected by the client. The solution is another tricky +complication to the client connection process. After the client connects, we +allow the SSL handshake to continue until just _after_ the SNI value has been +passed to us. Now we can pause the conversation, and initiate an upstream +connection using the correct SNI value, which then serves us the correct +upstream certificate, from which we can extract the expected CN and SANs. + +Putting it all together +^^^^^^^^^^^^^^^^^^^^^^^ + +Lets put all of this together into the complete explicitly proxied HTTPS flow. + +.. image:: schematics/how-mitmproxy-works-explicit-https.png + :align: center + +1. The client makes a connection to mitmproxy, and issues an HTTP CONNECT request. +2. Mitmproxy responds with a ``200 Connection Established``, as if it has set up the CONNECT pipe. +3. The client believes it's talking to the remote server, and initiates the SSL connection. + It uses SNI to indicate the hostname it is connecting to. +4. Mitmproxy connects to the server, and establishes an SSL connection using the SNI hostname + indicated by the client. +5. The server responds with the matching SSL certificate, which contains the CN and SAN values + needed to generate the interception certificate. +6. Mitmproxy generates the interception cert, and continues the + client SSL handshake paused in step 3. +7. The client sends the request over the established SSL connection. +8. Mitmproxy passes the request on to the server over the SSL connection initiated in step 4. + +Transparent HTTP +---------------- + +When a transparent proxy is used, the HTTP/S connection is redirected into a +proxy at the network layer, without any client configuration being required. +This makes transparent proxying ideal for those situations where you can't +change client behaviour - proxy-oblivious Android applications being a common +example. + +To achieve this, we need to introduce two extra components. The first is a +redirection mechanism that transparently reroutes a TCP connection destined for +a server on the Internet to a listening proxy server. This usually takes the +form of a firewall on the same host as the proxy server - `iptables`_ on Linux or +pf_ on OSX. Once the client has initiated the connection, it makes a vanilla HTTP request, +which might look something like this: + +.. code-block:: http + + GET /index.html HTTP/1.1 + +Note that this request differs from the explicit proxy variation, in that it +omits the scheme and hostname. How, then, do we know which upstream host to +forward the request to? The routing mechanism that has performed the +redirection keeps track of the original destination for us. Each routing +mechanism has a different way of exposing this data, so this introduces the +second component required for working transparent proxying: a host module that +knows how to retrieve the original destination address from the router. In +mitmproxy, this takes the form of a built-in set of +modules_ that know how to talk to each platform's redirection mechanism. +Once we have this information, the process is fairly straight-forward. + +.. image:: schematics/how-mitmproxy-works-transparent.png + :align: center + +1. The client makes a connection to the server. +2. The router redirects the connection to mitmproxy, which is typically listening on a local port + of the same host. Mitmproxy then consults the routing mechanism to establish what the original + destination was. +3. Now, we simply read the client's request... +4. ... and forward it upstream. + +Transparent HTTPS +----------------- + +The first step is to determine whether we should treat an incoming connection +as HTTPS. The mechanism for doing this is simple - we use the routing mechanism +to find out what the original destination port is. By default, we treat all +traffic destined for ports 443 and 8443 as SSL. + +From here, the process is a merger of the methods we've described for +transparently proxying HTTP, and explicitly proxying HTTPS. We use the routing +mechanism to establish the upstream server address, and then proceed as for +explicit HTTPS connections to establish the CN and SANs, and cope with SNI. + +.. image:: schematics/how-mitmproxy-works-transparent-https.png + :align: center + +1. The client makes a connection to the server. +2. The router redirects the connection to mitmproxy, which is typically listening on a local port of + the same host. Mitmproxy then consults the routing mechanism to establish what the original + destination was. +3. The client believes it's talking to the remote server, and initiates the SSL connection. It uses + SNI to indicate the hostname it is connecting to. +4. Mitmproxy connects to the server, and establishes an SSL connection using the SNI hostname + indicated by the client. +5. The server responds with the matching SSL certificate, which contains the CN and SAN values + needed to generate the interception certificate. +6. Mitmproxy generates the interception cert, and continues the client SSL handshake paused in + step 3. +7. The client sends the request over the established SSL connection. +8. Mitmproxy passes the request on to the server over the SSL connection initiated in step 4. + +.. rubric:: Footnotes + +.. [#ssl] I use "SSL" to refer to both SSL and TLS in the generic sense, unless otherwise specified. + +.. _Server Name Indication: https://en.wikipedia.org/wiki/Server_Name_Indication +.. _HTTP RFC: https://tools.ietf.org/html/rfc7230 +.. _Certificate Authority: https://en.wikipedia.org/wiki/Certificate_authority +.. _Subject Alternative Name: https://en.wikipedia.org/wiki/SubjectAltName +.. _iptables: http://www.netfilter.org/ +.. _pf: https://en.wikipedia.org/wiki/PF_\(firewall\) +.. _modules: https://github.com/mitmproxy/mitmproxy/tree/master/libmproxy/platform diff --git a/docs/index.rst b/docs/index.rst new file mode 100644 index 00000000..92583075 --- /dev/null +++ b/docs/index.rst @@ -0,0 +1,54 @@ +.. include:: introduction.rst + + +.. toctree:: + :hidden: + :maxdepth: 1 + + introduction + install + certinstall + howmitmproxy + modes + +.. toctree:: + :hidden: + :caption: Tools + + mitmproxy + mitmdump + config + +.. toctree:: + :hidden: + :caption: Features + + features/anticache + features/filters + features/replacements + features/clientreplay + features/upstreamcerts + +.. toctree:: + :hidden: + :caption: Scripting + + scripting/inlinescripts + scripting/libmproxy + + +.. toctree:: + :hidden: + :caption: Development + + dev/protocols + dev/proxy + dev/exceptions + dev/models + +.. Indices and tables + ================== + + * :ref:`genindex` + * :ref:`modindex` + diff --git a/docs/install.rst b/docs/install.rst new file mode 100644 index 00000000..e0a572af --- /dev/null +++ b/docs/install.rst @@ -0,0 +1,100 @@ +.. _install: + +Installation +============ + +.. _install-ubuntu: + +Installation On Ubuntu +---------------------- + +Ubuntu comes with Python but we need to install pip, python-dev and several libraries. +This was tested on a fully patched installation of Ubuntu 14.04. + +>>> sudo apt-get install python-pip python-dev libffi-dev libssl-dev libxml2-dev libxslt1-dev +>>> sudo pip install mitmproxy + +Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal. + +Installation From Source (Ubuntu) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you would like to install mitmproxy directly from the master branch on GitHub or would like to +get set up to contribute to the project, install the dependencies as you would for a regular +mitmproxy installation (see :ref:`install-ubuntu`). +Then see the Hacking_ section of the README on GitHub. + + + +Installation On Mac OS X +------------------------ + +The easiest way to get up and running on OSX is to download the pre-built binary packages from +`mitmproxy.org`_. + +There are a few bits of customization you might want to do to make mitmproxy comfortable to use on +OSX. The default color scheme is optimized for a dark background terminal, but you can select a +palette for a light terminal background with the ``--palette`` option. +You can use the OSX **open** program to create a simple and effective ``~/.mailcap`` file to view +request and response bodies: + +.. code-block:: none + + application/*; /usr/bin/open -Wn %s + audio/*; /usr/bin/open -Wn %s + image/*; /usr/bin/open -Wn %s + video/*; /usr/bin/open -Wn %s + +Once installation is complete you can run :ref:`mitmproxy` or :ref:`mitmdump` from a terminal. + + +Installation From Source (Mac OS X) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you would like to install mitmproxy directly from the master branch on GitHub or would like to +get set up to contribute to the project, there are a few OS X specific things to keep in mind. + +- Make sure that XCode is installed from the App Store, and that the command-line tools have been + downloaded (XCode/Preferences/Downloads). +- If you're running a Python interpreter installed with homebrew (or similar), you may have to + install some dependencies by hand. + +Then see the Hacking_ section of the README on GitHub. + +Installation On Windows +----------------------- + +.. note:: + Please note that mitmdump is the only component of mitmproxy that is supported on Windows at + the moment. + + **There is no interactive user interface on Windows.** + + +First, install the latest version of Python 2.7 from the `Python website`_. +If you already have an older version of Python 2.7 installed, make sure to install pip_ +(pip is included in Python 2.7.9+ by default). + +Next, add Python and the Python Scripts directory to your **PATH** variable. +You can do this easily by running the following in powershell: + +>>> [Environment]::SetEnvironmentVariable("Path", "$env:Path;C:\Python27;C:\Python27\Scripts", "User") + +Now, you can install mitmproxy by running + +>>> pip install mitmproxy + +Once the installation is complete, you can run :ref:`mitmdump` from a command prompt. + +Installation From Source (Windows) +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +If you would like to install mitmproxy directly from the master branch on GitHub or would like to +get set up to contribute to the project, install Python as outlined above, then see the +Hacking_ section of the README on GitHub. + + +.. _Hacking: https://github.com/mitmproxy/mitmproxy/blob/master/README.mkd#hacking +.. _mitmproxy.org: https://mitmproxy.org/ +.. _`Python website`: https://www.python.org/downloads/windows/ +.. _pip: https://pip.pypa.io/en/latest/installing.html
\ No newline at end of file diff --git a/docs/introduction.rst b/docs/introduction.rst new file mode 100644 index 00000000..6ce6add9 --- /dev/null +++ b/docs/introduction.rst @@ -0,0 +1,26 @@ +Introduction +============ + +**mitmproxy** is an interactive, SSL-capable man-in-the-middle proxy for HTTP +with a console interface. + +**mitmdump** is the command-line version of mitmproxy. Think tcpdump for HTTP. + +**libmproxy** is the library that mitmproxy and mitmdump are built on. + +Documentation, tutorials and distribution packages can be found on the +mitmproxy website: `mitmproxy.org <https://mitmproxy.org/>`_ + + +.. rubric:: Features + + +- Intercept HTTP requests and responses and modify them on the fly. +- Save complete HTTP conversations for later replay and analysis. +- Replay the client-side of an HTTP conversations. +- Replay HTTP responses of a previously recorded server. +- Reverse proxy mode to forward traffic to a specified server. +- Transparent proxy mode on OSX and Linux. +- Make scripted changes to HTTP traffic using Python. +- SSL certificates for interception are generated on the fly. +- And much, much more.
\ No newline at end of file diff --git a/docs/mitmdump.rst b/docs/mitmdump.rst new file mode 100644 index 00000000..d9b4a26b --- /dev/null +++ b/docs/mitmdump.rst @@ -0,0 +1,66 @@ +.. _mitmdump: +.. program:: mitmdump + +mitmdump +======== + + +**mitmdump** is the command-line companion to mitmproxy. It provides +tcpdump-like functionality to let you view, record, and programmatically +transform HTTP traffic. See the :option:`--help` flag output for complete +documentation. + + + +Examples +-------- + +Saving traffic +^^^^^^^^^^^^^^ + +>>> mitmdump -w outfile + +Start up mitmdump in proxy mode, and write all traffic to **outfile**. + + +Filtering saved traffic +^^^^^^^^^^^^^^^^^^^^^^^ + +>>> mitmdump -nr infile -w outfile "~m post" + +Start mitmdump without binding to the proxy port (:option:`-n`), read all flows from +infile, apply the specified filter expression (only match POSTs), and write to +outfile. + + +Client replay +^^^^^^^^^^^^^ + +>>> mitmdump -nc outfile + +Start mitmdump without binding to the proxy port (:option:`-n`), then replay all +requests from outfile (:option:`-c filename`). Flags combine in the obvious way, so +you can replay requests from one file, and write the resulting flows to +another: + +>>> mitmdump -nc srcfile -w dstfile + +See the :ref:`clientreplay` section for more information. + + +Running a script +^^^^^^^^^^^^^^^^ + +>>> mitmdump -s examples/add_header.py + +This runs the **add_header.py** example script, which simply adds a new header +to all responses. + +Scripted data transformation +^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +>>> mitmdump -ns examples/add_header.py -r srcfile -w dstfile + +This command loads flows from **srcfile**, transforms it according to the +specified script, then writes it back to **dstfile**. + diff --git a/docs/mitmproxy-long.png b/docs/mitmproxy-long.png Binary files differnew file mode 100644 index 00000000..f9397d1e --- /dev/null +++ b/docs/mitmproxy-long.png diff --git a/docs/mitmproxy.rst b/docs/mitmproxy.rst new file mode 100644 index 00000000..fa3b57c7 --- /dev/null +++ b/docs/mitmproxy.rst @@ -0,0 +1,126 @@ +.. _mitmproxy: +.. program:: mitmproxy + +mitmproxy +========= + + +**mitmproxy** is a console tool that allows interactive examination and +modification of HTTP traffic. It differs from mitmdump in that all flows are +kept in memory, which means that it's intended for taking and manipulating +small-ish samples. Use the :kbd:`?` shortcut key to view, context-sensitive +documentation from any **mitmproxy** screen. + +Flow list +--------- + +The flow list shows an index of captured flows in chronological order. + +.. image:: screenshots/mitmproxy.png + +- **1**: A GET request, returning a 302 Redirect response. +- **2**: A GET request, returning 16.75kb of text/html data. +- **3**: A replayed request. +- **4**: Intercepted flows are indicated with orange text. The user may edit + these flows, and then accept them (using the :kbd:`a` key) to continue. In this + case, the request has been intercepted on the way to the server. +- **5**: A response intercepted from the server on the way to the client. +- **6**: The event log can be toggled on and off using the :kbd:`e` shortcut key. This + pane shows events and errors that may not result in a flow that shows up in the + flow pane. +- **7**: Flow count. +- **8**: Various information on mitmproxy's state. In this case, we have an + interception pattern set to ``.*``. +- **9**: Bind address indicator - mitmproxy is listening on port 8080 of all + interfaces. + + +Flow view +--------- + +The **Flow View** lets you inspect and manipulate a single flow: + +.. image:: screenshots/mitmproxy-flowview.png + +- **1**: Flow summary. +- **2**: The Request/Response tabs, showing you which part of the flow you are + currently viewing. In the example above, we're viewing the Response. Hit :kbd:`tab` + to switch between the Response and the Request. +- **3**: Headers. +- **4**: Body. +- **5**: View Mode indicator. In this case, we're viewing the body in **hex** mode. The other + available modes are **pretty**, which uses a number of heuristics to show you a friendly + view of various content types, and **raw**, which shows you exactly what's there without any + changes. You can change modes using the :kbd:`m` key. + + +Grid Editor +----------- + +Much of the data that we'd like to interact with in mitmproxy is structured. +For instance, headers, queries and form data can all be thought of as a list of +key/value pairs. Mitmproxy has a built-in editor that lays this type of data +out in a grid for easy manipulation. + +At the moment, the Grid Editor is used in four parts of mitmproxy: + + - Editing request or response headers (:kbd:`e` for edit, then :kbd:`h` for headers in flow view) + - Editing a query string (:kbd:`e` for edit, then :kbd:`q` for query in flow view) + - Editing a URL-encoded form (:kbd:`e` for edit, then :kbd:`f` for form in flow view) + - Editing replacement patterns (:kbd:`o` for options, then :kbd:`R` for Replacement Patterns) + +If there is is no data, an empty editor will be started to let you add some. +Here is the editor showing the headers from a request: + +.. image:: screenshots/mitmproxy-kveditor.png + +To edit, navigate to the key or value you want to modify using the arrow or vi +navigation keys, and press enter. The background color will change to show that +you are in edit mode for the specified field: + +.. image:: screenshots/mitmproxy-kveditor-editmode.png + +Modify the field as desired, then press escape to exit edit mode when you're +done. You can also add a row (:kbd:`a` key), delete a row (:kbd:`d` key), spawn an +external editor on a field (:kbd:`e` key). Be sure to consult the context-sensitive +help (:kbd:`?` key) for more. + +Example: Interception +--------------------- + +**mitmproxy**'s interception functionality lets you pause an HTTP request or +response, inspect and modify it, and then accept it to send it on to the server +or client. + + +1: Set an interception pattern +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: screenshots/mitmproxy-intercept-filt.png + +We press :kbd:`i` to set an interception pattern. In this case, the ``~q`` filter +pattern tells **mitmproxy** to intercept all requests. For complete filter +syntax, see the :ref:`filters` section of the documentation, +or the built-in help function in **mitmproxy**. + +2: Intercepted connections are indicated with orange text: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: screenshots/mitmproxy-intercept-mid.png + +3: You can now view and modify the request: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: screenshots/mitmproxy-intercept-options.png + +In this case, we viewed the request by selecting it, pressed :kbd:`e` for "edit" +and :kbd:`m` for "method" to change the HTTP request method. + +4: Accept the intercept to continue: +^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^ + +.. image:: screenshots/mitmproxy-intercept-result.png + +Finally, we press :kbd:`a` to accept the modified request, which is then sent on to +the server. In this case, we changed the request from an HTTP GET to +OPTIONS, and Google's server has responded with a 405 "Method not allowed". diff --git a/docs/modes.rst b/docs/modes.rst new file mode 100644 index 00000000..197c0525 --- /dev/null +++ b/docs/modes.rst @@ -0,0 +1,193 @@ +.. _modes: + +Modes of Operation +================== + +Mitmproxy has four modes of operation that allow you to use mitmproxy in a +variety of scenarios: + +- **Regular** (the default) +- **Transparent** +- **Reverse Proxy** +- **Upstream Proxy** + + +Now, which one should you pick? Use this flow chart: + +.. image:: schematics/proxy-modes-flowchart.png + :align: center + +Regular Proxy +------------- + +Mitmproxy's regular mode is the simplest and the easiest to set up. + +1. Start mitmproxy. +2. Configure your client to use mitmproxy by explicitly setting an HTTP proxy. +3. Quick Check: You should already be able to visit an unencrypted HTTP site through the proxy. +4. Open the magic domain **mitm.it** and install the certificate for your device. + +.. note:: + Unfortunately, some applications bypass the system HTTP proxy settings - Android applications + are a common example. In these cases, you need to use mitmproxy's transparent mode. + +If you are proxying an external device, your network will probably look like this: + +.. image:: schematics/proxy-modes-regular.png + :align: center + +The square brackets signify the source and destination IP addresses. Your +client explicitly connects to mitmproxy and mitmproxy explicitly connects +to the target server. + +Transparent Proxy +----------------- + +In transparent mode, traffic is directed into a proxy at the network layer, +without any client configuration required. This makes transparent proxying +ideal for situations where you can't change client behaviour. In the graphic +below, a machine running mitmproxy has been inserted between the router and +the internet: + +.. image:: schematics/proxy-modes-transparent-1.png + :align: center + +The square brackets signify the source and destination IP addresses. Round +brackets mark the next hop on the *Ethernet/data link* layer. This distinction +is important: when the packet arrives at the mitmproxy machine, it must still +be addressed to the target server. This means that Network Address Translation +should not be applied before the traffic reaches mitmproxy, since this would +remove the target information, leaving mitmproxy unable to determine the real +destination. + +.. image:: schematics/proxy-modes-transparent-wrong.png + :align: center + +Common Configurations +^^^^^^^^^^^^^^^^^^^^^ + +There are many ways to configure your network for transparent proxying. We'll +look at two common scenarios: + +1. Configuring the client to use a custom gateway/router/"next hop" +2. Implementing custom routing on the router + +In most cases, the first option is recommended due to its ease of use. + +(a) Custom Gateway +~~~~~~~~~~~~~~~~~~ + +One simple way to get traffic to the mitmproxy machine with the destination IP +intact, is to simply configure the client with the mitmproxy box as the +default gateway. + +.. image:: schematics/proxy-modes-transparent-2.png + :align: center + +In this scenario, we would: + +1. Configure the proxy machine for transparent mode. You can find instructions + in the :ref:`transparent` section. +2. Configure the client to use the proxy machine's IP as the default gateway. +3. Quick Check: At this point, you should already be able to visit an + unencrypted HTTP site over the proxy. +4. Open the magic domain **mitm.it**mitm and install the certificate + for your device. + +Setting the custom gateway on clients can be automated by serving the settings +out to clients over DHCP. This lets set up an interception network where all +clients are proxied automatically, which can save time and effort. + +.. admonition:: Troubleshooting Transparent Mode + :class: note + + Incorrect transparent mode configurations are a frequent source of + error. If it doesn't work for you, try the following things: + + - Open mitmproxy's event log (press :kbd:`e`) - do you see clientconnect messages? + If not, the packets are not arriving at the proxy. One common cause is the occurrence of ICMP + redirects, which means that your machine is telling the client that there's a faster way to + the internet by contacting your router directly (see the :ref:`transparent` section on how to + disable them). If in doubt, Wireshark_ may help you to see whether something arrives at your + machine or not. + - Make sure you have not explicitly configured an HTTP proxy on the client. + This is not needed in transparent mode. + - Re-check the instructions in the :ref:`transparent` section. Anything you missed? + + If you encounter any other pitfalls that should be listed here, please let us know! + +(b) Custom Routing +~~~~~~~~~~~~~~~~~~ + +In some cases, you may need more fine-grained control of which traffic reaches +the mitmproxy instance, and which doesn't. You may, for instance, choose only +to divert traffic to some hosts into the transparent proxy. There are a huge +number of ways to accomplish this, and much will depend on the router or +packet filter you're using. In most cases, the configuration will look like +this: + +.. image:: schematics/proxy-modes-transparent-3.png + :align: center + + +Reverse Proxy +------------- + +mitmproxy is usually used with a client that uses the proxy to access the +Internet. Using reverse proxy mode, you can use mitmproxy to act like a normal +HTTP server: + +.. image:: schematics/proxy-modes-reverse.png + :align: center + +There are various use-cases: + +- Say you have an internal API running at http://example.local/. You could now + set up mitmproxy in reverse proxy mode at http://debug.example.local/ and + dynamically point clients to this new API endpoint, which provides them + with the same data and you with debug information. Similarly, you could move + your real server to a different IP/port and set up mitmproxy in the original + place to debug and or redirect all sessions. + +- Say you're a web developer working on http://example.com/ (with a development + version running on http://localhost:8000/). You can modify your hosts file so that + example.com points to 127.0.0.1 and then run mitmproxy in reverse proxy mode + on port 80. You can test your app on the example.com domain and get all + requests recorded in mitmproxy. + +- Say you have some toy project that should get SSL support. Simply set up + mitmproxy as a reverse proxy on port 443 and you're done (``mitmdump -p 443 -R + http://localhost:80/``). Mitmproxy auto-detects TLS traffic and intercepts it dynamically. + There are better tools for this specific task, but mitmproxy is very quick and simple way to + set up an SSL-speaking server. + +- Want to add a non-SSL-capable compression proxy in front of your server? You + could even spawn a mitmproxy instance that terminates SSL (``-R http://...``), + point it to the compression proxy and let the compression proxy point to a + SSL-initiating mitmproxy (``-R https://...``), which then points to the real + server. As you see, it's a fairly flexible thing. + +.. admonition:: Caveat: Interactive Use + :class: warning + + Reverse Proxy mode is usually not sufficient to create a copy of an interactive website at + different URL. The HTML served to the client remains unchanged - as soon as the user clicks on + an non-relative URL (or downloads a non-relative image resource), traffic no longer passes + through mitmproxy. + +Upstream Proxy +-------------- + +If you want to chain proxies by adding mitmproxy in front of a different proxy +appliance, you can use mitmproxy's upstream mode. In upstream mode, all +requests are unconditionally transferred to an upstream proxy of your choice. + +.. image:: schematics/proxy-modes-upstream.png + :align: center + +mitmproxy supports both explicit HTTP and explicit HTTPS in upstream proxy +mode. You could in theory chain multiple mitmproxy instances in a row, but +that doesn't make any sense in practice (i.e. outside of our tests). + + +.. _Wireshark: https://wireshark.org/
\ No newline at end of file diff --git a/docs/schematics/architecture.pdf b/docs/schematics/architecture.pdf Binary files differnew file mode 100644 index 00000000..77f5ad58 --- /dev/null +++ b/docs/schematics/architecture.pdf diff --git a/docs/schematics/architecture.png b/docs/schematics/architecture.png Binary files differnew file mode 100644 index 00000000..67d6c718 --- /dev/null +++ b/docs/schematics/architecture.png diff --git a/docs/schematics/architecture.vsdx b/docs/schematics/architecture.vsdx Binary files differnew file mode 100644 index 00000000..c4ff13d2 --- /dev/null +++ b/docs/schematics/architecture.vsdx diff --git a/docs/schematics/how-mitmproxy-works-explicit-https.png b/docs/schematics/how-mitmproxy-works-explicit-https.png Binary files differnew file mode 100644 index 00000000..1f1ca023 --- /dev/null +++ b/docs/schematics/how-mitmproxy-works-explicit-https.png diff --git a/docs/schematics/how-mitmproxy-works-explicit.png b/docs/schematics/how-mitmproxy-works-explicit.png Binary files differnew file mode 100644 index 00000000..c9ba26a7 --- /dev/null +++ b/docs/schematics/how-mitmproxy-works-explicit.png diff --git a/docs/schematics/how-mitmproxy-works-transparent-https.png b/docs/schematics/how-mitmproxy-works-transparent-https.png Binary files differnew file mode 100644 index 00000000..559cddd2 --- /dev/null +++ b/docs/schematics/how-mitmproxy-works-transparent-https.png diff --git a/docs/schematics/how-mitmproxy-works-transparent.png b/docs/schematics/how-mitmproxy-works-transparent.png Binary files differnew file mode 100644 index 00000000..3994d681 --- /dev/null +++ b/docs/schematics/how-mitmproxy-works-transparent.png diff --git a/docs/schematics/proxy-modes-flowchart.png b/docs/schematics/proxy-modes-flowchart.png Binary files differnew file mode 100644 index 00000000..716b5ee2 --- /dev/null +++ b/docs/schematics/proxy-modes-flowchart.png diff --git a/docs/schematics/proxy-modes-regular.png b/docs/schematics/proxy-modes-regular.png Binary files differnew file mode 100644 index 00000000..95bada08 --- /dev/null +++ b/docs/schematics/proxy-modes-regular.png diff --git a/docs/schematics/proxy-modes-reverse.png b/docs/schematics/proxy-modes-reverse.png Binary files differnew file mode 100644 index 00000000..071d3fc8 --- /dev/null +++ b/docs/schematics/proxy-modes-reverse.png diff --git a/docs/schematics/proxy-modes-transparent-1.png b/docs/schematics/proxy-modes-transparent-1.png Binary files differnew file mode 100644 index 00000000..002e0e76 --- /dev/null +++ b/docs/schematics/proxy-modes-transparent-1.png diff --git a/docs/schematics/proxy-modes-transparent-2.png b/docs/schematics/proxy-modes-transparent-2.png Binary files differnew file mode 100644 index 00000000..41997b05 --- /dev/null +++ b/docs/schematics/proxy-modes-transparent-2.png diff --git a/docs/schematics/proxy-modes-transparent-3.png b/docs/schematics/proxy-modes-transparent-3.png Binary files differnew file mode 100644 index 00000000..ee26cb4f --- /dev/null +++ b/docs/schematics/proxy-modes-transparent-3.png diff --git a/docs/schematics/proxy-modes-transparent-wrong.png b/docs/schematics/proxy-modes-transparent-wrong.png Binary files differnew file mode 100644 index 00000000..ca501e93 --- /dev/null +++ b/docs/schematics/proxy-modes-transparent-wrong.png diff --git a/docs/schematics/proxy-modes-upstream.png b/docs/schematics/proxy-modes-upstream.png Binary files differnew file mode 100644 index 00000000..d40a6494 --- /dev/null +++ b/docs/schematics/proxy-modes-upstream.png diff --git a/docs/schematics/proxy-modes.pdf b/docs/schematics/proxy-modes.pdf Binary files differnew file mode 100644 index 00000000..f07ea05e --- /dev/null +++ b/docs/schematics/proxy-modes.pdf diff --git a/docs/schematics/proxy-modes.vsdx b/docs/schematics/proxy-modes.vsdx Binary files differnew file mode 100644 index 00000000..c78cf8d0 --- /dev/null +++ b/docs/schematics/proxy-modes.vsdx diff --git a/docs/screenshots/firefox3-import.jpg b/docs/screenshots/firefox3-import.jpg Binary files differnew file mode 100644 index 00000000..47fcd672 --- /dev/null +++ b/docs/screenshots/firefox3-import.jpg diff --git a/docs/screenshots/firefox3-trust.jpg b/docs/screenshots/firefox3-trust.jpg Binary files differnew file mode 100644 index 00000000..50a2f341 --- /dev/null +++ b/docs/screenshots/firefox3-trust.jpg diff --git a/docs/screenshots/firefox3.jpg b/docs/screenshots/firefox3.jpg Binary files differnew file mode 100644 index 00000000..6c4613b6 --- /dev/null +++ b/docs/screenshots/firefox3.jpg diff --git a/docs/screenshots/ios-gateway.png b/docs/screenshots/ios-gateway.png Binary files differnew file mode 100644 index 00000000..2489cba3 --- /dev/null +++ b/docs/screenshots/ios-gateway.png diff --git a/docs/screenshots/ios-installed.png b/docs/screenshots/ios-installed.png Binary files differnew file mode 100644 index 00000000..2071e441 --- /dev/null +++ b/docs/screenshots/ios-installed.png diff --git a/docs/screenshots/ios-manual.png b/docs/screenshots/ios-manual.png Binary files differnew file mode 100644 index 00000000..3977acfe --- /dev/null +++ b/docs/screenshots/ios-manual.png diff --git a/docs/screenshots/ios-profile.png b/docs/screenshots/ios-profile.png Binary files differnew file mode 100644 index 00000000..5bcd5a0d --- /dev/null +++ b/docs/screenshots/ios-profile.png diff --git a/docs/screenshots/ios-reverse.png b/docs/screenshots/ios-reverse.png Binary files differnew file mode 100644 index 00000000..6ab5b7c0 --- /dev/null +++ b/docs/screenshots/ios-reverse.png diff --git a/docs/screenshots/ios-warning.png b/docs/screenshots/ios-warning.png Binary files differnew file mode 100644 index 00000000..d882c514 --- /dev/null +++ b/docs/screenshots/ios-warning.png diff --git a/docs/screenshots/mitmproxy-flowview.png b/docs/screenshots/mitmproxy-flowview.png Binary files differnew file mode 100644 index 00000000..154963fe --- /dev/null +++ b/docs/screenshots/mitmproxy-flowview.png diff --git a/docs/screenshots/mitmproxy-intercept-filt.png b/docs/screenshots/mitmproxy-intercept-filt.png Binary files differnew file mode 100644 index 00000000..60556ee7 --- /dev/null +++ b/docs/screenshots/mitmproxy-intercept-filt.png diff --git a/docs/screenshots/mitmproxy-intercept-mid.png b/docs/screenshots/mitmproxy-intercept-mid.png Binary files differnew file mode 100644 index 00000000..d5b03922 --- /dev/null +++ b/docs/screenshots/mitmproxy-intercept-mid.png diff --git a/docs/screenshots/mitmproxy-intercept-options.png b/docs/screenshots/mitmproxy-intercept-options.png Binary files differnew file mode 100644 index 00000000..8dc4ad2c --- /dev/null +++ b/docs/screenshots/mitmproxy-intercept-options.png diff --git a/docs/screenshots/mitmproxy-intercept-result.png b/docs/screenshots/mitmproxy-intercept-result.png Binary files differnew file mode 100644 index 00000000..7d9f5c94 --- /dev/null +++ b/docs/screenshots/mitmproxy-intercept-result.png diff --git a/docs/screenshots/mitmproxy-kveditor-editmode.png b/docs/screenshots/mitmproxy-kveditor-editmode.png Binary files differnew file mode 100644 index 00000000..a8315ee5 --- /dev/null +++ b/docs/screenshots/mitmproxy-kveditor-editmode.png diff --git a/docs/screenshots/mitmproxy-kveditor.png b/docs/screenshots/mitmproxy-kveditor.png Binary files differnew file mode 100644 index 00000000..144b9701 --- /dev/null +++ b/docs/screenshots/mitmproxy-kveditor.png diff --git a/docs/screenshots/mitmproxy.png b/docs/screenshots/mitmproxy.png Binary files differnew file mode 100644 index 00000000..42a10e32 --- /dev/null +++ b/docs/screenshots/mitmproxy.png diff --git a/docs/screenshots/osx-addcert-alwaystrust.png b/docs/screenshots/osx-addcert-alwaystrust.png Binary files differnew file mode 100644 index 00000000..4c5cc704 --- /dev/null +++ b/docs/screenshots/osx-addcert-alwaystrust.png diff --git a/docs/screenshots/win7-certstore-trustedroot.png b/docs/screenshots/win7-certstore-trustedroot.png Binary files differnew file mode 100644 index 00000000..e15a87f5 --- /dev/null +++ b/docs/screenshots/win7-certstore-trustedroot.png diff --git a/docs/screenshots/win7-certstore.png b/docs/screenshots/win7-certstore.png Binary files differnew file mode 100644 index 00000000..f8ce54bd --- /dev/null +++ b/docs/screenshots/win7-certstore.png diff --git a/docs/screenshots/win7-wizard.png b/docs/screenshots/win7-wizard.png Binary files differnew file mode 100644 index 00000000..eff6ad09 --- /dev/null +++ b/docs/screenshots/win7-wizard.png diff --git a/docs/screenshots/winpythoninstaller.jpg b/docs/screenshots/winpythoninstaller.jpg Binary files differnew file mode 100644 index 00000000..0473c66a --- /dev/null +++ b/docs/screenshots/winpythoninstaller.jpg diff --git a/docs/scripting/inlinescripts.rst b/docs/scripting/inlinescripts.rst new file mode 100644 index 00000000..5914def8 --- /dev/null +++ b/docs/scripting/inlinescripts.rst @@ -0,0 +1,214 @@ +.. _inline-scripts: + +Inline Scripts +============== + +**mitmproxy** has a powerful scripting API that allows you to modify flows +on-the-fly or rewrite previously saved flows locally. + +The mitmproxy scripting API is event driven - a script is simply a Python +module that exposes a set of event methods. Here's a complete mitmproxy script +that adds a new header to every HTTP response before it is returned to the +client: + +.. literalinclude:: ../../examples/add_header.py + :caption: examples/add_header.py + :language: python + +The first argument to each event method is an instance of +:py:class:`~libmproxy.script.ScriptContext` that lets the script interact with the global mitmproxy +state. The **response** event also gets an instance of :py:class:`~libmproxy.script.ScriptContext`, +which we can use to manipulate the response itself. + +We can now run this script using mitmdump or mitmproxy as follows: + +>>> mitmdump -s add_header.py + +The new header will be added to all responses passing through the proxy. + +Examples +-------- + +mitmproxy comes with a variety of example inline scripts, which demonstrate many basic tasks. +We encourage you to either browse them locally or on `GitHub`_. + + +Events +------ + +.. TODO: Split this into Connection, HTTP and TCP events once we have TCP events. + +The ``context`` argument passed to each event method is always a +:py:class:`~libmproxy.script.ScriptContext` instance. It is guaranteed to be the same object +for the scripts lifetime and is not shared between multiple inline scripts. You can safely use it +to store any form of state you require. + +Events are listed in the order they usually occur. + +.. py:function:: start(context, argv) + + Called once on startup, before any other events. + + :param List[str] argv: The inline scripts' arguments. + For example, ``mitmproxy -s 'example.py --foo 42'`` sets argv to ``["--foo", "42"]``. + +.. py:function:: clientconnect(context, root_layer) + + Called when a client initiates a connection to the proxy. Note that + a connection can correspond to multiple HTTP requests. + + .. versionchanged:: 0.14 + + :param Layer root_layer: The root layer (see :ref:`protocols` for an explanation what the root + layer is), which provides transparent access to all attributes of the + :py:class:`~libmproxy.proxy.RootContext`. For example, ``root_layer.client_conn.address`` + gives the remote address of the connecting client. + + +.. py:function:: request(context, flow) + + Called when a client request has been received. The ``flow`` object is + guaranteed to have a non-None ``request`` attribute. + + :param HTTPFlow flow: The flow containing the request which has been received. + The object is guaranteed to have a non-None ``request`` attribute. + +.. py:function:: serverconnect(context, server_conn) + + Called before the proxy initiates a connection to the target server. Note that + a connection can correspond to multiple HTTP requests. + + :param ServerConnection server_conn: The server connection object. It is guaranteed to have a + non-None ``address`` attribute. + +.. py:function:: responseheaders(context, flow) + + Called when the headers of a server response have been received. + This will always be called before the response hook. + + :param HTTPFlow flow: The flow containing the request and response. + The object is guaranteed to have non-None ``request`` and + ``response`` attributes. ``response.content`` will be ``None``, + as the response body has not been read yet. + +.. py:function:: response(context, flow) + + Called when a server response has been received. + + :param HTTPFlow flow: The flow containing the request and response. + The object is guaranteed to have non-None ``request`` and + ``response`` attributes. ``response.body`` will contain the raw response body, + unless response streaming has been enabled. + +.. py:function:: error(context, flow) + + Called when a flow error has occurred, e.g. invalid server responses, or + interrupted connections. This is distinct from a valid server HTTP error + response, which is simply a response with an HTTP error code. + + :param HTTPFlow flow: The flow containing the error. + It is guaranteed to have non-None ``error`` attribute. + +.. py:function:: serverdisconnect(context, server_conn) + + Called when the proxy has closed the server connection. + + .. versionadded:: 0.14 + + :param ServerConnection server_conn: see :py:func:`serverconnect` + +.. py:function:: clientdisconnect(context, root_layer) + + Called when a client disconnects from the proxy. + + .. versionchanged:: 0.14 + + :param Layer root_layer: see :py:func:`clientconnect` + +.. py:function:: done(context) + + Called once on script shutdown, after any other events. + + +API +--- + +The canonical API documentation is the code, which you can browse here, locally or on `GitHub`_. +*Use the Source, Luke!* + +The main classes you will deal with in writing mitmproxy scripts are: + +:py:class:`~libmproxy.script.ScriptContext` + - A handle for interacting with mitmproxy's Flow Master from within scripts. +:py:class:`~libmproxy.models.ClientConnection` + - Describes a client connection. +:py:class:`~libmproxy.models.ServerConnection` + - Describes a server connection. +:py:class:`~libmproxy.models.HTTPFlow` + - A collection of objects representing a single HTTP transaction. +:py:class:`~libmproxy.models.HTTPRequest` + - An HTTP request. +:py:class:`~libmproxy.models.HTTPResponse` + - An HTTP response. +:py:class:`~libmproxy.models.Error` + - A communications error. +:py:class:`netlib.http.Headers` + - A dictionary-like object for managing HTTP headers. +:py:class:`netlib.certutils.SSLCert` + - Exposes information SSL certificates. +:py:class:`libmproxy.flow.FlowMaster` + - The "heart" of libmproxy, usually subclassed as :py:class:`libmproxy.dump.DumpMaster` or + :py:class:`libmproxy.console.ConsoleMaster`. + +Script Context +-------------- + +.. autoclass:: libmproxy.script.ScriptContext + :members: + :undoc-members: + +Running scripts in parallel +--------------------------- + +We have a single flow primitive, so when a script is blocking, other requests are not processed. +While that's usually a very desirable behaviour, blocking scripts can be run threaded by using the +:py:obj:`libmproxy.script.concurrent` decorator. +**If your script does not block, you should avoid the overhead of the decorator.** + +.. literalinclude:: ../../examples/nonblocking.py + :caption: examples/nonblocking.py + :language: python + +Make scripts configurable with arguments +---------------------------------------- + +Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by +surrounding the script call with quotes, e.g. ```mitmdump -s 'script.py --foo 42'``. +The arguments are then exposed in the start event: + +.. literalinclude:: ../../examples/modify_response_body.py + :caption: examples/modify_response_body.py + :language: python + +Running scripts on saved flows +------------------------------ + +Sometimes, we want to run a script on :py:class:`~libmproxy.models.Flow` objects that are already +complete. This happens when you start a script, and then load a saved set of flows from a file +(see the "scripted data transformation" example `here <https://mitmproxy.org/doc/mitmdump.html>`_). +It also happens when you run a one-shot script on a single flow through the ``|`` (pipe) shortcut +in mitmproxy. + +In this case, there are no client connections, and the events are run in the following order: +**start**, **request**, **responseheaders**, **response**, **error**, **done**. +If the flow doesn't have a **response** or **error** associated with it, the matching events will +be skipped. + +Spaces in the script path +------------------------- + +By default, spaces are interpreted as a separator between the inline script and its arguments +(e.g. ``-s 'foo.py 42'``). Consequently, the script path needs to be wrapped in a separate pair of +quotes if it contains spaces: ``-s '\'./foo bar/baz.py\' 42'``. + +.. _GitHub: https://github.com/mitmproxy/mitmproxy diff --git a/docs/scripting/libmproxy.rst b/docs/scripting/libmproxy.rst new file mode 100644 index 00000000..e263b89b --- /dev/null +++ b/docs/scripting/libmproxy.rst @@ -0,0 +1,27 @@ +.. _libmproxy: + +libmproxy +========= + +.. note:: + + We strongly encourage you to use :ref:`inline-scripts` rather than libmproxy. + - Inline Scripts are equally powerful and provide an easier syntax. + - Most examples are written as inline scripts. + - Multiple inline scripts can be used together. + - Inline Scripts can either be executed headless with mitmdump or within the mitmproxy UI. + + +All of mitmproxy's basic functionality is exposed through the **libmproxy** +library. The example below shows a simple implementation of the "sticky cookie" +functionality included in the interactive mitmproxy program. Traffic is +monitored for ``Cookie`` and ``Set-Cookie`` headers, and requests are rewritten +to include a previously seen cookie if they don't already have one. In effect, +this lets you log in to a site using your browser, and then make subsequent +requests using a tool like curl, which will then seem to be part of the +authenticated session. + + +.. literalinclude:: ../../examples/stickycookies + :caption: examples/stickycookies + :language: python diff --git a/docs/transparent.rst b/docs/transparent.rst new file mode 100644 index 00000000..fbc94e08 --- /dev/null +++ b/docs/transparent.rst @@ -0,0 +1,6 @@ +.. _transparent: + +Transparent Proxying +==================== + +TODO
\ No newline at end of file diff --git a/libmproxy/script.py b/libmproxy/script.py index e13f0e2b..fb04f8c3 100644 --- a/libmproxy/script.py +++ b/libmproxy/script.py @@ -11,6 +11,10 @@ class ScriptError(Exception): class ScriptContext: + """ + The script context should be used to interact with the global mitmproxy state from within a + script. + """ def __init__(self, master): self._master = master @@ -41,7 +41,9 @@ dev_deps = { "nose-cov>=1.6", "coveralls>=0.4.1", "pathod>=%s, <%s" % (version.MINORVERSION, version.NEXT_MINORVERSION), - "countershape" + "sphinx>=1.3.1", + "sphinx-autobuild>=0.5.2", + "sphinxcontrib-documentedlist>=0.1", } # Add *all* script dependencies to developer dependencies. for script_deps in scripts.values(): |
