From fdb6a44245249a50b5c95cdf0d8d13ecddfe5726 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Wed, 12 Oct 2016 10:57:05 +1300 Subject: docs: cleanups improvements and fighting sphinx - Hide links to internal code listings, and link to github instead - Improve formatting of code/example captions - Fix outdated documentation of command-line options - Complete documentation of all events + improved formatting - tcp_open -> tcp_start, tcp_close -> tcp_end to reduce confusion --- docs/_static/theme_overrides.css | 29 +++++ docs/conf.py | 45 ++++++- docs/dev/models.rst | 81 ------------- docs/features/clientreplay.rst | 2 +- docs/features/responsestreaming.rst | 6 +- docs/features/serverreplay.rst | 2 +- docs/index.rst | 7 +- docs/scripting/api.rst | 8 ++ docs/scripting/context.rst | 4 + docs/scripting/events.rst | 202 ++++++++++++++++++++++++++++++++ docs/scripting/inlinescripts.rst | 227 ------------------------------------ docs/scripting/mitmproxy.rst | 26 ----- docs/scripting/overview.rst | 79 +++++++++++++ 13 files changed, 375 insertions(+), 343 deletions(-) delete mode 100644 docs/dev/models.rst create mode 100644 docs/scripting/api.rst create mode 100644 docs/scripting/context.rst create mode 100644 docs/scripting/events.rst delete mode 100644 docs/scripting/inlinescripts.rst delete mode 100644 docs/scripting/mitmproxy.rst create mode 100644 docs/scripting/overview.rst (limited to 'docs') diff --git a/docs/_static/theme_overrides.css b/docs/_static/theme_overrides.css index 585fdddb..849f9f25 100644 --- a/docs/_static/theme_overrides.css +++ b/docs/_static/theme_overrides.css @@ -4,6 +4,10 @@ white-space: normal; } +.wy-table-responsive > table > tbody > tr > td { + vertical-align: top !important; +} + .wy-table-responsive { margin-bottom: 24px; max-width: 100%; @@ -13,3 +17,28 @@ .wy-menu-vertical header, .wy-menu-vertical p.caption { color: #e0e0e0; } + +.code-block-caption { + height: 1.5em; +} + +.code-block-caption .caption-text { + font-size: 0.8em; + float: right; +} + +.code-block-caption .headerlink { + display: none !important; +} + +.function .headerlink { + display: none !important; +} + +dl .reference.internal { + display: none !important; +} + +dl .headerlink { + display: none !important; +} diff --git a/docs/conf.py b/docs/conf.py index 54a353ac..76dc83d4 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,11 +1,16 @@ import sys import os +import importlib +import inspect sys.path.insert(0, os.path.abspath('..')) import netlib.version + extensions = [ 'sphinx.ext.autodoc', 'sphinx.ext.doctest', + 'sphinx.ext.extlinks', + 'sphinx.ext.linkcode', 'sphinx.ext.viewcode', 'sphinx.ext.napoleon', 'sphinxcontrib.documentedlist' @@ -156,7 +161,7 @@ html_static_path = ['_static'] #html_split_index = False # If true, links to the reST sources are added to the pages. -#html_show_sourcelink = True +html_show_sourcelink = False # If true, "Created using Sphinx" is shown in the HTML footer. Default is True. #html_show_sphinx = True @@ -189,5 +194,43 @@ html_static_path = ['_static'] # Output file base name for HTML help builder. htmlhelp_basename = 'mitmproxydoc' + +# FIXME: change master to dynamic version before release +extlinks = dict( + src = ('https://github.com/mitmproxy/mitmproxy/blob/master/%s', '') +) + + +MODULE = "/mitmproxy/" + +def linkcode_resolve(domain, info): + if domain != 'py': + return None + module, fullname = info['module'], info['fullname'] + # TODO: attributes/properties don't have modules, maybe try to look + # them up based on their cached host object? + if not module: + return None + obj = importlib.import_module(module) + for item in fullname.split('.'): + obj = getattr(obj, item, None) + if obj is None: + return None + try: + obj = getattr(obj, '_orig') + except AttributeError: + pass + try: + obj_source_path = inspect.getsourcefile(obj) + _, line = inspect.getsourcelines(obj) + except (TypeError, IOError): + # obj doesn't have a module, or something + return None + off = obj_source_path.rfind(MODULE) + mpath = obj_source_path[off + len(MODULE):] + print(obj_source_path, mpath) + return "https://github.com/mitmproxy/mitmproxy/blob/master/%s" % mpath + + def setup(app): app.add_stylesheet('theme_overrides.css') diff --git a/docs/dev/models.rst b/docs/dev/models.rst deleted file mode 100644 index a333fb06..00000000 --- a/docs/dev/models.rst +++ /dev/null @@ -1,81 +0,0 @@ -.. _models: - -Datastructures -============== - -.. automodule:: mitmproxy.models - :members: HTTPFlow, HTTPRequest, HTTPResponse - - -.. automodule:: netlib.http - - .. autoclass:: Request - - .. rubric:: Data - .. autoattribute:: first_line_format - .. autoattribute:: method - .. autoattribute:: scheme - .. autoattribute:: host - .. autoattribute:: port - .. autoattribute:: path - .. autoattribute:: http_version - .. autoattribute:: headers - .. autoattribute:: content - .. autoattribute:: timestamp_start - .. autoattribute:: timestamp_end - .. rubric:: Computed Properties and Convenience Methods - .. autoattribute:: text - .. autoattribute:: url - .. autoattribute:: pretty_host - .. autoattribute:: pretty_url - .. autoattribute:: query - .. autoattribute:: cookies - .. autoattribute:: path_components - .. automethod:: anticache - .. automethod:: anticomp - .. automethod:: constrain_encoding - .. autoattribute:: urlencoded_form - .. autoattribute:: multipart_form - - .. autoclass:: Response - - .. automethod:: make - - .. rubric:: Data - .. autoattribute:: http_version - .. autoattribute:: status_code - .. autoattribute:: reason - .. autoattribute:: headers - .. autoattribute:: content - .. autoattribute:: timestamp_start - .. autoattribute:: timestamp_end - .. rubric:: Computed Properties and Convenience Methods - .. autoattribute:: text - .. autoattribute:: cookies - - .. autoclass:: Headers - :members: - :special-members: - :no-undoc-members: - -.. automodule:: netlib.multidict - - .. autoclass:: MultiDictView - - .. automethod:: get_all - .. automethod:: set_all - .. automethod:: add - .. automethod:: insert - .. automethod:: keys - .. automethod:: values - .. automethod:: items - .. automethod:: to_dict - -.. autoclass:: mitmproxy.models.Error - :show-inheritance: - -.. autoclass:: mitmproxy.models.ServerConnection - :show-inheritance: - -.. autoclass:: mitmproxy.models.ClientConnection - :show-inheritance: \ No newline at end of file diff --git a/docs/features/clientreplay.rst b/docs/features/clientreplay.rst index 50740bcf..ebe40b5f 100644 --- a/docs/features/clientreplay.rst +++ b/docs/features/clientreplay.rst @@ -14,5 +14,5 @@ You may want to use client-side replay in conjunction with the ================== =========== command-line ``-c path`` -mitmproxy shortcut :kbd:`c` +mitmproxy shortcut :kbd:`R` then :kbd:`c` ================== =========== diff --git a/docs/features/responsestreaming.rst b/docs/features/responsestreaming.rst index 66b5cae0..1d5726c4 100644 --- a/docs/features/responsestreaming.rst +++ b/docs/features/responsestreaming.rst @@ -35,10 +35,10 @@ command-line ``--stream SIZE`` Customizing Response Streaming ------------------------------ -You can also use an :ref:`inlinescripts` to customize exactly -which responses are streamed. +You can also use a script to customize exactly which responses are streamed. -Responses that should be tagged for streaming by setting their ``.stream`` attribute to ``True``: +Responses that should be tagged for streaming by setting their ``.stream`` +attribute to ``True``: .. literalinclude:: ../../examples/stream.py :caption: examples/stream.py diff --git a/docs/features/serverreplay.rst b/docs/features/serverreplay.rst index d70b6514..f545d4a5 100644 --- a/docs/features/serverreplay.rst +++ b/docs/features/serverreplay.rst @@ -35,5 +35,5 @@ the :kbd:`o` options shortcut within :program:`mitmproxy`. ================== =========== command-line ``-S path`` -mitmproxy shortcut :kbd:`S` +mitmproxy shortcut :kbd:`R` then :kbd:`s` ================== =========== diff --git a/docs/index.rst b/docs/index.rst index 143f79f7..9a948678 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -51,9 +51,10 @@ :hidden: :caption: Scripting - scripting/inlinescripts - dev/models - scripting/mitmproxy + scripting/overview + scripting/context + scripting/events + scripting/api .. toctree:: diff --git a/docs/scripting/api.rst b/docs/scripting/api.rst new file mode 100644 index 00000000..03d35920 --- /dev/null +++ b/docs/scripting/api.rst @@ -0,0 +1,8 @@ +.. _api: + +API +==== + +.. automodule:: mitmproxy.models.http + :inherited-members: + :members: HTTPFlow, HTTPRequest, HTTPResponse diff --git a/docs/scripting/context.rst b/docs/scripting/context.rst new file mode 100644 index 00000000..7c351598 --- /dev/null +++ b/docs/scripting/context.rst @@ -0,0 +1,4 @@ +.. _context: + +Context +======= diff --git a/docs/scripting/events.rst b/docs/scripting/events.rst new file mode 100644 index 00000000..c16c01f6 --- /dev/null +++ b/docs/scripting/events.rst @@ -0,0 +1,202 @@ +.. _events: + +Events +======= + +General +------- + +.. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - .. py:function:: configure(options, updated) + - Called once on startup, and whenever options change. + + *options* + An ``options.Options`` object with the total current configuration + state of mitmproxy. + *updated* + A set of strings indicating which configuration options have been + updated. This contains all options when *configure* is called on + startup, and only changed options subsequently. + + * - .. py:function:: done() + - Called once when the script shuts down, either because it's been + unloaded, or because the proxy itself is shutting down. + + * - .. py:function:: log(entry) + - Called whenever an event log is added. + + *entry* + An ``controller.LogEntry`` object - ``entry.msg`` is the log text, + and ``entry.level`` is the urgency level ("debug", "info", "warn", + "error"). + + * - .. py:function:: start() + - Called once on startup, before any other events. If you return a + value from this event, it will replace the current addon. This + allows you to, "boot into" an addon implemented as a class instance + from the module level. + + * - .. py:function:: tick() + - Called at a regular sub-second interval as long as the addon is + executing. + + +Connection +---------- + +.. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - .. py:function:: clientconnect(root_layer) + - Called when a client initiates a connection to the proxy. Note that a + connection can correspond to multiple HTTP requests. + + *root_layer* + The root layer (see `mitmproxy.protocol` for an explanation what + the root layer is), provides transparent access to all attributes + of the :py:class:`~mitmproxy.proxy.RootContext`. For example, + ``root_layer.client_conn.address`` gives the remote address of the + connecting client. + + * - .. py:function:: clientdisconnect(root_layer) + - Called when a client disconnects from the proxy. + + *root_layer* + The root layer object. + + * - .. py:function:: next_layer(layer) + + - Called whenever layers are switched. You may change which layer will + be used by returning a new layer object from this event. + + *layer* + The next layer, as determined by mitmpmroxy. + + * - .. py:function:: serverconnect(server_conn) + - Called before the proxy initiates a connection to the target server. + Note that a connection can correspond to multiple HTTP requests. + + *server_conn* + A ``ServerConnection`` object. It is guaranteed to have a non-None + ``address`` attribute. + + * - .. py:function:: serverdisconnect(server_conn) + - Called when the proxy has closed the server connection. + + *server_conn* + A ``ServerConnection`` object. + + +HTTP Events +----------- + +.. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - .. py:function:: request(flow) + - Called when a client request has been received. + + *flow* + A ``models.HTTPFlow`` object. At this point, the flow is + guaranteed to have a non-None ``request`` attribute. + + * - .. py:function:: requestheaders(flow) + - Called when the headers of a client request have been received, but + before the request body is read. + + *flow* + A ``models.HTTPFlow`` object. At this point, the flow is + guaranteed to have a non-None ``request`` attribute. + + * - .. py:function:: responseheaders(flow) + + - Called when the headers of a server response have been received, but + before the response body is read. + + *flow* + A ``models.HTTPFlow`` object. At this point, the flow is + guaranteed to have a non-none ``request`` and ``response`` + attributes, however the response will have no content. + + * - .. py:function:: response(flow) + + - Called when a server response has been received. + + *flow* + A ``models.HTTPFlow`` object. At this point, the flow is + guaranteed to have a non-none ``request`` and ``response`` + attributes. The raw response body will be in ``response.body``, + unless response streaming has been enabled. + + * - .. py:function:: error(flow) + - Called when a flow error has occurred, e.g. invalid server responses, + or interrupted connections. This is distinct from a valid server HTTP + error response, which is simply a response with an HTTP error code. + + *flow* + The flow containing the error. It is guaranteed to have + non-None ``error`` attribute. + + +WebSocket Events +----------------- + +.. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - .. py:function:: websockets_handshake(flow) + + - Called when a client wants to establish a WebSockets connection. The + WebSockets-specific headers can be manipulated to manipulate the + handshake. The ``flow`` object is guaranteed to have a non-None + ``request`` attribute. + + *flow* + The flow containing the HTTP websocket handshake request. The + object is guaranteed to have a non-None ``request`` attribute. + + +TCP Events +---------- + +These events are called only if the connection is in :ref:`TCP mode +`. So, for instance, TCP events are not called for ordinary HTTP/S +connections. + +.. list-table:: + :widths: 40 60 + :header-rows: 0 + + * - .. py:function:: tcp_end(flow) + - Called when TCP streaming ends. + + *flow* + A ``models.TCPFlow`` object. + + * - .. py:function:: tcp_error(flow) + - Called when a TCP error occurs - e.g. the connection closing + unexpectedly. + + *flow* + A ``models.TCPFlow`` object. + + * - .. py:function:: tcp_message(flow) + + - Called a TCP payload is received from the client or server. The + sender and receiver are identifiable. The most recent message will be + ``flow.messages[-1]``. The message is user-modifiable. + + *flow* + A ``models.TCPFlow`` object. + + * - .. py:function:: tcp_start(flow) + - Called when TCP streaming starts. + + *flow* + A ``models.TCPFlow`` object. diff --git a/docs/scripting/inlinescripts.rst b/docs/scripting/inlinescripts.rst deleted file mode 100644 index 74d4e714..00000000 --- a/docs/scripting/inlinescripts.rst +++ /dev/null @@ -1,227 +0,0 @@ -.. _inlinescripts: - -Inline Scripts -============== - -**mitmproxy** has a powerful scripting API that allows you to modify flows -on-the-fly or rewrite previously saved flows locally. - -The mitmproxy scripting API is event driven - a script is simply a Python -module that exposes a set of event methods. Here's a complete mitmproxy script -that adds a new header to every HTTP response before it is returned to the -client: - -.. literalinclude:: ../../examples/add_header.py - :caption: examples/add_header.py - :language: python - -All events that deal with an HTTP request get an instance of :py:class:`~mitmproxy.models.HTTPFlow`, -which we can use to manipulate the response itself. - -We can now run this script using mitmdump or mitmproxy as follows: - ->>> mitmdump -s add_header.py - -The new header will be added to all responses passing through the proxy. - -Examples --------- - -mitmproxy comes with a variety of example inline scripts, which demonstrate many basic tasks. -We encourage you to either browse them locally or on `GitHub`_. - - -Events ------- - -Script Lifecycle Events -^^^^^^^^^^^^^^^^^^^^^^^ - -.. py:function:: start(context) - - Called once on startup, before any other events. - - :param List[str] argv: The inline scripts' arguments. - For example, ``mitmproxy -s 'example.py --foo 42'`` sets argv to ``["--foo", "42"]``. - -.. py:function:: done(context) - - Called once on script shutdown, after any other events. - -Connection Events -^^^^^^^^^^^^^^^^^ - -.. py:function:: clientconnect(context, root_layer) - - Called when a client initiates a connection to the proxy. Note that - a connection can correspond to multiple HTTP requests. - - .. versionchanged:: 0.14 - - :param Layer root_layer: The root layer, which provides transparent access to all attributes of the - :py:class:`~mitmproxy.proxy.RootContext`. For example, ``root_layer.client_conn.address`` - gives the remote address of the connecting client. - -.. py:function:: clientdisconnect(context, root_layer) - - Called when a client disconnects from the proxy. - - .. versionchanged:: 0.14 - - :param Layer root_layer: see :py:func:`clientconnect` - -.. py:function:: serverconnect(context, server_conn) - - Called before the proxy initiates a connection to the target server. Note that - a connection can correspond to multiple HTTP requests. - - :param ServerConnection server_conn: The server connection object. It is guaranteed to have a - non-None ``address`` attribute. - -.. py:function:: serverdisconnect(context, server_conn) - - Called when the proxy has closed the server connection. - - .. versionadded:: 0.14 - - :param ServerConnection server_conn: see :py:func:`serverconnect` - -HTTP Events -^^^^^^^^^^^ - -.. py:function:: request(context, flow) - - Called when a client request has been received. The ``flow`` object is - guaranteed to have a non-None ``request`` attribute. - - :param HTTPFlow flow: The flow containing the request which has been received. - The object is guaranteed to have a non-None ``request`` attribute. - -.. py:function:: responseheaders(context, flow) - - Called when the headers of a server response have been received. - This will always be called before the response hook. - - :param HTTPFlow flow: The flow containing the request and response. - The object is guaranteed to have non-None ``request`` and - ``response`` attributes. ``response.content`` will be ``None``, - as the response body has not been read yet. - -.. py:function:: response(context, flow) - - Called when a server response has been received. - - :param HTTPFlow flow: The flow containing the request and response. - The object is guaranteed to have non-None ``request`` and - ``response`` attributes. ``response.body`` will contain the raw response body, - unless response streaming has been enabled. - -.. py:function:: error(context, flow) - - Called when a flow error has occurred, e.g. invalid server responses, or - interrupted connections. This is distinct from a valid server HTTP error - response, which is simply a response with an HTTP error code. - - :param HTTPFlow flow: The flow containing the error. - It is guaranteed to have non-None ``error`` attribute. - -WebSockets Events -^^^^^^^^^^^^^^^^^ - -.. py:function:: websocket_handshake(context, flow) - - Called when a client wants to establish a WebSockets connection. - The WebSockets-specific headers can be manipulated to manipulate the handshake. - The ``flow`` object is guaranteed to have a non-None ``request`` attribute. - - :param HTTPFlow flow: The flow containing the request which has been received. - The object is guaranteed to have a non-None ``request`` attribute. - -TCP Events -^^^^^^^^^^ - -.. py:function:: tcp_message(context, tcp_msg) - - .. warning:: API is subject to change - - If the proxy is in :ref:`TCP mode `, this event is called when it - receives a TCP payload from the client or server. - - The sender and receiver are identifiable. The message is user-modifiable. - - :param TcpMessage tcp_msg: see *examples/tcp_message.py* - -API ---- - -The canonical API documentation is the code, which you can browse here, locally or on `GitHub`_. -*Use the Source, Luke!* - -The main classes you will deal with in writing mitmproxy scripts are: - -:py:class:`mitmproxy.flow.FlowMaster` - - The "heart" of mitmproxy, usually subclassed as :py:class:`mitmproxy.dump.DumpMaster` or - :py:class:`mitmproxy.console.ConsoleMaster`. -:py:class:`~mitmproxy.models.ClientConnection` - - Describes a client connection. -:py:class:`~mitmproxy.models.ServerConnection` - - Describes a server connection. -:py:class:`~mitmproxy.models.HTTPFlow` - - A collection of objects representing a single HTTP transaction. -:py:class:`~mitmproxy.models.HTTPRequest` - - An HTTP request. -:py:class:`~mitmproxy.models.HTTPResponse` - - An HTTP response. -:py:class:`~mitmproxy.models.Error` - - A communications error. -:py:class:`netlib.http.Headers` - - A dictionary-like object for managing HTTP headers. -:py:class:`netlib.certutils.SSLCert` - - Exposes information SSL certificates. - - -Running scripts in parallel ---------------------------- - -We have a single flow primitive, so when a script is blocking, other requests are not processed. -While that's usually a very desirable behaviour, blocking scripts can be run threaded by using the -:py:obj:`mitmproxy.script.concurrent` decorator. -**If your script does not block, you should avoid the overhead of the decorator.** - -.. literalinclude:: ../../examples/nonblocking.py - :caption: examples/nonblocking.py - :language: python - -Make scripts configurable with arguments ----------------------------------------- - -Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by -surrounding the script call with quotes, e.g. ```mitmdump -s 'script.py --foo 42'``. -The arguments are then exposed in the start event: - -.. literalinclude:: ../../examples/modify_response_body.py - :caption: examples/modify_response_body.py - :language: python - -Running scripts on saved flows ------------------------------- - -Sometimes, we want to run a script on :py:class:`~mitmproxy.models.Flow` objects that are already -complete. This happens when you start a script, and then load a saved set of flows from a file -(see the "scripted data transformation" example :ref:`here `). -It also happens when you run a one-shot script on a single flow through the ``|`` (pipe) shortcut -in mitmproxy. - -In this case, there are no client connections, and the events are run in the following order: -**start**, **request**, **responseheaders**, **response**, **error**, **done**. -If the flow doesn't have a **response** or **error** associated with it, the matching events will -be skipped. - -Spaces in the script path -------------------------- - -By default, spaces are interpreted as a separator between the inline script and its arguments -(e.g. ``-s 'foo.py 42'``). Consequently, the script path needs to be wrapped in a separate pair of -quotes if it contains spaces: ``-s '\'./foo bar/baz.py\' 42'``. - -.. _GitHub: https://github.com/mitmproxy/mitmproxy diff --git a/docs/scripting/mitmproxy.rst b/docs/scripting/mitmproxy.rst deleted file mode 100644 index 9e331736..00000000 --- a/docs/scripting/mitmproxy.rst +++ /dev/null @@ -1,26 +0,0 @@ - -FlowMaster -========== - -.. note:: - - We strongly encourage you to use :ref:`inlinescripts` rather than subclassing mitmproxy's FlowMaster. - - Inline Scripts are equally powerful and provide an easier syntax. - - Most examples are written as inline scripts. - - Multiple inline scripts can be used together. - - Inline Scripts can either be executed headless with mitmdump or within the mitmproxy UI. - - -All of mitmproxy's basic functionality is exposed through the **mitmproxy** -library. The example below shows a simple implementation of the "sticky cookie" -functionality included in the interactive mitmproxy program. Traffic is -monitored for ``Cookie`` and ``Set-Cookie`` headers, and requests are rewritten -to include a previously seen cookie if they don't already have one. In effect, -this lets you log in to a site using your browser, and then make subsequent -requests using a tool like curl, which will then seem to be part of the -authenticated session. - - -.. literalinclude:: ../../examples/stickycookies - :caption: examples/stickycookies - :language: python diff --git a/docs/scripting/overview.rst b/docs/scripting/overview.rst new file mode 100644 index 00000000..a0dfe111 --- /dev/null +++ b/docs/scripting/overview.rst @@ -0,0 +1,79 @@ +.. _overview: + +Overview +========= + +Mitmproxy has a powerful scripting API that allows you to control almost any +aspect of traffic being proxied. In fact, much of mitmproxy's own core +functionality is implemented using the exact same API exposed to scripters (see +:src:`mitmproxy/builtins`). + +Scripting is event driven, with named handlers on the script object called at +appropriate points of mitmproxy's operation. Here's a complete mitmproxy script +that adds a new header to every HTTP response before it is returned to the +client: + +.. literalinclude:: ../../examples/add_header.py + :caption: :src:`examples/add_header.py` + :language: python + +All events that deal with an HTTP request get an instance of +:py:class:`~mitmproxy.models.HTTPFlow`, which we can use to manipulate the +response itself. We can now run this script using mitmdump or mitmproxy as +follows: + +>>> mitmdump -s add_header.py + +The new header will be added to all responses passing through the proxy. + + +mitmproxy comes with a variety of example inline scripts, which demonstrate +many basic tasks. + + +Running scripts in parallel +--------------------------- + +We have a single flow primitive, so when a script is blocking, other requests are not processed. +While that's usually a very desirable behaviour, blocking scripts can be run threaded by using the +:py:obj:`mitmproxy.script.concurrent` decorator. +**If your script does not block, you should avoid the overhead of the decorator.** + +.. literalinclude:: ../../examples/nonblocking.py + :caption: examples/nonblocking.py + :language: python + +Make scripts configurable with arguments +---------------------------------------- + +Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by +surrounding the script call with quotes, e.g. ```mitmdump -s 'script.py --foo 42'``. +The arguments are then exposed in the start event: + +.. literalinclude:: ../../examples/modify_response_body.py + :caption: examples/modify_response_body.py + :language: python + + +Running scripts on saved flows +------------------------------ + +Sometimes, we want to run a script on :py:class:`~mitmproxy.models.Flow` objects that are already +complete. This happens when you start a script, and then load a saved set of flows from a file +(see the "scripted data transformation" example :ref:`here `). +It also happens when you run a one-shot script on a single flow through the ``|`` (pipe) shortcut +in mitmproxy. + +In this case, there are no client connections, and the events are run in the following order: +**start**, **request**, **responseheaders**, **response**, **error**, **done**. +If the flow doesn't have a **response** or **error** associated with it, the matching events will +be skipped. + +Spaces in the script path +------------------------- + +By default, spaces are interpreted as a separator between the inline script and its arguments +(e.g. ``-s 'foo.py 42'``). Consequently, the script path needs to be wrapped in a separate pair of +quotes if it contains spaces: ``-s '\'./foo bar/baz.py\' 42'``. + +.. _GitHub: https://github.com/mitmproxy/mitmproxy -- cgit v1.2.3 From 61040a7bcd46c057e34fe4671ef20b9111649e74 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 14 Oct 2016 12:18:56 +1300 Subject: docs: improve external source links, tweak code docs --- docs/conf.py | 29 ++++++++++++++--------------- docs/scripting/api.rst | 20 +++++++++++++++++--- 2 files changed, 31 insertions(+), 18 deletions(-) (limited to 'docs') diff --git a/docs/conf.py b/docs/conf.py index 76dc83d4..ef5f0556 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -195,20 +195,19 @@ html_show_sourcelink = False htmlhelp_basename = 'mitmproxydoc' +SRCBASE = "https://github.com/mitmproxy/mitmproxy/blob/master" + + # FIXME: change master to dynamic version before release extlinks = dict( - src = ('https://github.com/mitmproxy/mitmproxy/blob/master/%s', '') + src = (SRCBASE + r"/%s", '') ) -MODULE = "/mitmproxy/" - def linkcode_resolve(domain, info): if domain != 'py': return None module, fullname = info['module'], info['fullname'] - # TODO: attributes/properties don't have modules, maybe try to look - # them up based on their cached host object? if not module: return None obj = importlib.import_module(module) @@ -217,19 +216,19 @@ def linkcode_resolve(domain, info): if obj is None: return None try: - obj = getattr(obj, '_orig') - except AttributeError: - pass - try: - obj_source_path = inspect.getsourcefile(obj) + spath = inspect.getsourcefile(obj) _, line = inspect.getsourcelines(obj) except (TypeError, IOError): - # obj doesn't have a module, or something return None - off = obj_source_path.rfind(MODULE) - mpath = obj_source_path[off + len(MODULE):] - print(obj_source_path, mpath) - return "https://github.com/mitmproxy/mitmproxy/blob/master/%s" % mpath + if spath.rfind("netlib") > -1: + off = spath.rfind("netlib") + mpath = spath[off:] + elif spath.rfind("mitmproxy") > -1: + off = spath.rfind("mitmproxy") + mpath = spath[off:] + else: + return None + return SRCBASE + "/%s#L%s" % (mpath, line) def setup(app): diff --git a/docs/scripting/api.rst b/docs/scripting/api.rst index 03d35920..a1769329 100644 --- a/docs/scripting/api.rst +++ b/docs/scripting/api.rst @@ -1,8 +1,22 @@ .. _api: + API -==== +=== + +- HTTP + - `mitmproxy.models.http.HTTPRequest <#mitmproxy.models.http.HTTPRequest>`_ + - `mitmproxy.models.http.HTTPResponse <#mitmproxy.models.http.HTTPResponse>`_ + - `mitmproxy.models.http.HTTPFlow <#mitmproxy.models.http.HTTPFlow>`_ + +HTTP +---- + +.. autoclass:: mitmproxy.models.http.HTTPRequest + :inherited-members: + +.. autoclass:: mitmproxy.models.http.HTTPResponse + :inherited-members: -.. automodule:: mitmproxy.models.http +.. autoclass:: mitmproxy.models.http.HTTPFlow :inherited-members: - :members: HTTPFlow, HTTPRequest, HTTPResponse -- cgit v1.2.3 From fb69c9c3453142ae6beb4040295accb41fdc6878 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Oct 2016 11:12:58 +1300 Subject: docs: overview, classes, arguments --- docs/scripting/api.rst | 8 +++++ docs/scripting/overview.rst | 72 ++++++++++++++++++++++++++++++++------------- 2 files changed, 60 insertions(+), 20 deletions(-) (limited to 'docs') diff --git a/docs/scripting/api.rst b/docs/scripting/api.rst index a1769329..a864b442 100644 --- a/docs/scripting/api.rst +++ b/docs/scripting/api.rst @@ -8,6 +8,8 @@ API - `mitmproxy.models.http.HTTPRequest <#mitmproxy.models.http.HTTPRequest>`_ - `mitmproxy.models.http.HTTPResponse <#mitmproxy.models.http.HTTPResponse>`_ - `mitmproxy.models.http.HTTPFlow <#mitmproxy.models.http.HTTPFlow>`_ +- Errors + - `mitmproxy.models.flow.Error <#mitmproxy.models.flow.Error>`_ HTTP ---- @@ -20,3 +22,9 @@ HTTP .. autoclass:: mitmproxy.models.http.HTTPFlow :inherited-members: + +Errors +------ + +.. autoclass:: mitmproxy.models.flow.Error + :inherited-members: diff --git a/docs/scripting/overview.rst b/docs/scripting/overview.rst index a0dfe111..f8dd9f2e 100644 --- a/docs/scripting/overview.rst +++ b/docs/scripting/overview.rst @@ -1,13 +1,17 @@ .. _overview: -Overview -========= +Introduction +============ Mitmproxy has a powerful scripting API that allows you to control almost any aspect of traffic being proxied. In fact, much of mitmproxy's own core functionality is implemented using the exact same API exposed to scripters (see :src:`mitmproxy/builtins`). + +A simple example +---------------- + Scripting is event driven, with named handlers on the script object called at appropriate points of mitmproxy's operation. Here's a complete mitmproxy script that adds a new header to every HTTP response before it is returned to the @@ -17,18 +21,57 @@ client: :caption: :src:`examples/add_header.py` :language: python -All events that deal with an HTTP request get an instance of -:py:class:`~mitmproxy.models.HTTPFlow`, which we can use to manipulate the -response itself. We can now run this script using mitmdump or mitmproxy as -follows: +All events that deal with an HTTP request get an instance of `HTTPFlow +`_, which we can use to manipulate the +response itself. We can now run this script using mitmdump, and the new header +will be added to all responses passing through the proxy: >>> mitmdump -s add_header.py -The new header will be added to all responses passing through the proxy. +Using classes +------------- + +In the example above, the script object is the ``add_header`` module itself. +That is, the handlers are declared at the global level of the script. This is +great for quick hacks, but soon becomes limiting as scripts become more +sophisticated. + +When a script first starts up, the `start `_, event is +called before anything else happens. You can replace the current script object +by returning it from this handler. Here's how this looks when applied to the +example above: + +.. literalinclude:: ../../examples/classes.py + :caption: :src:`examples/classes.py` + :language: python + +So here, we're using a module-level script to "boot up" into a class instance. +From this point on, the module-level script is removed from the handler chain, +and is replaced by the class instance. + + +Handling arguments +------------------ -mitmproxy comes with a variety of example inline scripts, which demonstrate -many basic tasks. +Scripts can handle their own command-line arguments, just like any other Python +program. Let's build on the example above to do something slightly more +sophisticated - replace one value with another in all responses. Mitmproxy's +`HTTPRequest `_ and `HTTPResponse +`_ objects have a handy `replace +`_ method that takes care +of all the details for us. + +.. literalinclude:: ../../examples/arguments.py + :caption: :src:`examples/arguments.py` + :language: python + +We can now call this script on the command-line like this: + +>>> mitmdump -dd -s "./arguments.py html faketml" + +Whenever a handler is called, mitpmroxy rewrites the script environment so that +it sees its own arguments as if it was invoked from the command-line. Running scripts in parallel @@ -43,17 +86,6 @@ While that's usually a very desirable behaviour, blocking scripts can be run thr :caption: examples/nonblocking.py :language: python -Make scripts configurable with arguments ----------------------------------------- - -Sometimes, you want to pass runtime arguments to the inline script. This can be simply done by -surrounding the script call with quotes, e.g. ```mitmdump -s 'script.py --foo 42'``. -The arguments are then exposed in the start event: - -.. literalinclude:: ../../examples/modify_response_body.py - :caption: examples/modify_response_body.py - :language: python - Running scripts on saved flows ------------------------------ -- cgit v1.2.3 From 55cb2a85472de8698b3dabc7ddc920b930e355d9 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Oct 2016 12:03:57 +1300 Subject: docs: logging and the context --- docs/index.rst | 1 - docs/scripting/api.rst | 20 +++++++++++++++----- docs/scripting/context.rst | 4 ---- docs/scripting/overview.rst | 43 +++++++++++++++++++++++++++++-------------- 4 files changed, 44 insertions(+), 24 deletions(-) delete mode 100644 docs/scripting/context.rst (limited to 'docs') diff --git a/docs/index.rst b/docs/index.rst index 9a948678..cd32a1f6 100644 --- a/docs/index.rst +++ b/docs/index.rst @@ -52,7 +52,6 @@ :caption: Scripting scripting/overview - scripting/context scripting/events scripting/api diff --git a/docs/scripting/api.rst b/docs/scripting/api.rst index a864b442..abc9ff3e 100644 --- a/docs/scripting/api.rst +++ b/docs/scripting/api.rst @@ -4,12 +4,22 @@ API === +- Errors + - `mitmproxy.models.flow.Error <#mitmproxy.models.flow.Error>`_ - HTTP - `mitmproxy.models.http.HTTPRequest <#mitmproxy.models.http.HTTPRequest>`_ - `mitmproxy.models.http.HTTPResponse <#mitmproxy.models.http.HTTPResponse>`_ - `mitmproxy.models.http.HTTPFlow <#mitmproxy.models.http.HTTPFlow>`_ -- Errors - - `mitmproxy.models.flow.Error <#mitmproxy.models.flow.Error>`_ +- Logging + - `mitmproxy.controller.Log <#mitmproxy.controller.Log>`_ + - `mitmproxy.controller.LogEntry <#mitmproxy.controller.LogEntry>`_ + + +Errors +------ + +.. autoclass:: mitmproxy.models.flow.Error + :inherited-members: HTTP ---- @@ -23,8 +33,8 @@ HTTP .. autoclass:: mitmproxy.models.http.HTTPFlow :inherited-members: -Errors ------- +Logging +-------- -.. autoclass:: mitmproxy.models.flow.Error +.. autoclass:: mitmproxy.controller.Log :inherited-members: diff --git a/docs/scripting/context.rst b/docs/scripting/context.rst deleted file mode 100644 index 7c351598..00000000 --- a/docs/scripting/context.rst +++ /dev/null @@ -1,4 +0,0 @@ -.. _context: - -Context -======= diff --git a/docs/scripting/overview.rst b/docs/scripting/overview.rst index f8dd9f2e..a3b83e44 100644 --- a/docs/scripting/overview.rst +++ b/docs/scripting/overview.rst @@ -74,18 +74,24 @@ Whenever a handler is called, mitpmroxy rewrites the script environment so that it sees its own arguments as if it was invoked from the command-line. -Running scripts in parallel ---------------------------- +Logging and the context +----------------------- -We have a single flow primitive, so when a script is blocking, other requests are not processed. -While that's usually a very desirable behaviour, blocking scripts can be run threaded by using the -:py:obj:`mitmproxy.script.concurrent` decorator. -**If your script does not block, you should avoid the overhead of the decorator.** +Scripts should not output straight to stderr or stdout. Instead, the `log +`_ object on the ``ctx`` contexzt module +should be used, so that the mitmproxy host program can handle output +appropriately. So, mitmdump can print colorised sript output to the terminal, +and mitmproxy console can place script output in the event buffer. -.. literalinclude:: ../../examples/nonblocking.py - :caption: examples/nonblocking.py +Here's how this looks: + +.. literalinclude:: ../../examples/logging.py + :caption: :src:`examples/logging.py` :language: python +The ``ctx`` module also exposes the mitmproxy master object at ``ctx.master`` +for advanced usage. + Running scripts on saved flows ------------------------------ @@ -101,11 +107,20 @@ In this case, there are no client connections, and the events are run in the fol If the flow doesn't have a **response** or **error** associated with it, the matching events will be skipped. -Spaces in the script path -------------------------- -By default, spaces are interpreted as a separator between the inline script and its arguments -(e.g. ``-s 'foo.py 42'``). Consequently, the script path needs to be wrapped in a separate pair of -quotes if it contains spaces: ``-s '\'./foo bar/baz.py\' 42'``. +Concurrency +----------- + +We have a single flow primitive, so when a script is blocking, other requests +are not processed. While that's usually a very desirable behaviour, blocking +scripts can be run threaded by using the :py:obj:`mitmproxy.script.concurrent` +decorator. + +.. literalinclude:: ../../examples/nonblocking.py + :caption: :src:`examples/nonblocking.py` + :language: python + -.. _GitHub: https://github.com/mitmproxy/mitmproxy + +Developing scripts +------------------ -- cgit v1.2.3 From 072fff90f119375395a9b1b2fbef9667a46f7236 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 15 Oct 2016 18:00:21 -0700 Subject: docs: link to correct tag --- docs/conf.py | 21 ++++++++++++++++----- 1 file changed, 16 insertions(+), 5 deletions(-) (limited to 'docs') diff --git a/docs/conf.py b/docs/conf.py index ef5f0556..e1cbc497 100644 --- a/docs/conf.py +++ b/docs/conf.py @@ -1,7 +1,9 @@ -import sys -import os import importlib import inspect +import os +import subprocess +import sys + sys.path.insert(0, os.path.abspath('..')) import netlib.version @@ -194,11 +196,20 @@ html_show_sourcelink = False # Output file base name for HTML help builder. htmlhelp_basename = 'mitmproxydoc' +last_tag, tag_dist, commit = ( + subprocess.check_output(["git", "describe", "--tags", "--long"]) + .decode() + .strip() + .rsplit("-", 2) +) +tag_dist = int(tag_dist) +if tag_dist == 0: + tag = last_tag +else: + tag = "master" -SRCBASE = "https://github.com/mitmproxy/mitmproxy/blob/master" - +SRCBASE = "https://github.com/mitmproxy/mitmproxy/blob/{}".format(tag) -# FIXME: change master to dynamic version before release extlinks = dict( src = (SRCBASE + r"/%s", '') ) -- cgit v1.2.3 From 57b8ed21a9a30eb79d9340d5e146e42bbafd0d46 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Oct 2016 18:25:59 +1300 Subject: docs: scripts on saved flows --- docs/scripting/overview.rst | 30 +++++++++++++++++++++++------- 1 file changed, 23 insertions(+), 7 deletions(-) (limited to 'docs') diff --git a/docs/scripting/overview.rst b/docs/scripting/overview.rst index a3b83e44..744f5eb4 100644 --- a/docs/scripting/overview.rst +++ b/docs/scripting/overview.rst @@ -1,7 +1,7 @@ .. _overview: -Introduction -============ +Overview +======== Mitmproxy has a powerful scripting API that allows you to control almost any aspect of traffic being proxied. In fact, much of mitmproxy's own core @@ -96,11 +96,27 @@ for advanced usage. Running scripts on saved flows ------------------------------ -Sometimes, we want to run a script on :py:class:`~mitmproxy.models.Flow` objects that are already -complete. This happens when you start a script, and then load a saved set of flows from a file -(see the "scripted data transformation" example :ref:`here `). -It also happens when you run a one-shot script on a single flow through the ``|`` (pipe) shortcut -in mitmproxy. +When a flow is loaded from disk, the sequence of events that the flow would +have gone through on the wire is partially replayed. So, for instance, an HTTP +flow loaded from disk will trigger `requestheaders +`_, `request `_, +`responseheaders `_ and `response +`_ in order. We can use this behaviour to transform saved +traffic using scripts. For example, we can invoke the replacer script from +above on saved traffic as follows: + +>>> mitmdump -dd -s "./arguments.py html faketml" + + + + + +:py:class:`~mitmproxy.models.Flow` +objects that are already complete. This happens when you start a script, and +then load a saved set of flows from a file (see the "scripted data +transformation" example :ref:`here `). It also happens when you run a +one-shot script on a single flow through the ``|`` (pipe) shortcut in +mitmproxy. In this case, there are no client connections, and the events are run in the following order: **start**, **request**, **responseheaders**, **response**, **error**, **done**. -- cgit v1.2.3 From 00603021d9d486e3e16511eee273d26f59a3ab10 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 16 Oct 2016 20:21:43 +1300 Subject: docs: concurrency, developing scripts --- docs/scripting/overview.rst | 39 +++++++++++++++++++-------------------- 1 file changed, 19 insertions(+), 20 deletions(-) (limited to 'docs') diff --git a/docs/scripting/overview.rst b/docs/scripting/overview.rst index 744f5eb4..5966eb1d 100644 --- a/docs/scripting/overview.rst +++ b/docs/scripting/overview.rst @@ -105,38 +105,37 @@ flow loaded from disk will trigger `requestheaders traffic using scripts. For example, we can invoke the replacer script from above on saved traffic as follows: ->>> mitmdump -dd -s "./arguments.py html faketml" - - - +>>> mitmdump -dd -s "./arguments.py html fakehtml" -r saved -w changed +This command starts the ``arguments`` script, reads all the flows from +``saved`` transforming them in the process, then writes them all to +``changed``. -:py:class:`~mitmproxy.models.Flow` -objects that are already complete. This happens when you start a script, and -then load a saved set of flows from a file (see the "scripted data -transformation" example :ref:`here `). It also happens when you run a -one-shot script on a single flow through the ``|`` (pipe) shortcut in -mitmproxy. - -In this case, there are no client connections, and the events are run in the following order: -**start**, **request**, **responseheaders**, **response**, **error**, **done**. -If the flow doesn't have a **response** or **error** associated with it, the matching events will -be skipped. +The mitmproxy console tool provides interactive ways to run transforming +scripts on flows - for instance, you can run a one-shot script on a single flow +through the ``|`` (pipe) shortcut. Concurrency ----------- -We have a single flow primitive, so when a script is blocking, other requests -are not processed. While that's usually a very desirable behaviour, blocking -scripts can be run threaded by using the :py:obj:`mitmproxy.script.concurrent` -decorator. +The mitmproxy script mechanism is single threaded, and the proxy blocks while +script handlers execute. This hugely simplifies the most common case, where +handlers are light-weight and the blocking doesn't have a performance impact. +It's possible to implement a concurrent mechanism on top of the blocking +framework, and mitmproxy includes a handy example of this that is fit for most +purposes. You can use it as follows: .. literalinclude:: ../../examples/nonblocking.py :caption: :src:`examples/nonblocking.py` :language: python - Developing scripts ------------------ + +Mitmprxoy monitors scripts for modifications, and reloads them on change. When +this happens, the script is shut down (the `done `_ event is +called), and the new instance is started up as if the script had just been +loaded (the `start `_ and `configure +`_ events are called). -- cgit v1.2.3