All new documentation

This patch does a lot.

- Ditch sphinx in favor of hugo. This gives us complete control of the layout
and presentation of our docs. Henceforth, docs will be hosted on our website
rather than ReadTheDocs.
- Create a simple, clean doc layout and theme.
- Remove large parts of the documentaion. I've ditched anything that was a)
woefully out of date, b) too detailed, or c) too hard to maintain in the long
term.
- Huge updates to the docs themselves: completely rewrite addons documentation,
add docs for core concepts like commands and options, and revise and tweak a
lot of the existing docs.

With this patch, we're also changing the way we publish and maintain the docs.
From now on, we don't publish docs for every release. Instead, the website will
contain ONE set of docs for each major release. The online docs will be updated
if needed as minor releases are made. Docs are free to improve during minor
releases, but anything that changes behaviour sufficiently to require a doc
change warrants a new major release. This also leaves us free to progressively
update and improve docs out of step with our release cadence.

With this new scheme, I feel CI over the docs is less important. I've removed
it for now, but won't object if someone wants to add it back in.

9 files changed, 332 insertions, 0 deletions

diff --git a/examples/addons/addheader.py b/examples/addons/addheader.py
new file mode 100644
index 00000000..f4b29268
--- /dev/null
+++ b/examples/addons/addheader.py
@@ -0,0 +1,13 @@
+
+class AddHeader:
+    def __init__(self):
+        self.num = 0
+
+    def response(self, flow):
+        self.num = self.num + 1
+        flow.response.headers["count"] = str(self.num)
+
+
+addons = [
+    AddHeader()
+]
diff --git a/examples/addons/anatomy.py b/examples/addons/anatomy.py
new file mode 100644
index 00000000..c60afeaa
--- /dev/null
+++ b/examples/addons/anatomy.py
@@ -0,0 +1,15 @@
+from mitmproxy import ctx
+
+
+class Counter:
+    def __init__(self):
+        self.num = 0
+
+    def request(self, flow):
+        self.num = self.num + 1
+        ctx.log.info("We've seen %d flows" % self.num)
+
+
+addons = [
+    Counter()
+]
diff --git a/examples/addons/commands-flows.py b/examples/addons/commands-flows.py
new file mode 100644
index 00000000..cebc8f9d
--- /dev/null
+++ b/examples/addons/commands-flows.py
@@ -0,0 +1,21 @@
+import typing
+
+from mitmproxy import command
+from mitmproxy import ctx
+from mitmproxy import flow
+
+
+class MyAddon:
+    def __init__(self):
+        self.num = 0
+
+    @command.command("myaddon.addheader")
+    def addheader(self, flows: typing.Sequence[flow.Flow]) -> None:
+        for f in flows:
+            f.request.headers["myheader"] = "value"
+        ctx.log.alert("done")
+
+
+addons = [
+    MyAddon()
+]
diff --git a/examples/addons/commands-paths.py b/examples/addons/commands-paths.py
new file mode 100644
index 00000000..f37a0fbc
--- /dev/null
+++ b/examples/addons/commands-paths.py
@@ -0,0 +1,32 @@
+import typing
+
+from mitmproxy import command
+from mitmproxy import ctx
+from mitmproxy import flow
+from mitmproxy import types
+
+
+class MyAddon:
+    def __init__(self):
+        self.num = 0
+
+    @command.command("myaddon.histogram")
+    def histogram(
+        self,
+        flows: typing.Sequence[flow.Flow],
+        path: types.Path,
+    ) -> None:
+        totals = {}
+        for f in flows:
+            totals[f.request.host] = totals.setdefault(f.request.host, 0) + 1
+
+        fp = open(path, "w+")
+        for cnt, dom in sorted([(v, k) for (k, v) in totals.items()]):
+            fp.write("%s: %s\n" % (cnt, dom))
+
+        ctx.log.alert("done")
+
+
+addons = [
+    MyAddon()
+]
diff --git a/examples/addons/commands-simple.py b/examples/addons/commands-simple.py
new file mode 100644
index 00000000..c9cd6341
--- /dev/null
+++ b/examples/addons/commands-simple.py
@@ -0,0 +1,17 @@
+from mitmproxy import command
+from mitmproxy import ctx
+
+
+class MyAddon:
+    def __init__(self):
+        self.num = 0
+
+    @command.command("myaddon.inc")
+    def inc(self) -> None:
+        self.num += 1
+        ctx.log.info("num = %s" % self.num)
+
+
+addons = [
+    MyAddon()
+]
diff --git a/examples/addons/events.py b/examples/addons/events.py
new file mode 100644
index 00000000..93664954
--- /dev/null
+++ b/examples/addons/events.py
@@ -0,0 +1,179 @@
+import typing
+
+import mitmproxy.addonmanager
+import mitmproxy.connections
+import mitmproxy.http
+import mitmproxy.log
+import mitmproxy.tcp
+import mitmproxy.websocket
+import mitmproxy.proxy.protocol
+
+
+class Events:
+    # HTTP lifecycle
+    def http_connect(self, flow: mitmproxy.http.HTTPFlow):
+        """
+            An HTTP CONNECT request was received. Setting a non 2xx response on
+            the flow will return the response to the client abort the
+            connection. CONNECT requests and responses do not generate the usual
+            HTTP handler events. CONNECT requests are only valid in regular and
+            upstream proxy modes.
+        """
+
+    def requestheaders(self, flow: mitmproxy.http.HTTPFlow):
+        """
+            HTTP request headers were successfully read. At this point, the body
+            is empty.
+        """
+
+    def request(self, flow: mitmproxy.http.HTTPFlow):
+        """
+            The full HTTP request has been read.
+        """
+
+    def responseheaders(self, flow: mitmproxy.http.HTTPFlow):
+        """
+            HTTP response headers were successfully read. At this point, the body
+            is empty.
+        """
+
+    def response(self, flow: mitmproxy.http.HTTPFlow):
+        """
+            The full HTTP response has been read.
+        """
+
+    def error(self, flow: mitmproxy.http.HTTPFlow):
+        """
+            An HTTP error has occurred, e.g. invalid server responses, or
+            interrupted connections. This is distinct from a valid server HTTP
+            error response, which is simply a response with an HTTP error code.
+        """
+
+    # TCP lifecycle
+    def tcp_start(self, flow: mitmproxy.tcp.TCPFlow):
+        """
+            A TCP connection has started.
+        """
+
+    def tcp_message(self, flow: mitmproxy.tcp.TCPFlow):
+        """
+            A TCP connection has received a message. The most recent message
+            will be flow.messages[-1]. The message is user-modifiable.
+        """
+
+    def tcp_error(self, flow: mitmproxy.tcp.TCPFlow):
+        """
+            A TCP error has occurred.
+        """
+
+    def tcp_end(self, flow: mitmproxy.tcp.TCPFlow):
+        """
+            A TCP connection has ended.
+        """
+
+    # Websocket lifecycle
+    def websocket_handshake(self, flow: mitmproxy.http.HTTPFlow):
+        """
+            Called when a client wants to establish a WebSocket connection. The
+            WebSocket-specific headers can be manipulated to alter the
+            handshake. The flow object is guaranteed to have a non-None request
+            attribute.
+        """
+
+    def websocket_start(self, flow: mitmproxy.websocket.WebsocketFlow):
+        """
+            A websocket connection has commenced.
+        """
+
+    def websocket_message(self, flow: mitmproxy.websocket.WebsocketFlow):
+        """
+            Called when a WebSocket message is received from the client or
+            server. The most recent message will be flow.messages[-1]. The
+            message is user-modifiable. Currently there are two types of
+            messages, corresponding to the BINARY and TEXT frame types.
+        """
+
+    def websocket_error(self, flow: mitmproxy.websocket.WebsocketFlow):
+        """
+            A websocket connection has had an error.
+        """
+
+    def websocket_end(self, flow: mitmproxy.websocket.WebsocketFlow):
+        """
+            A websocket connection has ended.
+        """
+
+    # Network lifecycle
+    def clientconnect(self, layer: mitmproxy.proxy.protocol.Layer):
+        """
+            A client has connected to mitmproxy. Note that a connection can
+            correspond to multiple HTTP requests.
+        """
+
+    def clientdisconnect(self, layer: mitmproxy.proxy.protocol.Layer):
+        """
+            A client has disconnected from mitmproxy.
+        """
+
+    def serverconnect(self, conn: mitmproxy.connections.ServerConnection):
+        """
+            Mitmproxy has connected to a server. Note that a connection can
+            correspond to multiple requests.
+        """
+
+    def serverdisconnect(self, conn: mitmproxy.connections.ServerConnection):
+        """
+            Mitmproxy has disconnected from a server.
+        """
+
+    def next_layer(self, layer: mitmproxy.proxy.protocol.Layer):
+        """
+            Network layers are being switched. You may change which layer will
+            be used by returning a new layer object from this event.
+        """
+
+    # General lifecycle
+    def configure(self, updated: typing.Set[str]):
+        """
+            Called when configuration changes. The updated argument is a
+            set-like object containing the keys of all changed options. This
+            event is called during startup with all options in the updated set.
+        """
+
+    def done(self):
+        """
+            Called when the addon shuts down, either by being removed from the
+            mitmproxy instance, or when mitmproxy itself shuts down.
+        """
+
+    def load(self, entry: mitmproxy.addonmanager.Loader):
+        """
+            Called when an addon is first loaded. This event receives a Loader
+            object, which contains methods for adding options and commands. This
+            method is where the addon configures itself.
+        """
+
+    def log(self, entry: mitmproxy.log.LogEntry):
+        """
+            Called whenver a new log entry is created through the mitmproxy
+            context. Be careful not to log from this event, which will cause an
+            infinite loop!
+        """
+
+    def running(self):
+        """
+            Called when the proxy is completely up and running. At this point,
+            you can expect the proxy to be bound to a port, and all addons to be
+            loaded.
+        """
+
+    def tick(self):
+        """
+            A regular ticker - called approximately once every 100ms.
+        """
+
+    def update(self, flows: typing.Sequence[mitmproxy.flow.Flow]):
+        """
+            Update is called when one or more flow objects have been modified,
+            usually from a different addon.
+        """
diff --git a/examples/addons/options-configure.py b/examples/addons/options-configure.py
new file mode 100644
index 00000000..c7638e87
--- /dev/null
+++ b/examples/addons/options-configure.py
@@ -0,0 +1,28 @@
+import typing
+
+from mitmproxy import ctx
+from mitmproxy import exceptions
+
+
+class AddHeader:
+    def load(self, loader):
+        loader.add_option(
+            name = "addheader",
+            typespec = typing.Optional[int],
+            default = None,
+            help = "Add a header to responses",
+        )
+
+    def configure(self, updates):
+        if "addheader" in updates:
+            if ctx.options.addheader is not None and ctx.options.addheader > 100:
+                raise exceptions.OptionsError("addheader must be <= 100")
+
+    def response(self, flow):
+        if ctx.options.addheader is not None:
+            flow.response.headers["addheader"] = str(ctx.options.addheader)
+
+
+addons = [
+    AddHeader()
+]
diff --git a/examples/addons/options-simple.py b/examples/addons/options-simple.py
new file mode 100644
index 00000000..0acefb3f
--- /dev/null
+++ b/examples/addons/options-simple.py
@@ -0,0 +1,24 @@
+from mitmproxy import ctx
+
+
+class AddHeader:
+    def __init__(self):
+        self.num = 0
+
+    def load(self, loader):
+        loader.add_option(
+            name = "addheader",
+            typespec = bool,
+            default = False,
+            help = "Add a count header to responses",
+        )
+
+    def response(self, flow):
+        if ctx.options.addheader:
+            self.num = self.num + 1
+            flow.response.headers["count"] = str(self.num)
+
+
+addons = [
+    AddHeader()
+]
diff --git a/examples/addons/scripting.py b/examples/addons/scripting.py
new file mode 100644
index 00000000..8b23680e
--- /dev/null
+++ b/examples/addons/scripting.py
@@ -0,0 +1,3 @@
+
+def request(flow):
+    flow.request.headers["myheader"] = "value"