Merge branch 'master' into master

18 files changed, 98 insertions, 161 deletions

diff --git a/mitmproxy/addons/block.py b/mitmproxy/addons/block.py
index 91f9f709..4ccde0e1 100644
--- a/mitmproxy/addons/block.py
+++ b/mitmproxy/addons/block.py
@@ -36,4 +36,4 @@ class Block:
             layer.reply.kill()
         if ctx.options.block_global and address.is_global:
             ctx.log.warn("Client connection from %s killed by block_global" % astr)
-            layer.reply.kill()
\ No newline at end of file
+            layer.reply.kill()
diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py
index c56c0e74..7bdaeb33 100644
--- a/mitmproxy/addons/clientplayback.py
+++ b/mitmproxy/addons/clientplayback.py
@@ -203,8 +203,9 @@ class ClientPlayback:
             # https://github.com/mitmproxy/mitmproxy/issues/2197
             if hf.request.http_version == "HTTP/2.0":
                 hf.request.http_version = "HTTP/1.1"
-                host = hf.request.headers.pop(":authority")
-                hf.request.headers.insert(0, "host", host)
+                host = hf.request.headers.pop(":authority", None)
+                if host is not None:
+                    hf.request.headers.insert(0, "host", host)
             self.q.put(hf)
         ctx.master.addons.trigger("update", lst)
 
diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py
index a908dbb3..5c9bbcd0 100644
--- a/mitmproxy/addons/core.py
+++ b/mitmproxy/addons/core.py
@@ -289,7 +289,7 @@ class Core:
         """
             The possible values for an encoding specification.
         """
-        return ["gzip", "deflate", "br"]
+        return ["gzip", "deflate", "br", "zstd"]
 
     @command.command("options.load")
     def options_load(self, path: mitmproxy.types.Path) -> None:
diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py
index 6bb52e84..9aff2878 100644
--- a/mitmproxy/addons/cut.py
+++ b/mitmproxy/addons/cut.py
@@ -126,20 +126,18 @@ class Cut:
             format is UTF-8 encoded CSV. If there is exactly one row and one
             column, the data is written to file as-is, with raw bytes preserved.
         """
+        v: typing.Union[str, bytes]
         fp = io.StringIO(newline="")
         if len(cuts) == 1 and len(flows) == 1:
             v = extract(cuts[0], flows[0])
-            if isinstance(v, bytes):
-                fp.write(strutils.always_str(v))
-            else:
-                fp.write(v)
+            fp.write(strutils.always_str(v))  # type: ignore
             ctx.log.alert("Clipped single cut.")
         else:
             writer = csv.writer(fp)
             for f in flows:
                 vals = [extract(c, f) for c in cuts]
                 writer.writerow(
-                    [strutils.always_str(v) or "" for v in vals]  # type: ignore
+                    [strutils.always_str(v) for v in vals]
                 )
             ctx.log.alert("Clipped %s cuts as CSV." % len(cuts))
         try:
diff --git a/mitmproxy/addons/eventstore.py b/mitmproxy/addons/eventstore.py
index 50fea7ab..188a3b39 100644
--- a/mitmproxy/addons/eventstore.py
+++ b/mitmproxy/addons/eventstore.py
@@ -14,7 +14,7 @@ class EventStore:
         self.sig_refresh = blinker.Signal()
 
     @property
-    def size(self) -> int:
+    def size(self) -> typing.Optional[int]:
         return self.data.maxlen
 
     def log(self, entry: LogEntry) -> None:
diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py
index 90e95d3e..2776118a 100644
--- a/mitmproxy/addons/export.py
+++ b/mitmproxy/addons/export.py
@@ -11,17 +11,23 @@ import mitmproxy.types
 import pyperclip
 
 
-def raise_if_missing_request(f: flow.Flow) -> None:
+def cleanup_request(f: flow.Flow):
     if not hasattr(f, "request"):
         raise exceptions.CommandError("Can't export flow with no request.")
+    request = f.request.copy()  # type: ignore
+    request.decode(strict=False)
+    # a bit of clean-up
+    if request.method == 'GET' and request.headers.get("content-length", None) == "0":
+        request.headers.pop('content-length')
+    request.headers.pop(':authority', None)
+    return request
 
 
 def curl_command(f: flow.Flow) -> str:
-    raise_if_missing_request(f)
     data = "curl "
-    request = f.request.copy()  # type: ignore
-    request.decode(strict=False)
+    request = cleanup_request(f)
     for k, v in request.headers.items(multi=True):
+        data += "--compressed " if k == 'accept-encoding' else ""
         data += "-H '%s:%s' " % (k, v)
     if request.method != "GET":
         data += "-X %s " % request.method
@@ -35,11 +41,8 @@ def curl_command(f: flow.Flow) -> str:
 
 
 def httpie_command(f: flow.Flow) -> str:
-    raise_if_missing_request(f)
-    request = f.request.copy()  # type: ignore
-    data = "http %s " % request.method
-    request.decode(strict=False)
-    data += "%s" % request.url
+    request = cleanup_request(f)
+    data = "http %s %s" % (request.method, request.url)
     for k, v in request.headers.items(multi=True):
         data += " '%s:%s'" % (k, v)
     if request.content:
@@ -51,8 +54,7 @@ def httpie_command(f: flow.Flow) -> str:
 
 
 def raw(f: flow.Flow) -> bytes:
-    raise_if_missing_request(f)
-    return assemble.assemble_request(f.request)  # type: ignore
+    return assemble.assemble_request(cleanup_request(f))  # type: ignore
 
 
 formats = dict(
diff --git a/mitmproxy/addons/onboarding.py b/mitmproxy/addons/onboarding.py
index 900acb08..94ca7c49 100644
--- a/mitmproxy/addons/onboarding.py
+++ b/mitmproxy/addons/onboarding.py
@@ -10,7 +10,7 @@ class Onboarding(wsgiapp.WSGIApp):
     name = "onboarding"
 
     def __init__(self):
-        super().__init__(app.Adapter(app.application), None, None)
+        super().__init__(app, None, None)
 
     def load(self, loader):
         loader.add_option(
@@ -32,6 +32,7 @@ class Onboarding(wsgiapp.WSGIApp):
     def configure(self, updated):
         self.host = ctx.options.onboarding_host
         self.port = ctx.options.onboarding_port
+        app.config["CONFDIR"] = ctx.options.confdir
 
     def request(self, f):
         if ctx.options.onboarding:
diff --git a/mitmproxy/addons/onboardingapp/__init__.py b/mitmproxy/addons/onboardingapp/__init__.py
index e69de29b..722fed03 100644
--- a/mitmproxy/addons/onboardingapp/__init__.py
+++ b/mitmproxy/addons/onboardingapp/__init__.py
@@ -0,0 +1,37 @@
+import os
+
+from flask import Flask, render_template
+
+from mitmproxy.options import CONF_BASENAME, CONF_DIR
+
+app = Flask(__name__)
+# will be overridden in the addon, setting this here so that the Flask app can be run standalone.
+app.config["CONFDIR"] = CONF_DIR
+
+
+@app.route('/')
+def index():
+    return render_template("index.html")
+
+
+@app.route('/cert/pem')
+def pem():
+    return read_cert("pem", "application/x-x509-ca-cert")
+
+
+@app.route('/cert/p12')
+def p12():
+    return read_cert("p12", "application/x-pkcs12")
+
+
+def read_cert(ext, content_type):
+    filename = CONF_BASENAME + f"-ca-cert.{ext}"
+    p = os.path.join(app.config["CONFDIR"], filename)
+    p = os.path.expanduser(p)
+    with open(p, "rb") as f:
+        cert = f.read()
+
+    return cert, {
+        "Content-Type": content_type,
+        "Content-Disposition": f"inline; filename={filename}",
+    }
diff --git a/mitmproxy/addons/onboardingapp/app.py b/mitmproxy/addons/onboardingapp/app.py
deleted file mode 100644
index ab136778..00000000
--- a/mitmproxy/addons/onboardingapp/app.py
+++ /dev/null
@@ -1,118 +0,0 @@
-import os
-
-import tornado.template
-import tornado.web
-import tornado.wsgi
-
-from mitmproxy.utils import data
-from mitmproxy.proxy import config
-
-loader = tornado.template.Loader(data.pkg_data.path("addons/onboardingapp/templates"))
-
-
-class Adapter(tornado.wsgi.WSGIAdapter):
-    # Tornado doesn't make the WSGI environment available to pages, so this
-    # hideous monkey patch is the easiest way to get to the mitmproxy.master
-    # variable.
-
-    def __init__(self, application):
-        self._application = application
-
-    def application(self, request):
-        request.master = self.environ["mitmproxy.master"]
-        return self._application(request)
-
-    def __call__(self, environ, start_response):
-        self.environ = environ
-        return tornado.wsgi.WSGIAdapter.__call__(
-            self,
-            environ,
-            start_response
-        )
-
-
-class Index(tornado.web.RequestHandler):
-
-    def get(self):
-        t = loader.load("index.html")
-        self.write(t.generate())
-
-
-class PEM(tornado.web.RequestHandler):
-
-    @property
-    def filename(self):
-        return config.CONF_BASENAME + "-ca-cert.pem"
-
-    def head(self):
-        p = os.path.join(self.request.master.options.confdir, self.filename)
-        p = os.path.expanduser(p)
-        content_length = os.path.getsize(p)
-
-        self.set_header("Content-Type", "application/x-x509-ca-cert")
-        self.set_header(
-            "Content-Disposition",
-            "inline; filename={}".format(
-                self.filename))
-        self.set_header("Content-Length", content_length)
-
-    def get(self):
-        p = os.path.join(self.request.master.options.confdir, self.filename)
-        p = os.path.expanduser(p)
-        self.set_header("Content-Type", "application/x-x509-ca-cert")
-        self.set_header(
-            "Content-Disposition",
-            "inline; filename={}".format(
-                self.filename))
-
-        with open(p, "rb") as f:
-            self.write(f.read())
-
-
-class P12(tornado.web.RequestHandler):
-
-    @property
-    def filename(self):
-        return config.CONF_BASENAME + "-ca-cert.p12"
-
-    def head(self):
-        p = os.path.join(self.request.master.options.confdir, self.filename)
-        p = os.path.expanduser(p)
-        content_length = os.path.getsize(p)
-
-        self.set_header("Content-Type", "application/x-pkcs12")
-        self.set_header(
-            "Content-Disposition",
-            "inline; filename={}".format(
-                self.filename))
-
-        self.set_header("Content-Length", content_length)
-
-    def get(self):
-        p = os.path.join(self.request.master.options.confdir, self.filename)
-        p = os.path.expanduser(p)
-        self.set_header("Content-Type", "application/x-pkcs12")
-        self.set_header(
-            "Content-Disposition",
-            "inline; filename={}".format(
-                self.filename))
-
-        with open(p, "rb") as f:
-            self.write(f.read())
-
-
-application = tornado.web.Application(
-    [
-        (r"/", Index),
-        (r"/cert/pem", PEM),
-        (r"/cert/p12", P12),
-        (
-            r"/static/(.*)",
-            tornado.web.StaticFileHandler,
-            {
-                "path": data.pkg_data.path("addons/onboardingapp/static")
-            }
-        ),
-    ],
-    # debug=True
-)
diff --git a/mitmproxy/addons/onboardingapp/static/mitmproxy.css b/mitmproxy/addons/onboardingapp/static/mitmproxy.css
index 969bd62b..e654d56b 100644
--- a/mitmproxy/addons/onboardingapp/static/mitmproxy.css
+++ b/mitmproxy/addons/onboardingapp/static/mitmproxy.css
@@ -15,7 +15,7 @@
     height: 300px;
 }
 
-.bigtitle>div {
+.bigtitle > div {
     display: table-cell;
     vertical-align: middle;
 }
@@ -31,7 +31,7 @@ section {
 
 .innerlink {
     text-decoration: none;
-    border-bottom:1px dotted;
+    border-bottom: 1px dotted;
     margin-bottom: 15px;
 }
 
diff --git a/mitmproxy/addons/onboardingapp/templates/frame.html b/mitmproxy/addons/onboardingapp/templates/frame.html
index f00e1a66..13003f3c 100644
--- a/mitmproxy/addons/onboardingapp/templates/frame.html
+++ b/mitmproxy/addons/onboardingapp/templates/frame.html
@@ -3,7 +3,7 @@
 <div class="row">
     <div class="span12">
         {% block body %}  
-        {% end %}  
+        {% endblock %}
     </div>
 </div>
-{% end %}  
+{% endblock %}
diff --git a/mitmproxy/addons/onboardingapp/templates/index.html b/mitmproxy/addons/onboardingapp/templates/index.html
index 38aa27ed..aa471668 100644
--- a/mitmproxy/addons/onboardingapp/templates/index.html
+++ b/mitmproxy/addons/onboardingapp/templates/index.html
@@ -135,19 +135,19 @@ function changeTo(device) {
 <h2 class="text-center"> Click to install your mitmproxy certificate </h2>
 <div id="certbank" class="row">
     <div class="col-md-3">
-        <a onclick="changeTo('apple')" href="/cert/pem"><i class="fa fa-apple fa-5x"></i></a>
+        <a target="_blank" onclick="changeTo('apple')" href="/cert/pem"><i class="fa fa-apple fa-5x"></i></a>
         <p>Apple</p>
     </div>
     <div class="col-md-3">
-        <a onclick="changeTo('windows')" href="/cert/p12"><i class="fa fa-windows fa-5x"></i></a>
+        <a target="_blank" onclick="changeTo('windows')" href="/cert/p12"><i class="fa fa-windows fa-5x"></i></a>
         <p>Windows</p>
     </div>
     <div class="col-md-3">
-        <a onclick="changeTo('android')" href="/cert/pem"><i class="fa fa-android fa-5x"></i></a>
+        <a target="_blank" onclick="changeTo('android')" href="/cert/pem"><i class="fa fa-android fa-5x"></i></a>
         <p>Android</p>
     </div>
     <div class="col-md-3">
-        <a onclick="changeTo('asterisk')" href="/cert/pem"><i class="fa fa-asterisk fa-5x"></i></a>
+        <a target="_blank" onclick="changeTo('asterisk')" href="/cert/pem"><i class="fa fa-asterisk fa-5x"></i></a>
         <p>Other</p>
     </div>
 </div>
@@ -167,4 +167,4 @@ function changeTo(device) {
     between mitmproxy installations.
 </div>
 
-{% end %}
+{% endblock %}
diff --git a/mitmproxy/addons/onboardingapp/templates/layout.html b/mitmproxy/addons/onboardingapp/templates/layout.html
index f6e1b286..cea8373b 100644
--- a/mitmproxy/addons/onboardingapp/templates/layout.html
+++ b/mitmproxy/addons/onboardingapp/templates/layout.html
@@ -28,7 +28,7 @@
 
 <div class="container">
     {% block content %}
-    {% end %}
+    {% endblock %}
 </div>
 
   </body>
diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py
index a39ce5ce..3b2568c9 100644
--- a/mitmproxy/addons/script.py
+++ b/mitmproxy/addons/script.py
@@ -16,7 +16,7 @@ from mitmproxy import ctx
 import mitmproxy.types as mtypes
 
 
-def load_script(path: str) -> types.ModuleType:
+def load_script(path: str) -> typing.Optional[types.ModuleType]:
     fullname = "__mitmproxy_script__.{}".format(
         os.path.splitext(os.path.basename(path))[0]
     )
diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py
index e3192a4c..18bc3545 100644
--- a/mitmproxy/addons/serverplayback.py
+++ b/mitmproxy/addons/serverplayback.py
@@ -68,6 +68,13 @@ class ServerPlayback:
             to replay.
             """
         )
+        loader.add_option(
+            "server_replay_ignore_port", bool, False,
+            """
+            Ignore request's destination port while searching for a saved flow
+            to replay.
+            """
+        )
 
     @command.command("replay.server")
     def load_flows(self, flows: typing.Sequence[flow.Flow]) -> None:
@@ -110,7 +117,7 @@ class ServerPlayback:
         _, _, path, _, query, _ = urllib.parse.urlparse(r.url)
         queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
 
-        key: typing.List[typing.Any] = [str(r.port), str(r.scheme), str(r.method), str(path)]
+        key: typing.List[typing.Any] = [str(r.scheme), str(r.method), str(path)]
         if not ctx.options.server_replay_ignore_content:
             if ctx.options.server_replay_ignore_payload_params and r.multipart_form:
                 key.extend(
@@ -129,6 +136,8 @@ class ServerPlayback:
 
         if not ctx.options.server_replay_ignore_host:
             key.append(r.pretty_host)
+        if not ctx.options.server_replay_ignore_port:
+            key.append(r.port)
 
         filtered = []
         ignore_params = ctx.options.server_replay_ignore_params or []
diff --git a/mitmproxy/addons/session.py b/mitmproxy/addons/session.py
index f9073c3e..6636b500 100644
--- a/mitmproxy/addons/session.py
+++ b/mitmproxy/addons/session.py
@@ -215,8 +215,8 @@ class Session:
     def __init__(self):
         self.db_store: SessionDB = None
         self._hot_store: collections.OrderedDict = collections.OrderedDict()
-        self._order_store: typing.Dict[str, typing.Dict[str, typing.Union[int, float, str]]] = {}
-        self._view: typing.List[typing.Tuple[typing.Union[int, float, str], str]] = []
+        self._order_store: typing.Dict[str, typing.Dict[str, typing.Union[int, float, str, None]]] = {}
+        self._view: typing.List[typing.Tuple[typing.Union[int, float, str, None], str]] = []
         self.order: str = orders[0]
         self.filter = matchall
         self._flush_period: float = self._FP_DEFAULT
diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py
index fd530aaa..1651c1f6 100644
--- a/mitmproxy/addons/stickycookie.py
+++ b/mitmproxy/addons/stickycookie.py
@@ -53,6 +53,7 @@ class StickyCookie:
                 self.flt = None
 
     def response(self, flow: http.HTTPFlow):
+        assert flow.response
         if self.flt:
             for name, (value, attrs) in flow.response.cookies.items(multi=True):
                 # FIXME: We now know that Cookie.py screws up some cookies with
diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py
index 8d27840f..da9d19f9 100644
--- a/mitmproxy/addons/view.py
+++ b/mitmproxy/addons/view.py
@@ -238,18 +238,24 @@ class View(collections.abc.Sequence):
         """
             Set focus to the next flow.
         """
-        idx = self.focus.index + 1
-        if self.inbounds(idx):
-            self.focus.flow = self[idx]
+        if self.focus.index is not None:
+            idx = self.focus.index + 1
+            if self.inbounds(idx):
+                self.focus.flow = self[idx]
+        else:
+            pass
 
     @command.command("view.focus.prev")
     def focus_prev(self) -> None:
         """
             Set focus to the previous flow.
         """
-        idx = self.focus.index - 1
-        if self.inbounds(idx):
-            self.focus.flow = self[idx]
+        if self.focus.index is not None:
+            idx = self.focus.index - 1
+            if self.inbounds(idx):
+                self.focus.flow = self[idx]
+        else:
+            pass
 
     # Order
     @command.command("view.order.options")
@@ -584,7 +590,7 @@ class Focus:
     """
     def __init__(self, v: View) -> None:
         self.view = v
-        self._flow: mitmproxy.flow.Flow = None
+        self._flow: typing.Optional[mitmproxy.flow.Flow] = None
         self.sig_change = blinker.Signal()
         if len(self.view):
             self.flow = self.view[0]