Merge branch 'master' into master

67 files changed, 973 insertions, 457 deletions

diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py
index 4214d6ea..8a565a73 100644
--- a/mitmproxy/addonmanager.py
+++ b/mitmproxy/addonmanager.py
@@ -184,7 +184,7 @@ class AddonManager:
                 raise exceptions.AddonManagerError("No such addon: %s" % n)
             self.chain = [i for i in self.chain if i is not a]
             del self.lookup[_get_name(a)]
-        self.invoke_addon(a, "done")
+        self.invoke_addon(addon, "done")
 
     def __len__(self):
         return len(self.chain)
diff --git a/mitmproxy/addons/block.py b/mitmproxy/addons/block.py
index 91f9f709..4ccde0e1 100644
--- a/mitmproxy/addons/block.py
+++ b/mitmproxy/addons/block.py
@@ -36,4 +36,4 @@ class Block:
             layer.reply.kill()
         if ctx.options.block_global and address.is_global:
             ctx.log.warn("Client connection from %s killed by block_global" % astr)
-            layer.reply.kill()
\ No newline at end of file
+            layer.reply.kill()
diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py
index c56c0e74..7bdaeb33 100644
--- a/mitmproxy/addons/clientplayback.py
+++ b/mitmproxy/addons/clientplayback.py
@@ -203,8 +203,9 @@ class ClientPlayback:
             # https://github.com/mitmproxy/mitmproxy/issues/2197
             if hf.request.http_version == "HTTP/2.0":
                 hf.request.http_version = "HTTP/1.1"
-                host = hf.request.headers.pop(":authority")
-                hf.request.headers.insert(0, "host", host)
+                host = hf.request.headers.pop(":authority", None)
+                if host is not None:
+                    hf.request.headers.insert(0, "host", host)
             self.q.put(hf)
         ctx.master.addons.trigger("update", lst)
 
diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py
index a908dbb3..5c9bbcd0 100644
--- a/mitmproxy/addons/core.py
+++ b/mitmproxy/addons/core.py
@@ -289,7 +289,7 @@ class Core:
         """
             The possible values for an encoding specification.
         """
-        return ["gzip", "deflate", "br"]
+        return ["gzip", "deflate", "br", "zstd"]
 
     @command.command("options.load")
     def options_load(self, path: mitmproxy.types.Path) -> None:
diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py
index 6bb52e84..9aff2878 100644
--- a/mitmproxy/addons/cut.py
+++ b/mitmproxy/addons/cut.py
@@ -126,20 +126,18 @@ class Cut:
             format is UTF-8 encoded CSV. If there is exactly one row and one
             column, the data is written to file as-is, with raw bytes preserved.
         """
+        v: typing.Union[str, bytes]
         fp = io.StringIO(newline="")
         if len(cuts) == 1 and len(flows) == 1:
             v = extract(cuts[0], flows[0])
-            if isinstance(v, bytes):
-                fp.write(strutils.always_str(v))
-            else:
-                fp.write(v)
+            fp.write(strutils.always_str(v))  # type: ignore
             ctx.log.alert("Clipped single cut.")
         else:
             writer = csv.writer(fp)
             for f in flows:
                 vals = [extract(c, f) for c in cuts]
                 writer.writerow(
-                    [strutils.always_str(v) or "" for v in vals]  # type: ignore
+                    [strutils.always_str(v) for v in vals]
                 )
             ctx.log.alert("Clipped %s cuts as CSV." % len(cuts))
         try:
diff --git a/mitmproxy/addons/eventstore.py b/mitmproxy/addons/eventstore.py
index 50fea7ab..188a3b39 100644
--- a/mitmproxy/addons/eventstore.py
+++ b/mitmproxy/addons/eventstore.py
@@ -14,7 +14,7 @@ class EventStore:
         self.sig_refresh = blinker.Signal()
 
     @property
-    def size(self) -> int:
+    def size(self) -> typing.Optional[int]:
         return self.data.maxlen
 
     def log(self, entry: LogEntry) -> None:
diff --git a/mitmproxy/addons/export.py b/mitmproxy/addons/export.py
index 90e95d3e..2776118a 100644
--- a/mitmproxy/addons/export.py
+++ b/mitmproxy/addons/export.py
@@ -11,17 +11,23 @@ import mitmproxy.types
 import pyperclip
 
 
-def raise_if_missing_request(f: flow.Flow) -> None:
+def cleanup_request(f: flow.Flow):
     if not hasattr(f, "request"):
         raise exceptions.CommandError("Can't export flow with no request.")
+    request = f.request.copy()  # type: ignore
+    request.decode(strict=False)
+    # a bit of clean-up
+    if request.method == 'GET' and request.headers.get("content-length", None) == "0":
+        request.headers.pop('content-length')
+    request.headers.pop(':authority', None)
+    return request
 
 
 def curl_command(f: flow.Flow) -> str:
-    raise_if_missing_request(f)
     data = "curl "
-    request = f.request.copy()  # type: ignore
-    request.decode(strict=False)
+    request = cleanup_request(f)
     for k, v in request.headers.items(multi=True):
+        data += "--compressed " if k == 'accept-encoding' else ""
         data += "-H '%s:%s' " % (k, v)
     if request.method != "GET":
         data += "-X %s " % request.method
@@ -35,11 +41,8 @@ def curl_command(f: flow.Flow) -> str:
 
 
 def httpie_command(f: flow.Flow) -> str:
-    raise_if_missing_request(f)
-    request = f.request.copy()  # type: ignore
-    data = "http %s " % request.method
-    request.decode(strict=False)
-    data += "%s" % request.url
+    request = cleanup_request(f)
+    data = "http %s %s" % (request.method, request.url)
     for k, v in request.headers.items(multi=True):
         data += " '%s:%s'" % (k, v)
     if request.content:
@@ -51,8 +54,7 @@ def httpie_command(f: flow.Flow) -> str:
 
 
 def raw(f: flow.Flow) -> bytes:
-    raise_if_missing_request(f)
-    return assemble.assemble_request(f.request)  # type: ignore
+    return assemble.assemble_request(cleanup_request(f))  # type: ignore
 
 
 formats = dict(
diff --git a/mitmproxy/addons/onboarding.py b/mitmproxy/addons/onboarding.py
index 900acb08..94ca7c49 100644
--- a/mitmproxy/addons/onboarding.py
+++ b/mitmproxy/addons/onboarding.py
@@ -10,7 +10,7 @@ class Onboarding(wsgiapp.WSGIApp):
     name = "onboarding"
 
     def __init__(self):
-        super().__init__(app.Adapter(app.application), None, None)
+        super().__init__(app, None, None)
 
     def load(self, loader):
         loader.add_option(
@@ -32,6 +32,7 @@ class Onboarding(wsgiapp.WSGIApp):
     def configure(self, updated):
         self.host = ctx.options.onboarding_host
         self.port = ctx.options.onboarding_port
+        app.config["CONFDIR"] = ctx.options.confdir
 
     def request(self, f):
         if ctx.options.onboarding:
diff --git a/mitmproxy/addons/onboardingapp/__init__.py b/mitmproxy/addons/onboardingapp/__init__.py
index e69de29b..722fed03 100644
--- a/mitmproxy/addons/onboardingapp/__init__.py
+++ b/mitmproxy/addons/onboardingapp/__init__.py
@@ -0,0 +1,37 @@
+import os
+
+from flask import Flask, render_template
+
+from mitmproxy.options import CONF_BASENAME, CONF_DIR
+
+app = Flask(__name__)
+# will be overridden in the addon, setting this here so that the Flask app can be run standalone.
+app.config["CONFDIR"] = CONF_DIR
+
+
+@app.route('/')
+def index():
+    return render_template("index.html")
+
+
+@app.route('/cert/pem')
+def pem():
+    return read_cert("pem", "application/x-x509-ca-cert")
+
+
+@app.route('/cert/p12')
+def p12():
+    return read_cert("p12", "application/x-pkcs12")
+
+
+def read_cert(ext, content_type):
+    filename = CONF_BASENAME + f"-ca-cert.{ext}"
+    p = os.path.join(app.config["CONFDIR"], filename)
+    p = os.path.expanduser(p)
+    with open(p, "rb") as f:
+        cert = f.read()
+
+    return cert, {
+        "Content-Type": content_type,
+        "Content-Disposition": f"inline; filename={filename}",
+    }
diff --git a/mitmproxy/addons/onboardingapp/app.py b/mitmproxy/addons/onboardingapp/app.py
deleted file mode 100644
index ab136778..00000000
--- a/mitmproxy/addons/onboardingapp/app.py
+++ /dev/null
@@ -1,118 +0,0 @@
-import os
-
-import tornado.template
-import tornado.web
-import tornado.wsgi
-
-from mitmproxy.utils import data
-from mitmproxy.proxy import config
-
-loader = tornado.template.Loader(data.pkg_data.path("addons/onboardingapp/templates"))
-
-
-class Adapter(tornado.wsgi.WSGIAdapter):
-    # Tornado doesn't make the WSGI environment available to pages, so this
-    # hideous monkey patch is the easiest way to get to the mitmproxy.master
-    # variable.
-
-    def __init__(self, application):
-        self._application = application
-
-    def application(self, request):
-        request.master = self.environ["mitmproxy.master"]
-        return self._application(request)
-
-    def __call__(self, environ, start_response):
-        self.environ = environ
-        return tornado.wsgi.WSGIAdapter.__call__(
-            self,
-            environ,
-            start_response
-        )
-
-
-class Index(tornado.web.RequestHandler):
-
-    def get(self):
-        t = loader.load("index.html")
-        self.write(t.generate())
-
-
-class PEM(tornado.web.RequestHandler):
-
-    @property
-    def filename(self):
-        return config.CONF_BASENAME + "-ca-cert.pem"
-
-    def head(self):
-        p = os.path.join(self.request.master.options.confdir, self.filename)
-        p = os.path.expanduser(p)
-        content_length = os.path.getsize(p)
-
-        self.set_header("Content-Type", "application/x-x509-ca-cert")
-        self.set_header(
-            "Content-Disposition",
-            "inline; filename={}".format(
-                self.filename))
-        self.set_header("Content-Length", content_length)
-
-    def get(self):
-        p = os.path.join(self.request.master.options.confdir, self.filename)
-        p = os.path.expanduser(p)
-        self.set_header("Content-Type", "application/x-x509-ca-cert")
-        self.set_header(
-            "Content-Disposition",
-            "inline; filename={}".format(
-                self.filename))
-
-        with open(p, "rb") as f:
-            self.write(f.read())
-
-
-class P12(tornado.web.RequestHandler):
-
-    @property
-    def filename(self):
-        return config.CONF_BASENAME + "-ca-cert.p12"
-
-    def head(self):
-        p = os.path.join(self.request.master.options.confdir, self.filename)
-        p = os.path.expanduser(p)
-        content_length = os.path.getsize(p)
-
-        self.set_header("Content-Type", "application/x-pkcs12")
-        self.set_header(
-            "Content-Disposition",
-            "inline; filename={}".format(
-                self.filename))
-
-        self.set_header("Content-Length", content_length)
-
-    def get(self):
-        p = os.path.join(self.request.master.options.confdir, self.filename)
-        p = os.path.expanduser(p)
-        self.set_header("Content-Type", "application/x-pkcs12")
-        self.set_header(
-            "Content-Disposition",
-            "inline; filename={}".format(
-                self.filename))
-
-        with open(p, "rb") as f:
-            self.write(f.read())
-
-
-application = tornado.web.Application(
-    [
-        (r"/", Index),
-        (r"/cert/pem", PEM),
-        (r"/cert/p12", P12),
-        (
-            r"/static/(.*)",
-            tornado.web.StaticFileHandler,
-            {
-                "path": data.pkg_data.path("addons/onboardingapp/static")
-            }
-        ),
-    ],
-    # debug=True
-)
diff --git a/mitmproxy/addons/onboardingapp/static/mitmproxy.css b/mitmproxy/addons/onboardingapp/static/mitmproxy.css
index 969bd62b..e654d56b 100644
--- a/mitmproxy/addons/onboardingapp/static/mitmproxy.css
+++ b/mitmproxy/addons/onboardingapp/static/mitmproxy.css
@@ -15,7 +15,7 @@
     height: 300px;
 }
 
-.bigtitle>div {
+.bigtitle > div {
     display: table-cell;
     vertical-align: middle;
 }
@@ -31,7 +31,7 @@ section {
 
 .innerlink {
     text-decoration: none;
-    border-bottom:1px dotted;
+    border-bottom: 1px dotted;
     margin-bottom: 15px;
 }
 
diff --git a/mitmproxy/addons/onboardingapp/templates/frame.html b/mitmproxy/addons/onboardingapp/templates/frame.html
index f00e1a66..13003f3c 100644
--- a/mitmproxy/addons/onboardingapp/templates/frame.html
+++ b/mitmproxy/addons/onboardingapp/templates/frame.html
@@ -3,7 +3,7 @@
 <div class="row">
     <div class="span12">
         {% block body %}  
-        {% end %}  
+        {% endblock %}
     </div>
 </div>
-{% end %}  
+{% endblock %}
diff --git a/mitmproxy/addons/onboardingapp/templates/index.html b/mitmproxy/addons/onboardingapp/templates/index.html
index 38aa27ed..aa471668 100644
--- a/mitmproxy/addons/onboardingapp/templates/index.html
+++ b/mitmproxy/addons/onboardingapp/templates/index.html
@@ -135,19 +135,19 @@ function changeTo(device) {
 <h2 class="text-center"> Click to install your mitmproxy certificate </h2>
 <div id="certbank" class="row">
     <div class="col-md-3">
-        <a onclick="changeTo('apple')" href="/cert/pem"><i class="fa fa-apple fa-5x"></i></a>
+        <a target="_blank" onclick="changeTo('apple')" href="/cert/pem"><i class="fa fa-apple fa-5x"></i></a>
         <p>Apple</p>
     </div>
     <div class="col-md-3">
-        <a onclick="changeTo('windows')" href="/cert/p12"><i class="fa fa-windows fa-5x"></i></a>
+        <a target="_blank" onclick="changeTo('windows')" href="/cert/p12"><i class="fa fa-windows fa-5x"></i></a>
         <p>Windows</p>
     </div>
     <div class="col-md-3">
-        <a onclick="changeTo('android')" href="/cert/pem"><i class="fa fa-android fa-5x"></i></a>
+        <a target="_blank" onclick="changeTo('android')" href="/cert/pem"><i class="fa fa-android fa-5x"></i></a>
         <p>Android</p>
     </div>
     <div class="col-md-3">
-        <a onclick="changeTo('asterisk')" href="/cert/pem"><i class="fa fa-asterisk fa-5x"></i></a>
+        <a target="_blank" onclick="changeTo('asterisk')" href="/cert/pem"><i class="fa fa-asterisk fa-5x"></i></a>
         <p>Other</p>
     </div>
 </div>
@@ -167,4 +167,4 @@ function changeTo(device) {
     between mitmproxy installations.
 </div>
 
-{% end %}
+{% endblock %}
diff --git a/mitmproxy/addons/onboardingapp/templates/layout.html b/mitmproxy/addons/onboardingapp/templates/layout.html
index f6e1b286..cea8373b 100644
--- a/mitmproxy/addons/onboardingapp/templates/layout.html
+++ b/mitmproxy/addons/onboardingapp/templates/layout.html
@@ -28,7 +28,7 @@
 
 <div class="container">
     {% block content %}
-    {% end %}
+    {% endblock %}
 </div>
 
   </body>
diff --git a/mitmproxy/addons/script.py b/mitmproxy/addons/script.py
index a39ce5ce..3b2568c9 100644
--- a/mitmproxy/addons/script.py
+++ b/mitmproxy/addons/script.py
@@ -16,7 +16,7 @@ from mitmproxy import ctx
 import mitmproxy.types as mtypes
 
 
-def load_script(path: str) -> types.ModuleType:
+def load_script(path: str) -> typing.Optional[types.ModuleType]:
     fullname = "__mitmproxy_script__.{}".format(
         os.path.splitext(os.path.basename(path))[0]
     )
diff --git a/mitmproxy/addons/serverplayback.py b/mitmproxy/addons/serverplayback.py
index e3192a4c..18bc3545 100644
--- a/mitmproxy/addons/serverplayback.py
+++ b/mitmproxy/addons/serverplayback.py
@@ -68,6 +68,13 @@ class ServerPlayback:
             to replay.
             """
         )
+        loader.add_option(
+            "server_replay_ignore_port", bool, False,
+            """
+            Ignore request's destination port while searching for a saved flow
+            to replay.
+            """
+        )
 
     @command.command("replay.server")
     def load_flows(self, flows: typing.Sequence[flow.Flow]) -> None:
@@ -110,7 +117,7 @@ class ServerPlayback:
         _, _, path, _, query, _ = urllib.parse.urlparse(r.url)
         queriesArray = urllib.parse.parse_qsl(query, keep_blank_values=True)
 
-        key: typing.List[typing.Any] = [str(r.port), str(r.scheme), str(r.method), str(path)]
+        key: typing.List[typing.Any] = [str(r.scheme), str(r.method), str(path)]
         if not ctx.options.server_replay_ignore_content:
             if ctx.options.server_replay_ignore_payload_params and r.multipart_form:
                 key.extend(
@@ -129,6 +136,8 @@ class ServerPlayback:
 
         if not ctx.options.server_replay_ignore_host:
             key.append(r.pretty_host)
+        if not ctx.options.server_replay_ignore_port:
+            key.append(r.port)
 
         filtered = []
         ignore_params = ctx.options.server_replay_ignore_params or []
diff --git a/mitmproxy/addons/session.py b/mitmproxy/addons/session.py
index f9073c3e..6636b500 100644
--- a/mitmproxy/addons/session.py
+++ b/mitmproxy/addons/session.py
@@ -215,8 +215,8 @@ class Session:
     def __init__(self):
         self.db_store: SessionDB = None
         self._hot_store: collections.OrderedDict = collections.OrderedDict()
-        self._order_store: typing.Dict[str, typing.Dict[str, typing.Union[int, float, str]]] = {}
-        self._view: typing.List[typing.Tuple[typing.Union[int, float, str], str]] = []
+        self._order_store: typing.Dict[str, typing.Dict[str, typing.Union[int, float, str, None]]] = {}
+        self._view: typing.List[typing.Tuple[typing.Union[int, float, str, None], str]] = []
         self.order: str = orders[0]
         self.filter = matchall
         self._flush_period: float = self._FP_DEFAULT
diff --git a/mitmproxy/addons/stickycookie.py b/mitmproxy/addons/stickycookie.py
index fd530aaa..1651c1f6 100644
--- a/mitmproxy/addons/stickycookie.py
+++ b/mitmproxy/addons/stickycookie.py
@@ -53,6 +53,7 @@ class StickyCookie:
                 self.flt = None
 
     def response(self, flow: http.HTTPFlow):
+        assert flow.response
         if self.flt:
             for name, (value, attrs) in flow.response.cookies.items(multi=True):
                 # FIXME: We now know that Cookie.py screws up some cookies with
diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py
index 8d27840f..da9d19f9 100644
--- a/mitmproxy/addons/view.py
+++ b/mitmproxy/addons/view.py
@@ -238,18 +238,24 @@ class View(collections.abc.Sequence):
         """
             Set focus to the next flow.
         """
-        idx = self.focus.index + 1
-        if self.inbounds(idx):
-            self.focus.flow = self[idx]
+        if self.focus.index is not None:
+            idx = self.focus.index + 1
+            if self.inbounds(idx):
+                self.focus.flow = self[idx]
+        else:
+            pass
 
     @command.command("view.focus.prev")
     def focus_prev(self) -> None:
         """
             Set focus to the previous flow.
         """
-        idx = self.focus.index - 1
-        if self.inbounds(idx):
-            self.focus.flow = self[idx]
+        if self.focus.index is not None:
+            idx = self.focus.index - 1
+            if self.inbounds(idx):
+                self.focus.flow = self[idx]
+        else:
+            pass
 
     # Order
     @command.command("view.order.options")
@@ -584,7 +590,7 @@ class Focus:
     """
     def __init__(self, v: View) -> None:
         self.view = v
-        self._flow: mitmproxy.flow.Flow = None
+        self._flow: typing.Optional[mitmproxy.flow.Flow] = None
         self.sig_change = blinker.Signal()
         if len(self.view):
             self.flow = self.view[0]
diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py
index 6f5f8c09..d574c027 100644
--- a/mitmproxy/certs.py
+++ b/mitmproxy/certs.py
@@ -36,9 +36,9 @@ rD693XKIHUCWOjMh1if6omGXKHH40QuME2gNa50+YPn1iYDl88uDbbMCAQI=
 """
 
 
-def create_ca(organization, cn, exp):
+def create_ca(organization, cn, exp, key_size):
     key = OpenSSL.crypto.PKey()
-    key.generate_key(OpenSSL.crypto.TYPE_RSA, 2048)
+    key.generate_key(OpenSSL.crypto.TYPE_RSA, key_size)
     cert = OpenSSL.crypto.X509()
     cert.set_serial_number(int(time.time() * 10000))
     cert.set_version(2)
@@ -115,6 +115,13 @@ def dummy_cert(privkey, cacert, commonname, sans, organization):
         cert.set_version(2)
         cert.add_extensions(
             [OpenSSL.crypto.X509Extension(b"subjectAltName", False, ss)])
+    cert.add_extensions([
+        OpenSSL.crypto.X509Extension(
+            b"extendedKeyUsage",
+            False,
+            b"serverAuth,clientAuth"
+        )
+    ])
     cert.set_pubkey(cacert.get_pubkey())
     cert.sign(privkey, "sha256")
     return Cert(cert)
@@ -182,10 +189,10 @@ class CertStore:
             return dh
 
     @classmethod
-    def from_store(cls, path, basename):
+    def from_store(cls, path, basename, key_size):
         ca_path = os.path.join(path, basename + "-ca.pem")
         if not os.path.exists(ca_path):
-            key, ca = cls.create_store(path, basename)
+            key, ca = cls.create_store(path, basename, key_size)
         else:
             with open(ca_path, "rb") as f:
                 raw = f.read()
@@ -215,14 +222,14 @@ class CertStore:
             os.umask(original_umask)
 
     @staticmethod
-    def create_store(path, basename, organization=None, cn=None, expiry=DEFAULT_EXP):
+    def create_store(path, basename, key_size, organization=None, cn=None, expiry=DEFAULT_EXP):
         if not os.path.exists(path):
             os.makedirs(path)
 
         organization = organization or basename
         cn = cn or basename
 
-        key, ca = create_ca(organization=organization, cn=cn, exp=expiry)
+        key, ca = create_ca(organization=organization, cn=cn, exp=expiry, key_size=key_size)
         # Dump the CA plus private key
         with CertStore.umask_secret(), open(os.path.join(path, basename + "-ca.pem"), "wb") as f:
             f.write(
@@ -308,7 +315,12 @@ class CertStore:
             ret.append(b"*." + b".".join(parts[i:]))
         return ret
 
-    def get_cert(self, commonname: typing.Optional[bytes], sans: typing.List[bytes], organization: typing.Optional[bytes] = None):
+    def get_cert(
+            self,
+            commonname: typing.Optional[bytes],
+            sans: typing.List[bytes],
+            organization: typing.Optional[bytes] = None
+    ) -> typing.Tuple["Cert", OpenSSL.SSL.PKey, str]:
         """
             Returns an (cert, privkey, cert_chain) tuple.
 
diff --git a/mitmproxy/command.py b/mitmproxy/command.py
index 27f0921d..0998601c 100644
--- a/mitmproxy/command.py
+++ b/mitmproxy/command.py
@@ -44,6 +44,8 @@ def typename(t: type) -> str:
 
 
 class Command:
+    returntype: typing.Optional[typing.Type]
+
     def __init__(self, manager, path, func) -> None:
         self.path = path
         self.manager = manager
@@ -177,7 +179,7 @@ class CommandManager(mitmproxy.types._CommandBase):
 
         parse: typing.List[ParseResult] = []
         params: typing.List[type] = []
-        typ: typing.Type = None
+        typ: typing.Type
         for i in range(len(parts)):
             if i == 0:
                 typ = mitmproxy.types.Cmd
diff --git a/mitmproxy/contentviews/__init__.py b/mitmproxy/contentviews/__init__.py
index 01c6d221..1e71d942 100644
--- a/mitmproxy/contentviews/__init__.py
+++ b/mitmproxy/contentviews/__init__.py
@@ -135,7 +135,9 @@ def get_content_view(viewmode: View, data: bytes, **metadata):
     # Third-party viewers can fail in unexpected ways...
     except Exception:
         desc = "Couldn't parse: falling back to Raw"
-        _, content = get("Raw")(data, **metadata)
+        raw = get("Raw")
+        assert raw
+        content = raw(data, **metadata)[1]
         error = "{} Content viewer failed: \n{}".format(
             getattr(viewmode, "name"),
             traceback.format_exc()
diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py
index 6072dfb7..81f2e487 100644
--- a/mitmproxy/contentviews/base.py
+++ b/mitmproxy/contentviews/base.py
@@ -9,8 +9,8 @@ TViewResult = typing.Tuple[str, typing.Iterator[TViewLine]]
 
 
 class View:
-    name: str = None
-    content_types: typing.List[str] = []
+    name: typing.ClassVar[str]
+    content_types: typing.ClassVar[typing.List[str]] = []
 
     def __call__(self, data: bytes, **metadata) -> TViewResult:
         """
@@ -37,7 +37,7 @@ class View:
 
 def format_pairs(
         items: typing.Iterable[typing.Tuple[TTextType, TTextType]]
-)-> typing.Iterator[TViewLine]:
+) -> typing.Iterator[TViewLine]:
 
     """
     Helper function that accepts a list of (k,v) pairs into a list of
diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py
index fcc50cb5..d5bb404f 100644
--- a/mitmproxy/contentviews/image/image_parser.py
+++ b/mitmproxy/contentviews/image/image_parser.py
@@ -54,7 +54,7 @@ def parse_gif(data: bytes) -> Metadata:
         entries = block.body.body.entries
         for entry in entries:
             comment = entry.bytes
-            if comment is not b'':
+            if comment != b'':
                 parts.append(('comment', str(comment)))
     return parts
 
diff --git a/mitmproxy/contentviews/xml_html.py b/mitmproxy/contentviews/xml_html.py
index 00a62a15..f2fa47cb 100644
--- a/mitmproxy/contentviews/xml_html.py
+++ b/mitmproxy/contentviews/xml_html.py
@@ -1,7 +1,7 @@
 import io
 import re
 import textwrap
-from typing import Iterable
+from typing import Iterable, Optional
 
 from mitmproxy.contentviews import base
 from mitmproxy.utils import sliding_window
@@ -124,14 +124,14 @@ def indent_text(data: str, prefix: str) -> str:
     return textwrap.indent(dedented, prefix[:32])
 
 
-def is_inline_text(a: Token, b: Token, c: Token) -> bool:
+def is_inline_text(a: Optional[Token], b: Optional[Token], c: Optional[Token]) -> bool:
     if isinstance(a, Tag) and isinstance(b, Text) and isinstance(c, Tag):
         if a.is_opening and "\n" not in b.data and c.is_closing and a.tag == c.tag:
             return True
     return False
 
 
-def is_inline(prev2: Token, prev1: Token, t: Token, next1: Token, next2: Token) -> bool:
+def is_inline(prev2: Optional[Token], prev1: Optional[Token], t: Optional[Token], next1: Optional[Token], next2: Optional[Token]) -> bool:
     if isinstance(t, Text):
         return is_inline_text(prev1, t, next1)
     elif isinstance(t, Tag):
diff --git a/mitmproxy/ctx.py b/mitmproxy/ctx.py
index 5df6f9c1..2ce9c7c2 100644
--- a/mitmproxy/ctx.py
+++ b/mitmproxy/ctx.py
@@ -1,7 +1,7 @@
-import mitmproxy.master  # noqa
-import mitmproxy.log  # noqa
-import mitmproxy.options  # noqa
+import mitmproxy.log
+import mitmproxy.master
+import mitmproxy.options
 
-master = None  # type: mitmproxy.master.Master
-log: mitmproxy.log.Log = None
-options: mitmproxy.options.Options = None
+log: "mitmproxy.log.Log"
+master: "mitmproxy.master.Master"
+options: "mitmproxy.options.Options"
diff --git a/mitmproxy/flowfilter.py b/mitmproxy/flowfilter.py
index 7f8df96f..b222d2a8 100644
--- a/mitmproxy/flowfilter.py
+++ b/mitmproxy/flowfilter.py
@@ -32,19 +32,17 @@
         rex         Equivalent to ~u rex
 """
 
+import functools
 import re
 import sys
-import functools
+from typing import Callable, ClassVar, Optional, Sequence, Type
+
+import pyparsing as pp
 
+from mitmproxy import flow
 from mitmproxy import http
-from mitmproxy import websocket
 from mitmproxy import tcp
-from mitmproxy import flow
-
-from mitmproxy.utils import strutils
-
-import pyparsing as pp
-from typing import Callable, Sequence, Type  # noqa
+from mitmproxy import websocket
 
 
 def only(*types):
@@ -54,7 +52,9 @@ def only(*types):
             if isinstance(flow, types):
                 return fn(self, flow)
             return False
+
         return filter_types
+
     return decorator
 
 
@@ -69,8 +69,8 @@ class _Token:
 
 
 class _Action(_Token):
-    code: str = None
-    help: str = None
+    code: ClassVar[str]
+    help: ClassVar[str]
 
     @classmethod
     def make(klass, s, loc, toks):
@@ -146,10 +146,10 @@ class _Rex(_Action):
     def __init__(self, expr):
         self.expr = expr
         if self.is_binary:
-            expr = strutils.escaped_str_to_bytes(expr)
+            expr = expr.encode()
         try:
             self.re = re.compile(expr, self.flags)
-        except:
+        except Exception:
             raise ValueError("Cannot compile expression.")
 
 
@@ -336,6 +336,7 @@ class FUrl(_Rex):
     code = "u"
     help = "URL"
     is_binary = False
+
     # FUrl is special, because it can be "naked".
 
     @classmethod
@@ -469,45 +470,51 @@ def _make():
     # Order is important - multi-char expressions need to come before narrow
     # ones.
     parts = []
-    for klass in filter_unary:
-        f = pp.Literal("~%s" % klass.code) + pp.WordEnd()
-        f.setParseAction(klass.make)
+    for cls in filter_unary:
+        f = pp.Literal(f"~{cls.code}") + pp.WordEnd()
+        f.setParseAction(cls.make)
         parts.append(f)
 
-    simplerex = "".join(c for c in pp.printables if c not in "()~'\"")
-    rex = pp.Word(simplerex) |\
-        pp.QuotedString("\"", escChar='\\') |\
-        pp.QuotedString("'", escChar='\\')
-    for klass in filter_rex:
-        f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + rex.copy()
-        f.setParseAction(klass.make)
+    # This is a bit of a hack to simulate Word(pyparsing_unicode.printables),
+    # which has a horrible performance with len(pyparsing.pyparsing_unicode.printables) == 1114060
+    unicode_words = pp.CharsNotIn("()~'\"" + pp.ParserElement.DEFAULT_WHITE_CHARS)
+    unicode_words.skipWhitespace = True
+    regex = (
+            unicode_words
+            | pp.QuotedString('"', escChar='\\')
+            | pp.QuotedString("'", escChar='\\')
+    )
+    for cls in filter_rex:
+        f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + regex.copy()
+        f.setParseAction(cls.make)
         parts.append(f)
 
-    for klass in filter_int:
-        f = pp.Literal("~%s" % klass.code) + pp.WordEnd() + pp.Word(pp.nums)
-        f.setParseAction(klass.make)
+    for cls in filter_int:
+        f = pp.Literal(f"~{cls.code}") + pp.WordEnd() + pp.Word(pp.nums)
+        f.setParseAction(cls.make)
         parts.append(f)
 
     # A naked rex is a URL rex:
-    f = rex.copy()
+    f = regex.copy()
     f.setParseAction(FUrl.make)
     parts.append(f)
 
     atom = pp.MatchFirst(parts)
-    expr = pp.operatorPrecedence(atom,
-                                 [(pp.Literal("!").suppress(),
-                                   1,
-                                   pp.opAssoc.RIGHT,
-                                   lambda x: FNot(*x)),
-                                     (pp.Literal("&").suppress(),
-                                      2,
-                                      pp.opAssoc.LEFT,
-                                      lambda x: FAnd(*x)),
-                                     (pp.Literal("|").suppress(),
-                                      2,
-                                      pp.opAssoc.LEFT,
-                                      lambda x: FOr(*x)),
-                                  ])
+    expr = pp.infixNotation(
+        atom,
+        [(pp.Literal("!").suppress(),
+          1,
+          pp.opAssoc.RIGHT,
+          lambda x: FNot(*x)),
+         (pp.Literal("&").suppress(),
+          2,
+          pp.opAssoc.LEFT,
+          lambda x: FAnd(*x)),
+         (pp.Literal("|").suppress(),
+          2,
+          pp.opAssoc.LEFT,
+          lambda x: FOr(*x)),
+         ])
     expr = pp.OneOrMore(expr)
     return expr.setParseAction(lambda x: FAnd(x) if len(x) != 1 else x)
 
@@ -516,7 +523,7 @@ bnf = _make()
 TFilter = Callable[[flow.Flow], bool]
 
 
-def parse(s: str) -> TFilter:
+def parse(s: str) -> Optional[TFilter]:
     try:
         flt = bnf.parseString(s, parseAll=True)[0]
         flt.pattern = s
@@ -547,15 +554,15 @@ def match(flt, flow):
 help = []
 for a in filter_unary:
     help.append(
-        ("~%s" % a.code, a.help)
+        (f"~{a.code}", a.help)
     )
 for b in filter_rex:
     help.append(
-        ("~%s regex" % b.code, b.help)
+        (f"~{b.code} regex", b.help)
     )
 for c in filter_int:
     help.append(
-        ("~%s int" % c.code, c.help)
+        (f"~{c.code} int", c.help)
     )
 help.sort()
 help.extend(
diff --git a/mitmproxy/http.py b/mitmproxy/http.py
index 3c16b807..6b527e75 100644
--- a/mitmproxy/http.py
+++ b/mitmproxy/http.py
@@ -1,15 +1,13 @@
 import html
 from typing import Optional
 
+from mitmproxy import connections
 from mitmproxy import flow
-
-from mitmproxy.net import http
 from mitmproxy import version
-from mitmproxy import connections  # noqa
+from mitmproxy.net import http
 
 
 class HTTPRequest(http.Request):
-
     """
     A mitmproxy HTTP request.
     """
@@ -85,10 +83,10 @@ class HTTPRequest(http.Request):
 
 
 class HTTPResponse(http.Response):
-
     """
     A mitmproxy HTTP response.
     """
+
     # This is a very thin wrapper on top of :py:class:`mitmproxy.net.http.Response` and
     # may be removed in the future.
 
@@ -136,34 +134,28 @@ class HTTPResponse(http.Response):
 
 
 class HTTPFlow(flow.Flow):
-
     """
     An HTTPFlow is a collection of objects representing a single HTTP
     transaction.
     """
+    request: HTTPRequest
+    response: Optional[HTTPResponse] = None
+    error: Optional[flow.Error] = None
+    """
+    Note that it's possible for a Flow to have both a response and an error
+    object. This might happen, for instance, when a response was received
+    from the server, but there was an error sending it back to the client.
+    """
+    server_conn: connections.ServerConnection
+    client_conn: connections.ClientConnection
+    intercepted: bool = False
+    """ Is this flow currently being intercepted? """
+    mode: str
+    """ What mode was the proxy layer in when receiving this request? """
 
     def __init__(self, client_conn, server_conn, live=None, mode="regular"):
         super().__init__("http", client_conn, server_conn, live)
-
-        self.request: HTTPRequest = None
-        """ :py:class:`HTTPRequest` object """
-        self.response: HTTPResponse = None
-        """ :py:class:`HTTPResponse` object """
-        self.error: flow.Error = None
-        """ :py:class:`Error` object
-
-        Note that it's possible for a Flow to have both a response and an error
-        object. This might happen, for instance, when a response was received
-        from the server, but there was an error sending it back to the client.
-        """
-        self.server_conn: connections.ServerConnection = server_conn
-        """ :py:class:`ServerConnection` object """
-        self.client_conn: connections.ClientConnection = client_conn
-        """:py:class:`ClientConnection` object """
-        self.intercepted: bool = False
-        """ Is this flow currently being intercepted? """
         self.mode = mode
-        """ What mode was the proxy layer in when receiving this request? """
 
     _stateobject_attributes = flow.Flow._stateobject_attributes.copy()
     # mypy doesn't support update with kwargs
@@ -205,8 +197,8 @@ class HTTPFlow(flow.Flow):
 
 def make_error_response(
         status_code: int,
-        message: str="",
-        headers: Optional[http.Headers]=None,
+        message: str = "",
+        headers: Optional[http.Headers] = None,
 ) -> HTTPResponse:
     reason = http.status_codes.RESPONSES.get(status_code, "Unknown")
     body = """
diff --git a/mitmproxy/io/tnetstring.py b/mitmproxy/io/tnetstring.py
index aa1f5670..de84279b 100644
--- a/mitmproxy/io/tnetstring.py
+++ b/mitmproxy/io/tnetstring.py
@@ -192,22 +192,22 @@ def parse(data_type: int, data: bytes) -> TSerializable:
         try:
             return int(data)
         except ValueError:
-            raise ValueError("not a tnetstring: invalid integer literal: {}".format(data))
+            raise ValueError(f"not a tnetstring: invalid integer literal: {data!r}")
     if data_type == ord(b'^'):
         try:
             return float(data)
         except ValueError:
-            raise ValueError("not a tnetstring: invalid float literal: {}".format(data))
+            raise ValueError(f"not a tnetstring: invalid float literal: {data!r}")
     if data_type == ord(b'!'):
         if data == b'true':
             return True
         elif data == b'false':
             return False
         else:
-            raise ValueError("not a tnetstring: invalid boolean literal: {}".format(data))
+            raise ValueError(f"not a tnetstring: invalid boolean literal: {data!r}")
     if data_type == ord(b'~'):
         if data:
-            raise ValueError("not a tnetstring: invalid null literal")
+            raise ValueError(f"not a tnetstring: invalid null literal: {data!r}")
         return None
     if data_type == ord(b']'):
         l = []
@@ -236,7 +236,7 @@ def pop(data: bytes) -> typing.Tuple[TSerializable, bytes]:
         blength, data = data.split(b':', 1)
         length = int(blength)
     except ValueError:
-        raise ValueError("not a tnetstring: missing or invalid length prefix: {}".format(data))
+        raise ValueError(f"not a tnetstring: missing or invalid length prefix: {data!r}")
     try:
         data, data_type, remain = data[:length], data[length], data[length + 1:]
     except IndexError:
diff --git a/mitmproxy/net/http/cookies.py b/mitmproxy/net/http/cookies.py
index 1472ab55..2745701f 100644
--- a/mitmproxy/net/http/cookies.py
+++ b/mitmproxy/net/http/cookies.py
@@ -304,7 +304,7 @@ def refresh_set_cookie_header(c: str, delta: int) -> str:
             e = email.utils.parsedate_tz(attrs["expires"])
             if e:
                 f = email.utils.mktime_tz(e) + delta
-                attrs.set_all("expires", [email.utils.formatdate(f)])
+                attrs.set_all("expires", [email.utils.formatdate(f, usegmt=True)])
             else:
                 # This can happen when the expires tag is invalid.
                 # reddit.com sends a an expires tag like this: "Thu, 31 Dec
diff --git a/mitmproxy/net/http/encoding.py b/mitmproxy/net/http/encoding.py
index 8cb96e5c..16d399ca 100644
--- a/mitmproxy/net/http/encoding.py
+++ b/mitmproxy/net/http/encoding.py
@@ -9,6 +9,7 @@ from io import BytesIO
 import gzip
 import zlib
 import brotli
+import zstandard as zstd
 
 from typing import Union, Optional, AnyStr  # noqa
 
@@ -52,7 +53,7 @@ def decode(
             decoded = custom_decode[encoding](encoded)
         except KeyError:
             decoded = codecs.decode(encoded, encoding, errors)
-        if encoding in ("gzip", "deflate", "br"):
+        if encoding in ("gzip", "deflate", "br", "zstd"):
             _cache = CachedDecode(encoded, encoding, errors, decoded)
         return decoded
     except TypeError:
@@ -93,7 +94,7 @@ def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optio
             encoded = custom_encode[encoding](decoded)
         except KeyError:
             encoded = codecs.encode(decoded, encoding, errors)
-        if encoding in ("gzip", "deflate", "br"):
+        if encoding in ("gzip", "deflate", "br", "zstd"):
             _cache = CachedDecode(encoded, encoding, errors, decoded)
         return encoded
     except TypeError:
@@ -140,6 +141,23 @@ def encode_brotli(content: bytes) -> bytes:
     return brotli.compress(content)
 
 
+def decode_zstd(content: bytes) -> bytes:
+    if not content:
+        return b""
+    zstd_ctx = zstd.ZstdDecompressor()
+    try:
+        return zstd_ctx.decompress(content)
+    except zstd.ZstdError:
+        # If the zstd stream is streamed without a size header,
+        # try decoding with a 10MiB output buffer
+        return zstd_ctx.decompress(content, max_output_size=10 * 2**20)
+
+
+def encode_zstd(content: bytes) -> bytes:
+    zstd_ctx = zstd.ZstdCompressor()
+    return zstd_ctx.compress(content)
+
+
 def decode_deflate(content: bytes) -> bytes:
     """
         Returns decompressed data for DEFLATE. Some servers may respond with
@@ -170,6 +188,7 @@ custom_decode = {
     "gzip": decode_gzip,
     "deflate": decode_deflate,
     "br": decode_brotli,
+    "zstd": decode_zstd,
 }
 custom_encode = {
     "none": identity,
@@ -177,6 +196,7 @@ custom_encode = {
     "gzip": encode_gzip,
     "deflate": encode_deflate,
     "br": encode_brotli,
+    "zstd": encode_zstd,
 }
 
 __all__ = ["encode", "decode"]
diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py
index 86782e8a..af7b032b 100644
--- a/mitmproxy/net/http/message.py
+++ b/mitmproxy/net/http/message.py
@@ -1,14 +1,18 @@
 import re
-from typing import Optional, Union  # noqa
+from typing import Optional  # noqa
 
 from mitmproxy.utils import strutils
 from mitmproxy.net.http import encoding
 from mitmproxy.coretypes import serializable
-from mitmproxy.net.http import headers
+from mitmproxy.net.http import headers as mheaders
 
 
 class MessageData(serializable.Serializable):
-    content: bytes = None
+    headers: mheaders.Headers
+    content: bytes
+    http_version: bytes
+    timestamp_start: float
+    timestamp_end: float
 
     def __eq__(self, other):
         if isinstance(other, MessageData):
@@ -18,7 +22,7 @@ class MessageData(serializable.Serializable):
     def set_state(self, state):
         for k, v in state.items():
             if k == "headers":
-                v = headers.Headers.from_state(v)
+                v = mheaders.Headers.from_state(v)
             setattr(self, k, v)
 
     def get_state(self):
@@ -28,12 +32,12 @@ class MessageData(serializable.Serializable):
 
     @classmethod
     def from_state(cls, state):
-        state["headers"] = headers.Headers.from_state(state["headers"])
+        state["headers"] = mheaders.Headers.from_state(state["headers"])
         return cls(**state)
 
 
 class Message(serializable.Serializable):
-    data: MessageData = None
+    data: MessageData
 
     def __eq__(self, other):
         if isinstance(other, Message):
@@ -48,7 +52,7 @@ class Message(serializable.Serializable):
 
     @classmethod
     def from_state(cls, state):
-        state["headers"] = headers.Headers.from_state(state["headers"])
+        state["headers"] = mheaders.Headers.from_state(state["headers"])
         return cls(**state)
 
     @property
@@ -78,7 +82,7 @@ class Message(serializable.Serializable):
     def raw_content(self, content):
         self.data.content = content
 
-    def get_content(self, strict: bool=True) -> bytes:
+    def get_content(self, strict: bool=True) -> Optional[bytes]:
         """
         The uncompressed HTTP message body as bytes.
 
@@ -160,7 +164,7 @@ class Message(serializable.Serializable):
         self.data.timestamp_end = timestamp_end
 
     def _get_content_type_charset(self) -> Optional[str]:
-        ct = headers.parse_content_type(self.headers.get("content-type", ""))
+        ct = mheaders.parse_content_type(self.headers.get("content-type", ""))
         if ct:
             return ct[2].get("charset")
         return None
@@ -191,10 +195,9 @@ class Message(serializable.Serializable):
 
         See also: :py:attr:`content`, :py:class:`raw_content`
         """
-        if self.raw_content is None:
-            return None
-
         content = self.get_content(strict)
+        if content is None:
+            return None
         enc = self._guess_encoding(content)
         try:
             return encoding.decode(content, enc)
@@ -213,9 +216,9 @@ class Message(serializable.Serializable):
             self.content = encoding.encode(text, enc)
         except ValueError:
             # Fall back to UTF-8 and update the content-type header.
-            ct = headers.parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {})
+            ct = mheaders.parse_content_type(self.headers.get("content-type", "")) or ("text", "plain", {})
             ct[2]["charset"] = "utf-8"
-            self.headers["content-type"] = headers.assemble_content_type(*ct)
+            self.headers["content-type"] = mheaders.assemble_content_type(*ct)
             enc = "utf8"
             self.content = text.encode(enc, "surrogateescape")
 
@@ -236,7 +239,7 @@ class Message(serializable.Serializable):
 
     def encode(self, e):
         """
-        Encodes body with the encoding e, where e is "gzip", "deflate", "identity", or "br".
+        Encodes body with the encoding e, where e is "gzip", "deflate", "identity", "br", or "zstd".
         Any existing content-encodings are overwritten,
         the content is not decoded beforehand.
 
diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py
index 959fdd33..1569ea72 100644
--- a/mitmproxy/net/http/request.py
+++ b/mitmproxy/net/http/request.py
@@ -1,5 +1,6 @@
 import re
 import urllib
+import time
 from typing import Optional, AnyStr, Dict, Iterable, Tuple, Union
 
 from mitmproxy.coretypes import multidict
@@ -63,6 +64,8 @@ class Request(message.Message):
     """
     An HTTP request.
     """
+    data: RequestData
+
     def __init__(self, *args, **kwargs):
         super().__init__()
         self.data = RequestData(*args, **kwargs)
@@ -101,6 +104,7 @@ class Request(message.Message):
         )
 
         req.url = url
+        req.timestamp_start = time.time()
 
         # Headers can be list or dict, we differentiate here.
         if isinstance(headers, dict):
@@ -421,7 +425,7 @@ class Request(message.Message):
             self.headers["accept-encoding"] = (
                 ', '.join(
                     e
-                    for e in {"gzip", "identity", "deflate", "br"}
+                    for e in {"gzip", "identity", "deflate", "br", "zstd"}
                     if e in accept_encoding
                 )
             )
diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py
index 48527d63..2e864405 100644
--- a/mitmproxy/net/http/response.py
+++ b/mitmproxy/net/http/response.py
@@ -47,6 +47,8 @@ class Response(message.Message):
     """
     An HTTP response.
     """
+    data: ResponseData
+
     def __init__(self, *args, **kwargs):
         super().__init__()
         self.data = ResponseData(*args, **kwargs)
@@ -186,7 +188,7 @@ class Response(message.Message):
                 d = parsedate_tz(self.headers[i])
                 if d:
                     new = mktime_tz(d) + delta
-                    self.headers[i] = formatdate(new)
+                    self.headers[i] = formatdate(new, usegmt=True)
         c = []
         for set_cookie_header in self.headers.get_all("set-cookie"):
             try:
diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py
index f938cb12..d8e14aeb 100644
--- a/mitmproxy/net/http/url.py
+++ b/mitmproxy/net/http/url.py
@@ -21,16 +21,25 @@ def parse(url):
         Raises:
             ValueError, if the URL is not properly formatted.
     """
-    parsed = urllib.parse.urlparse(url)
+    # Size of Ascii character after encoding is 1 byte which is same as its size
+    # But non-Ascii character's size after encoding will be more than its size
+    def ascii_check(l):
+        if len(l) == len(str(l).encode()):
+            return True
+        return False
+
+    if isinstance(url, bytes):
+        url = url.decode()
+        if not ascii_check(url):
+            url = urllib.parse.urlsplit(url)
+            url = list(url)
+            url[3] = urllib.parse.quote(url[3])
+            url = urllib.parse.urlunsplit(url)
 
+    parsed = urllib.parse.urlparse(url)
     if not parsed.hostname:
         raise ValueError("No hostname given")
 
-    if isinstance(url, bytes):
-        host = parsed.hostname
-
-        # this should not raise a ValueError,
-        # but we try to be very forgiving here and accept just everything.
     else:
         host = parsed.hostname.encode("idna")
         if isinstance(parsed, urllib.parse.ParseResult):
diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py
index 4dc61969..d68a008f 100644
--- a/mitmproxy/net/tls.py
+++ b/mitmproxy/net/tls.py
@@ -295,6 +295,17 @@ def create_client_context(
     return context
 
 
+def accept_all(
+        conn_: SSL.Connection,
+        x509: SSL.X509,
+        errno: int,
+        err_depth: int,
+        is_cert_verified: bool,
+) -> bool:
+    # Return true to prevent cert verification error
+    return True
+
+
 def create_server_context(
         cert: typing.Union[certs.Cert, str],
         key: SSL.PKey,
@@ -324,16 +335,6 @@ def create_server_context(
         until then we're conservative.
     """
 
-    def accept_all(
-            conn_: SSL.Connection,
-            x509: SSL.X509,
-            errno: int,
-            err_depth: int,
-            is_cert_verified: bool,
-    ) -> bool:
-        # Return true to prevent cert verification error
-        return True
-
     if request_client_cert:
         verify = SSL.VERIFY_PEER
     else:
@@ -425,7 +426,7 @@ class ClientHello:
         return self._client_hello.cipher_suites.cipher_suites
 
     @property
-    def sni(self):
+    def sni(self) -> typing.Optional[bytes]:
         if self._client_hello.extensions:
             for extension in self._client_hello.extensions.extensions:
                 is_valid_sni_extension = (
@@ -435,7 +436,7 @@ class ClientHello:
                     check.is_valid_host(extension.body.server_names[0].host_name)
                 )
                 if is_valid_sni_extension:
-                    return extension.body.server_names[0].host_name.decode("idna")
+                    return extension.body.server_names[0].host_name
         return None
 
     @property
@@ -473,10 +474,8 @@ class ClientHello:
             return cls(raw_client_hello)
         except EOFError as e:
             raise exceptions.TlsProtocolException(
-                'Cannot parse Client Hello: %s, Raw Client Hello: %s' %
-                (repr(e), binascii.hexlify(raw_client_hello))
+                f"Cannot parse Client Hello: {e!r}, Raw Client Hello: {binascii.hexlify(raw_client_hello)!r}"
             )
 
     def __repr__(self):
-        return "ClientHello(sni: %s, alpn_protocols: %s, cipher_suites: %s)" % \
-               (self.sni, self.alpn_protocols, self.cipher_suites)
+        return f"ClientHello(sni: {self.sni}, alpn_protocols: {self.alpn_protocols})"
diff --git a/mitmproxy/net/websockets/masker.py b/mitmproxy/net/websockets/masker.py
index 47b1a688..6134e09e 100644
--- a/mitmproxy/net/websockets/masker.py
+++ b/mitmproxy/net/websockets/masker.py
@@ -1,3 +1,6 @@
+import sys
+
+
 class Masker:
     """
     Data sent from the server must be masked to prevent malicious clients
@@ -12,12 +15,13 @@ class Masker:
         self.offset = 0
 
     def mask(self, offset, data):
-        result = bytearray(data)
-        for i in range(len(data)):
-            result[i] ^= self.key[offset % 4]
-            offset += 1
-        result = bytes(result)
-        return result
+        datalen = len(data)
+        offset_mod = offset % 4
+        data = int.from_bytes(data, sys.byteorder)
+        num_keys = (datalen + offset_mod + 3) // 4
+        mask = int.from_bytes((self.key * num_keys)[offset_mod:datalen +
+                                                    offset_mod], sys.byteorder)
+        return (data ^ mask).to_bytes(datalen, sys.byteorder)
 
     def __call__(self, data):
         ret = self.mask(self.offset, data)
diff --git a/mitmproxy/options.py b/mitmproxy/options.py
index a6ab3d50..1583e9fc 100644
--- a/mitmproxy/options.py
+++ b/mitmproxy/options.py
@@ -5,8 +5,10 @@ from mitmproxy.net import tls
 
 
 CONF_DIR = "~/.mitmproxy"
+CONF_BASENAME = "mitmproxy"
 LISTEN_PORT = 8080
 CONTENT_VIEW_LINES_CUTOFF = 512
+KEY_SIZE = 2048
 
 
 class Options(optmanager.OptManager):
@@ -68,6 +70,10 @@ class Options(optmanager.OptManager):
             """
         )
         self.add_option(
+            "allow_hosts", Sequence[str], [],
+            "Opposite of --ignore-hosts."
+        )
+        self.add_option(
             "listen_host", str, "",
             "Address to bind proxy to."
         )
@@ -169,5 +175,11 @@ class Options(optmanager.OptManager):
             speedup flows browsing.
             """
         )
+        self.add_option(
+            "key_size", int, KEY_SIZE,
+            """
+            TLS key size for certificates and CA.
+            """
+        )
 
         self.update(**kwargs)
diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py
index 06e696c0..f42aa645 100644
--- a/mitmproxy/optmanager.py
+++ b/mitmproxy/optmanager.py
@@ -320,7 +320,9 @@ class OptManager:
         update = {}
         for optname, optval in self.deferred.items():
             if optname in self._options:
-                update[optname] = self.parse_setval(self._options[optname], optval)
+                if isinstance(optval, str):
+                    optval = self.parse_setval(self._options[optname], optval)
+                update[optname] = optval
         self.update(**update)
         for k in update.keys():
             del self.deferred[k]
@@ -549,7 +551,9 @@ def serialize(opts: OptManager, text: str, defaults: bool = False) -> str:
     for k in list(data.keys()):
         if k not in opts._options:
             del data[k]
-    return ruamel.yaml.round_trip_dump(data)
+    ret = ruamel.yaml.round_trip_dump(data)
+    assert ret
+    return ret
 
 
 def save(opts: OptManager, path: str, defaults: bool =False) -> None:
diff --git a/mitmproxy/platform/__init__.py b/mitmproxy/platform/__init__.py
index 61946ec4..7e690789 100644
--- a/mitmproxy/platform/__init__.py
+++ b/mitmproxy/platform/__init__.py
@@ -1,7 +1,7 @@
 import re
 import socket
 import sys
-from typing import Tuple
+from typing import Callable, Optional, Tuple
 
 
 def init_transparent_mode() -> None:
@@ -10,30 +10,34 @@ def init_transparent_mode() -> None:
     """
 
 
-def original_addr(csock: socket.socket) -> Tuple[str, int]:
-    """
-    Get the original destination for the given socket.
-    This function will be None if transparent mode is not supported.
-    """
-
+original_addr: Optional[Callable[[socket.socket], Tuple[str, int]]]
+"""
+Get the original destination for the given socket.
+This function will be None if transparent mode is not supported.
+"""
 
 if re.match(r"linux(?:2)?", sys.platform):
     from . import linux
 
-    original_addr = linux.original_addr  # noqa
+    original_addr = linux.original_addr
 elif sys.platform == "darwin" or sys.platform.startswith("freebsd"):
     from . import osx
 
-    original_addr = osx.original_addr  # noqa
+    original_addr = osx.original_addr
 elif sys.platform.startswith("openbsd"):
     from . import openbsd
 
-    original_addr = openbsd.original_addr  # noqa
+    original_addr = openbsd.original_addr
 elif sys.platform == "win32":
     from . import windows
 
     resolver = windows.Resolver()
     init_transparent_mode = resolver.setup  # noqa
-    original_addr = resolver.original_addr  # noqa
+    original_addr = resolver.original_addr
 else:
-    original_addr = None  # noqa
+    original_addr = None
+
+__all__ = [
+    "original_addr",
+    "init_transparent_mode"
+]
diff --git a/mitmproxy/platform/pf.py b/mitmproxy/platform/pf.py
index 5e22ec31..74e077a4 100644
--- a/mitmproxy/platform/pf.py
+++ b/mitmproxy/platform/pf.py
@@ -13,9 +13,15 @@ def lookup(address, port, s):
     # Those still appear as "127.0.0.1" in the table, so we need to strip the prefix.
     address = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", address)
     s = s.decode()
-    spec = "%s:%s" % (address, port)
+
+    # ALL tcp 192.168.1.13:57474 -> 23.205.82.58:443       ESTABLISHED:ESTABLISHED
+    specv4 = "%s:%s" % (address, port)
+
+    # ALL tcp 2a01:e35:8bae:50f0:9d9b:ef0d:2de3:b733[58505] -> 2606:4700:30::681f:4ad0[443]       ESTABLISHED:ESTABLISHED
+    specv6 = "%s[%s]" % (address, port)
+
     for i in s.split("\n"):
-        if "ESTABLISHED:ESTABLISHED" in i and spec in i:
+        if "ESTABLISHED:ESTABLISHED" in i and specv4 in i:
             s = i.split()
             if len(s) > 4:
                 if sys.platform.startswith("freebsd"):
@@ -26,4 +32,11 @@ def lookup(address, port, s):
 
                 if len(s) == 2:
                     return s[0], int(s[1])
+        elif "ESTABLISHED:ESTABLISHED" in i and specv6 in i:
+            s = i.split()
+            if len(s) > 4:
+                s = s[4].split("[")
+                port = s[1].split("]")
+                port = port[0]
+                return s[0], int(port)
     raise RuntimeError("Could not resolve original destination.")
diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py
index cb0a7096..19d9abd4 100644
--- a/mitmproxy/platform/windows.py
+++ b/mitmproxy/platform/windows.py
@@ -13,6 +13,7 @@ import typing
 
 import click
 import collections
+import collections.abc
 import pydivert
 import pydivert.consts
 
@@ -171,7 +172,7 @@ def MIB_TCPTABLE_OWNER_PID(size):
 TCP_TABLE_OWNER_PID_CONNECTIONS = 4
 
 
-class TcpConnectionTable(collections.Mapping):
+class TcpConnectionTable(collections.abc.Mapping):
     DEFAULT_TABLE_SIZE = 4096
 
     def __init__(self):
diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py
index f32d3086..e98faabf 100644
--- a/mitmproxy/proxy/config.py
+++ b/mitmproxy/proxy/config.py
@@ -4,17 +4,15 @@ import typing
 
 from OpenSSL import crypto
 
+from mitmproxy import certs
 from mitmproxy import exceptions
 from mitmproxy import options as moptions
-from mitmproxy import certs
 from mitmproxy.net import server_spec
 
-CONF_BASENAME = "mitmproxy"
-
 
 class HostMatcher:
-
-    def __init__(self, patterns=tuple()):
+    def __init__(self, handle, patterns=tuple()):
+        self.handle = handle
         self.patterns = list(patterns)
         self.regexes = [re.compile(p, re.IGNORECASE) for p in self.patterns]
 
@@ -22,10 +20,10 @@ class HostMatcher:
         if not address:
             return False
         host = "%s:%s" % address
-        if any(rex.search(host) for rex in self.regexes):
-            return True
-        else:
-            return False
+        if self.handle in ["ignore", "tcp"]:
+            return any(rex.search(host) for rex in self.regexes)
+        else:  # self.handle == "allow"
+            return any(not rex.search(host) for rex in self.regexes)
 
     def __bool__(self):
         return bool(self.patterns)
@@ -36,18 +34,26 @@ class ProxyConfig:
     def __init__(self, options: moptions.Options) -> None:
         self.options = options
 
-        self.check_ignore: HostMatcher = None
-        self.check_tcp: HostMatcher = None
-        self.certstore: certs.CertStore = None
+        self.certstore: certs.CertStore
+        self.check_filter: typing.Optional[HostMatcher] = None
+        self.check_tcp: typing.Optional[HostMatcher] = None
         self.upstream_server: typing.Optional[server_spec.ServerSpec] = None
         self.configure(options, set(options.keys()))
         options.changed.connect(self.configure)
 
     def configure(self, options: moptions.Options, updated: typing.Any) -> None:
-        if "ignore_hosts" in updated:
-            self.check_ignore = HostMatcher(options.ignore_hosts)
+        if options.allow_hosts and options.ignore_hosts:
+            raise exceptions.OptionsError("--ignore-hosts and --allow-hosts are mutually "
+                                          "exclusive; please choose one.")
+
+        if options.ignore_hosts:
+            self.check_filter = HostMatcher("ignore", options.ignore_hosts)
+        elif options.allow_hosts:
+            self.check_filter = HostMatcher("allow", options.allow_hosts)
+        else:
+            self.check_filter = HostMatcher(False)
         if "tcp_hosts" in updated:
-            self.check_tcp = HostMatcher(options.tcp_hosts)
+            self.check_tcp = HostMatcher("tcp", options.tcp_hosts)
 
         certstore_path = os.path.expanduser(options.confdir)
         if not os.path.exists(os.path.dirname(certstore_path)):
@@ -55,9 +61,11 @@ class ProxyConfig:
                 "Certificate Authority parent directory does not exist: %s" %
                 os.path.dirname(certstore_path)
             )
+        key_size = options.key_size
         self.certstore = certs.CertStore.from_store(
             certstore_path,
-            CONF_BASENAME
+            moptions.CONF_BASENAME,
+            key_size
         )
 
         for c in options.certs:
diff --git a/mitmproxy/proxy/protocol/http2.py b/mitmproxy/proxy/protocol/http2.py
index 42b61f4d..a5870e6c 100644
--- a/mitmproxy/proxy/protocol/http2.py
+++ b/mitmproxy/proxy/protocol/http2.py
@@ -1,7 +1,7 @@
 import threading
 import time
 import functools
-from typing import Dict, Callable, Any, List  # noqa
+from typing import Dict, Callable, Any, List, Optional  # noqa
 
 import h2.exceptions
 from h2 import connection
@@ -382,15 +382,15 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr
             ctx, name="Http2SingleStreamLayer-{}".format(stream_id)
         )
         self.h2_connection = h2_connection
-        self.zombie: float = None
+        self.zombie: Optional[float] = None
         self.client_stream_id: int = stream_id
-        self.server_stream_id: int = None
+        self.server_stream_id: Optional[int] = None
         self.request_headers = request_headers
-        self.response_headers: mitmproxy.net.http.Headers = None
+        self.response_headers: Optional[mitmproxy.net.http.Headers] = None
         self.pushed = False
 
-        self.timestamp_start: float = None
-        self.timestamp_end: float = None
+        self.timestamp_start: Optional[float] = None
+        self.timestamp_end: Optional[float] = None
 
         self.request_arrived = threading.Event()
         self.request_data_queue: queue.Queue[bytes] = queue.Queue()
@@ -404,9 +404,9 @@ class Http2SingleStreamLayer(httpbase._HttpTransmissionLayer, basethread.BaseThr
 
         self.no_body = False
 
-        self.priority_exclusive: bool = None
-        self.priority_depends_on: int = None
-        self.priority_weight: int = None
+        self.priority_exclusive: bool
+        self.priority_depends_on: Optional[int] = None
+        self.priority_weight: Optional[int] = None
         self.handled_priority_event: Any = None
 
     def kill(self):
diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py
index 096aae9f..282df60d 100644
--- a/mitmproxy/proxy/protocol/tls.py
+++ b/mitmproxy/proxy/protocol/tls.py
@@ -196,17 +196,14 @@ CIPHER_ID_NAME_MAP = {
 }
 
 # We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default.
-# https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old
+# https://ssl-config.mozilla.org/#config=old
 DEFAULT_CLIENT_CIPHERS = (
-    "ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES256-GCM-SHA384:"
-    "ECDHE-ECDSA-AES256-GCM-SHA384:DHE-RSA-AES128-GCM-SHA256:DHE-DSS-AES128-GCM-SHA256:kEDH+AESGCM:"
-    "ECDHE-RSA-AES128-SHA256:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES128-SHA:"
-    "ECDHE-RSA-AES256-SHA384:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA:ECDHE-ECDSA-AES256-SHA:"
-    "DHE-RSA-AES128-SHA256:DHE-RSA-AES128-SHA:DHE-DSS-AES128-SHA256:DHE-RSA-AES256-SHA256:DHE-DSS-AES256-SHA:"
-    "DHE-RSA-AES256-SHA:ECDHE-RSA-DES-CBC3-SHA:ECDHE-ECDSA-DES-CBC3-SHA:AES128-GCM-SHA256:AES256-GCM-SHA384:"
-    "AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:AES:DES-CBC3-SHA:"
-    "HIGH:!aNULL:!eNULL:!EXPORT:!DES:!RC4:!MD5:!PSK:!aECDH:"
-    "!EDH-DSS-DES-CBC3-SHA:!EDH-RSA-DES-CBC3-SHA:!KRB5-DES-CBC3-SHA"
+    "ECDHE-ECDSA-AES128-GCM-SHA256:ECDHE-RSA-AES128-GCM-SHA256:ECDHE-ECDSA-AES256-GCM-SHA384:"
+    "ECDHE-RSA-AES256-GCM-SHA384:ECDHE-ECDSA-CHACHA20-POLY1305:ECDHE-RSA-CHACHA20-POLY1305:DHE-RSA-AES128-GCM-SHA256:"
+    "DHE-RSA-AES256-GCM-SHA384:DHE-RSA-CHACHA20-POLY1305:ECDHE-ECDSA-AES128-SHA256:ECDHE-RSA-AES128-SHA256:"
+    "ECDHE-ECDSA-AES128-SHA:ECDHE-RSA-AES128-SHA:ECDHE-ECDSA-AES256-SHA384:ECDHE-RSA-AES256-SHA384:"
+    "ECDHE-ECDSA-AES256-SHA:ECDHE-RSA-AES256-SHA:DHE-RSA-AES128-SHA256:DHE-RSA-AES256-SHA256:AES128-GCM-SHA256:"
+    "AES256-GCM-SHA384:AES128-SHA256:AES256-SHA256:AES128-SHA:AES256-SHA:DES-CBC3-SHA"
 )
 
 
@@ -323,14 +320,18 @@ class TlsLayer(base.Layer):
         return self._server_tls
 
     @property
-    def server_sni(self):
+    def server_sni(self) -> Optional[str]:
         """
         The Server Name Indication we want to send with the next server TLS handshake.
         """
         if self._custom_server_sni is False:
             return None
+        elif self._custom_server_sni:
+            return self._custom_server_sni
+        elif self._client_hello and self._client_hello.sni:
+            return self._client_hello.sni.decode("idna")
         else:
-            return self._custom_server_sni or self._client_hello and self._client_hello.sni
+            return None
 
     @property
     def alpn_for_client_connection(self):
@@ -391,11 +392,12 @@ class TlsLayer(base.Layer):
             # raises ann error.
             self.client_conn.rfile.peek(1)
         except exceptions.TlsException as e:
+            sni_str = self._client_hello.sni and self._client_hello.sni.decode("idna")
             raise exceptions.ClientHandshakeException(
                 "Cannot establish TLS with client (sni: {sni}): {e}".format(
-                    sni=self._client_hello.sni, e=repr(e)
+                    sni=sni_str, e=repr(e)
                 ),
-                self._client_hello.sni or repr(self.server_conn.address)
+                sni_str or repr(self.server_conn.address)
             )
 
     def _establish_tls_with_server(self):
@@ -493,7 +495,7 @@ class TlsLayer(base.Layer):
                 organization = upstream_cert.organization
         # Also add SNI values.
         if self._client_hello.sni:
-            sans.add(self._client_hello.sni.encode("idna"))
+            sans.add(self._client_hello.sni)
         if self._custom_server_sni:
             sans.add(self._custom_server_sni.encode("idna"))
 
diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py
index eb0008cf..3d4e8660 100644
--- a/mitmproxy/proxy/root_context.py
+++ b/mitmproxy/proxy/root_context.py
@@ -48,17 +48,18 @@ class RootContext:
             raise exceptions.ProtocolException(str(e))
         client_tls = tls.is_tls_record_magic(d)
 
-        # 1. check for --ignore
-        if self.config.check_ignore:
-            ignore = self.config.check_ignore(top_layer.server_conn.address)
-            if not ignore and client_tls:
+        # 1. check for filter
+        if self.config.check_filter:
+            is_filtered = self.config.check_filter(top_layer.server_conn.address)
+            if not is_filtered and client_tls:
                 try:
                     client_hello = tls.ClientHello.from_file(self.client_conn.rfile)
                 except exceptions.TlsProtocolException as e:
                     self.log("Cannot parse Client Hello: %s" % repr(e), "error")
                 else:
-                    ignore = self.config.check_ignore((client_hello.sni, 443))
-            if ignore:
+                    sni_str = client_hello.sni and client_hello.sni.decode("idna")
+                    is_filtered = self.config.check_filter((sni_str, 443))
+            if is_filtered:
                 return protocol.RawTCPLayer(top_layer, ignore=True)
 
         # 2. Always insert a TLS layer, even if there's neither client nor server tls.
diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py
index 44ae5697..3688b677 100644
--- a/mitmproxy/proxy/server.py
+++ b/mitmproxy/proxy/server.py
@@ -35,6 +35,7 @@ class DummyServer:
 class ProxyServer(tcp.TCPServer):
     allow_reuse_address = True
     bound = True
+    channel: controller.Channel
 
     def __init__(self, config: config.ProxyConfig) -> None:
         """
@@ -53,7 +54,6 @@ class ProxyServer(tcp.TCPServer):
             raise exceptions.ServerException(
                 'Error starting proxy server: ' + repr(e)
             ) from e
-        self.channel: controller.Channel = None
 
     def set_channel(self, channel):
         self.channel = channel
diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py
index 2c16dcda..76329236 100644
--- a/mitmproxy/stateobject.py
+++ b/mitmproxy/stateobject.py
@@ -1,7 +1,5 @@
-import typing
-from typing import Any  # noqa
-from typing import MutableMapping  # noqa
 import json
+import typing
 
 from mitmproxy.coretypes import serializable
 from mitmproxy.utils import typecheck
@@ -15,7 +13,7 @@ class StateObject(serializable.Serializable):
     or StateObject instances themselves.
     """
 
-    _stateobject_attributes: MutableMapping[str, Any] = None
+    _stateobject_attributes: typing.ClassVar[typing.MutableMapping[str, typing.Any]]
     """
     An attribute-name -> class-or-type dict containing all attributes that
     should be serialized. If the attribute is a class, it must implement the
@@ -42,7 +40,7 @@ class StateObject(serializable.Serializable):
             if val is None:
                 setattr(self, attr, val)
             else:
-                curr = getattr(self, attr)
+                curr = getattr(self, attr, None)
                 if hasattr(curr, "set_state"):
                     curr.set_state(val)
                 else:
diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py
index f1c763b2..a00a3e98 100644
--- a/mitmproxy/tools/_main.py
+++ b/mitmproxy/tools/_main.py
@@ -6,19 +6,16 @@ Feel free to import and use whatever new package you deem necessary.
 import os
 import sys
 import asyncio
-import argparse  # noqa
-import signal  # noqa
-import typing  # noqa
+import argparse
+import signal
+import typing
 
-from mitmproxy.tools import cmdline  # noqa
-from mitmproxy import exceptions, master  # noqa
-from mitmproxy import options  # noqa
-from mitmproxy import optmanager  # noqa
-from mitmproxy import proxy  # noqa
-from mitmproxy import log  # noqa
-from mitmproxy.utils import debug, arg_check  # noqa
-
-OPTIONS_FILE_NAME = "config.yaml"
+from mitmproxy.tools import cmdline
+from mitmproxy import exceptions, master
+from mitmproxy import options
+from mitmproxy import optmanager
+from mitmproxy import proxy
+from mitmproxy.utils import debug, arg_check
 
 
 def assert_utf8_env():
@@ -87,10 +84,11 @@ def run(
         arg_check.check()
         sys.exit(1)
     try:
-        opts.confdir = args.confdir
+        opts.set(*args.setoptions, defer=True)
         optmanager.load_paths(
             opts,
-            os.path.join(opts.confdir, OPTIONS_FILE_NAME),
+            os.path.join(opts.confdir, "config.yaml"),
+            os.path.join(opts.confdir, "config.yml"),
         )
         pconf = process_options(parser, opts, args)
         server: typing.Any = None
@@ -110,7 +108,6 @@ def run(
         if args.commands:
             master.commands.dump()
             sys.exit(0)
-        opts.set(*args.setoptions, defer=True)
         if extra:
             opts.update(**extra(args))
 
diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py
index ad934ca2..e9ff973f 100644
--- a/mitmproxy/tools/cmdline.py
+++ b/mitmproxy/tools/cmdline.py
@@ -1,7 +1,5 @@
 import argparse
 
-from mitmproxy.addons import core
-
 
 def common_options(parser, opts):
     parser.add_argument(
@@ -21,12 +19,6 @@ def common_options(parser, opts):
         help="Show all commands and their signatures",
     )
     parser.add_argument(
-        "--confdir",
-        type=str, dest="confdir", default=core.CONF_DIR,
-        metavar="PATH",
-        help="Path to the mitmproxy config directory"
-    )
-    parser.add_argument(
         "--set",
         type=str, dest="setoptions", default=[],
         action="append",
@@ -65,6 +57,7 @@ def common_options(parser, opts):
     opts.make_parser(group, "listen_port", metavar="PORT", short="p")
     opts.make_parser(group, "server", short="n")
     opts.make_parser(group, "ignore_hosts", metavar="HOST")
+    opts.make_parser(group, "allow_hosts", metavar="HOST")
     opts.make_parser(group, "tcp_hosts", metavar="HOST")
     opts.make_parser(group, "upstream_auth", metavar="USER:PASS")
     opts.make_parser(group, "proxyauth", metavar="SPEC")
@@ -75,6 +68,7 @@ def common_options(parser, opts):
     group = parser.add_argument_group("SSL")
     opts.make_parser(group, "certs", metavar="SPEC")
     opts.make_parser(group, "ssl_insecure", short="k")
+    opts.make_parser(group, "key_size", metavar="KEY_SIZE")
 
     # Client replay
     group = parser.add_argument_group("Client Replay")
@@ -85,6 +79,7 @@ def common_options(parser, opts):
     opts.make_parser(group, "server_replay", metavar="PATH", short="S")
     opts.make_parser(group, "server_replay_kill_extra")
     opts.make_parser(group, "server_replay_nopop")
+    opts.make_parser(group, "server_replay_refresh")
 
     # Replacements
     group = parser.add_argument_group("Replacements")
diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py
index e8550f86..f291b8fd 100644
--- a/mitmproxy/tools/console/commander/commander.py
+++ b/mitmproxy/tools/console/commander/commander.py
@@ -55,7 +55,7 @@ class CommandBuffer:
         self.text = self.flatten(start)
         # Cursor is always within the range [0:len(buffer)].
         self._cursor = len(self.text)
-        self.completion: CompletionState = None
+        self.completion: typing.Optional[CompletionState] = None
 
     @property
     def cursor(self) -> int:
diff --git a/mitmproxy/tools/console/commandexecutor.py b/mitmproxy/tools/console/commandexecutor.py
index 3db03d3e..c738e349 100644
--- a/mitmproxy/tools/console/commandexecutor.py
+++ b/mitmproxy/tools/console/commandexecutor.py
@@ -34,4 +34,4 @@ class CommandExecutor:
                                 ret,
                             ),
                             valign="top"
-                        )
\ No newline at end of file
+                        )
diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py
index 5d7ee09d..3a5b4aeb 100644
--- a/mitmproxy/tools/console/common.py
+++ b/mitmproxy/tools/console/common.py
@@ -1,6 +1,10 @@
 import platform
 import typing
+import datetime
+import time
+import math
 from functools import lru_cache
+from publicsuffix2 import get_sld, get_tld
 
 import urwid
 import urwid.util
@@ -34,7 +38,7 @@ KEY_MAX = 30
 
 
 def format_keyvals(
-        entries: typing.List[typing.Tuple[str, typing.Union[None, str, urwid.Widget]]],
+        entries: typing.Iterable[typing.Tuple[str, typing.Union[None, str, urwid.Widget]]],
         key_format: str = "key",
         value_format: str = "text",
         indent: int = 0
@@ -97,16 +101,180 @@ if urwid.util.detected_encoding and not IS_WSL:
     SYMBOL_MARK = u"\u25cf"
     SYMBOL_UP = u"\u21E7"
     SYMBOL_DOWN = u"\u21E9"
+    SYMBOL_ELLIPSIS = u"\u2026"
 else:
     SYMBOL_REPLAY = u"[r]"
     SYMBOL_RETURN = u"<-"
     SYMBOL_MARK = "[m]"
     SYMBOL_UP = "^"
     SYMBOL_DOWN = " "
+    SYMBOL_ELLIPSIS = "~"
+
+
+def fixlen(s, maxlen):
+    if len(s) <= maxlen:
+        return s.ljust(maxlen)
+    else:
+        return s[0:maxlen - len(SYMBOL_ELLIPSIS)] + SYMBOL_ELLIPSIS
+
+
+def fixlen_r(s, maxlen):
+    if len(s) <= maxlen:
+        return s.rjust(maxlen)
+    else:
+        return SYMBOL_ELLIPSIS + s[len(s) - maxlen + len(SYMBOL_ELLIPSIS):]
+
+
+class TruncatedText(urwid.Widget):
+    def __init__(self, text, attr, align='left'):
+        self.text = text
+        self.attr = attr
+        self.align = align
+        super(TruncatedText, self).__init__()
+
+    def pack(self, size, focus=False):
+        return (len(self.text), 1)
+
+    def rows(self, size, focus=False):
+        return 1
+
+    def render(self, size, focus=False):
+        text = self.text
+        attr = self.attr
+        if self.align == 'right':
+            text = text[::-1]
+            attr = attr[::-1]
+
+        text_len = len(text)  # TODO: unicode?
+        if size is not None and len(size) > 0:
+            width = size[0]
+        else:
+            width = text_len
+
+        if width >= text_len:
+            remaining = width - text_len
+            if remaining > 0:
+                c_text = text + ' ' * remaining
+                c_attr = attr + [('text', remaining)]
+            else:
+                c_text = text
+                c_attr = attr
+        else:
+            visible_len = width - len(SYMBOL_ELLIPSIS)
+            visible_text = text[0:visible_len]
+            c_text = visible_text + SYMBOL_ELLIPSIS
+            c_attr = (urwid.util.rle_subseg(attr, 0, len(visible_text.encode())) +
+                      [('focus', len(SYMBOL_ELLIPSIS.encode()))])
+
+        if self.align == 'right':
+            c_text = c_text[::-1]
+            c_attr = c_attr[::-1]
+
+        return urwid.TextCanvas([c_text.encode()], [c_attr], maxcol=width)
+
+
+def truncated_plain(text, attr, align='left'):
+    return TruncatedText(text, [(attr, len(text.encode()))], align)
+
+
+# Work around https://github.com/urwid/urwid/pull/330
+def rle_append_beginning_modify(rle, a_r):
+    """
+    Append (a, r) (unpacked from *a_r*) to BEGINNING of rle.
+    Merge with first run when possible
+
+    MODIFIES rle parameter contents. Returns None.
+    """
+    a, r = a_r
+    if not rle:
+        rle[:] = [(a, r)]
+    else:
+        al, run = rle[0]
+        if a == al:
+            rle[0] = (a, run + r)
+        else:
+            rle[0:0] = [(a, r)]
+
+
+def colorize_host(host):
+    tld = get_tld(host)
+    sld = get_sld(host)
+
+    attr = []
+
+    tld_size = len(tld)
+    sld_size = len(sld) - tld_size
+
+    for letter in reversed(range(len(host))):
+        character = host[letter]
+        if tld_size > 0:
+            style = 'url_domain'
+            tld_size -= 1
+        elif tld_size == 0:
+            style = 'text'
+            tld_size -= 1
+        elif sld_size > 0:
+            sld_size -= 1
+            style = 'url_extension'
+        else:
+            style = 'text'
+        rle_append_beginning_modify(attr, (style, len(character.encode())))
+    return attr
+
+
+def colorize_req(s):
+    path = s.split('?', 2)[0]
+    i_query = len(path)
+    i_last_slash = path.rfind('/')
+    i_ext = path[i_last_slash + 1:].rfind('.')
+    i_ext = i_last_slash + i_ext if i_ext >= 0 else len(s)
+    in_val = False
+    attr = []
+    for i in range(len(s)):
+        c = s[i]
+        if ((i < i_query and c == '/') or
+            (i < i_query and i > i_last_slash and c == '.') or
+           (i == i_query)):
+            a = 'url_punctuation'
+        elif i > i_query:
+            if in_val:
+                if c == '&':
+                    in_val = False
+                    a = 'url_punctuation'
+                else:
+                    a = 'url_query_value'
+            else:
+                if c == '=':
+                    in_val = True
+                    a = 'url_punctuation'
+                else:
+                    a = 'url_query_key'
+        elif i > i_ext:
+            a = 'url_extension'
+        elif i > i_last_slash:
+            a = 'url_filename'
+        else:
+            a = 'text'
+        urwid.util.rle_append_modify(attr, (a, len(c.encode())))
+    return attr
+
+
+def colorize_url(url):
+    parts = url.split('/', 3)
+    if len(parts) < 4 or len(parts[1]) > 0 or parts[0][-1:] != ':':
+        return [('error', len(url))]  # bad URL
+    schemes = {
+        'http:': 'scheme_http',
+        'https:': 'scheme_https',
+    }
+    return [
+        (schemes.get(parts[0], "scheme_other"), len(parts[0]) - 1),
+        ('url_punctuation', 3),  # ://
+    ] + colorize_host(parts[2]) + colorize_req('/' + parts[3])
 
 
 @lru_cache(maxsize=800)
-def raw_format_flow(f):
+def raw_format_list(f):
     f = dict(f)
     pile = []
     req = []
@@ -139,8 +307,8 @@ def raw_format_flow(f):
 
     url = f["req_url"]
 
-    if f["max_url_len"] and len(url) > f["max_url_len"]:
-        url = url[:f["max_url_len"]] + "…"
+    if f["cols"] and len(url) > f["cols"]:
+        url = url[:f["cols"]] + "…"
 
     if f["req_http_version"] not in ("HTTP/1.0", "HTTP/1.1"):
         url += " " + f["req_http_version"]
@@ -177,7 +345,8 @@ def raw_format_flow(f):
         if f["resp_ctype"]:
             resp.append(fcol(f["resp_ctype"], rc))
         resp.append(fcol(f["resp_clen"], rc))
-        resp.append(fcol(f["roundtrip"], rc))
+        pretty_duration = human.pretty_duration(f["duration"])
+        resp.append(fcol(pretty_duration, rc))
 
     elif f["err_msg"]:
         resp.append(fcol(SYMBOL_RETURN, "error"))
@@ -193,49 +362,203 @@ def raw_format_flow(f):
     return urwid.Pile(pile)
 
 
-def format_flow(f, focus, extended=False, hostheader=False, max_url_len=False):
+@lru_cache(maxsize=800)
+def raw_format_table(f):
+    f = dict(f)
+    pile = []
+    req = []
+
+    cursor = [' ', 'focus']
+    if f.get('resp_is_replay', False):
+        cursor[0] = SYMBOL_REPLAY
+        cursor[1] = 'replay'
+    if f['marked']:
+        if cursor[0] == ' ':
+            cursor[0] = SYMBOL_MARK
+        cursor[1] = 'mark'
+    if f['focus']:
+        cursor[0] = '>'
+
+    req.append(fcol(*cursor))
+
+    if f["two_line"]:
+        req.append(TruncatedText(f["req_url"], colorize_url(f["req_url"]), 'left'))
+        pile.append(urwid.Columns(req, dividechars=1))
+
+        req = []
+        req.append(fcol('  ', 'text'))
+
+    if f["intercepted"] and not f["acked"]:
+        uc = "intercept"
+    elif "resp_code" in f or f["err_msg"] is not None:
+        uc = "highlight"
+    else:
+        uc = "title"
+
+    if f["extended"]:
+        s = human.format_timestamp(f["req_timestamp"])
+    else:
+        s = datetime.datetime.fromtimestamp(time.mktime(time.localtime(f["req_timestamp"]))).strftime("%H:%M:%S")
+    req.append(fcol(s, uc))
+
+    methods = {
+        'GET': 'method_get',
+        'POST': 'method_post',
+        'DELETE': 'method_delete',
+        'HEAD': 'method_head',
+        'PUT': 'method_put'
+    }
+    uc = methods.get(f["req_method"], "method_other")
+    if f['extended']:
+        req.append(fcol(f["req_method"], uc))
+        if f["req_promise"]:
+            req.append(fcol('PUSH_PROMISE', 'method_http2_push'))
+    else:
+        if f["req_promise"]:
+            uc = 'method_http2_push'
+        req.append(("fixed", 4, truncated_plain(f["req_method"], uc)))
+
+    if f["two_line"]:
+        req.append(fcol(f["req_http_version"], 'text'))
+    else:
+        schemes = {
+            'http': 'scheme_http',
+            'https': 'scheme_https',
+        }
+        req.append(fcol(fixlen(f["req_scheme"].upper(), 5), schemes.get(f["req_scheme"], "scheme_other")))
+
+        req.append(('weight', 0.25, TruncatedText(f["req_host"], colorize_host(f["req_host"]), 'right')))
+        req.append(('weight', 1.0, TruncatedText(f["req_path"], colorize_req(f["req_path"]), 'left')))
+
+    ret = (' ' * len(SYMBOL_RETURN), 'text')
+    status = ('', 'text')
+    content = ('', 'text')
+    size = ('', 'text')
+    duration = ('', 'text')
+
+    if "resp_code" in f:
+        codes = {
+            2: "code_200",
+            3: "code_300",
+            4: "code_400",
+            5: "code_500",
+        }
+        ccol = codes.get(f["resp_code"] // 100, "code_other")
+        ret = (SYMBOL_RETURN, ccol)
+        status = (str(f["resp_code"]), ccol)
+
+        if f["resp_len"] < 0:
+            if f["intercepted"] and f["resp_code"] and not f["acked"]:
+                rc = "intercept"
+            else:
+                rc = "content_none"
+
+            if f["resp_len"] == -1:
+                contentdesc = "[content missing]"
+            else:
+                contentdesc = "[no content]"
+            content = (contentdesc, rc)
+        else:
+            if f["resp_ctype"]:
+                ctype = f["resp_ctype"].split(";")[0]
+                if ctype.endswith('/javascript'):
+                    rc = 'content_script'
+                elif ctype.startswith('text/'):
+                    rc = 'content_text'
+                elif (ctype.startswith('image/') or
+                      ctype.startswith('video/') or
+                      ctype.startswith('font/') or
+                      "/x-font-" in ctype):
+                    rc = 'content_media'
+                elif ctype.endswith('/json') or ctype.endswith('/xml'):
+                    rc = 'content_data'
+                elif ctype.startswith('application/'):
+                    rc = 'content_raw'
+                else:
+                    rc = 'content_other'
+                content = (ctype, rc)
+
+            rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + f["resp_len"]) / 20, 0.99))
+
+            size_str = human.pretty_size(f["resp_len"])
+            if not f['extended']:
+                # shorten to 5 chars max
+                if len(size_str) > 5:
+                    size_str = size_str[0:4].rstrip('.') + size_str[-1:]
+            size = (size_str, rc)
+
+        if f['duration'] is not None:
+            rc = 'gradient_%02d' % int(99 - 100 * min(math.log2(1 + 1000 * f['duration']) / 12, 0.99))
+            duration = (human.pretty_duration(f['duration']), rc)
+
+    elif f["err_msg"]:
+        status = ('Err', 'error')
+        content = f["err_msg"], 'error'
+
+    if f["two_line"]:
+        req.append(fcol(*ret))
+    req.append(fcol(fixlen(status[0], 3), status[1]))
+    req.append(('weight', 0.15, truncated_plain(content[0], content[1], 'right')))
+    if f['extended']:
+        req.append(fcol(*size))
+    else:
+        req.append(fcol(fixlen_r(size[0], 5), size[1]))
+    req.append(fcol(fixlen_r(duration[0], 5), duration[1]))
+
+    pile.append(urwid.Columns(req, dividechars=1, min_width=15))
+
+    return urwid.Pile(pile)
+
+
+def format_flow(f, focus, extended=False, hostheader=False, cols=False, layout='default'):
     acked = False
     if f.reply and f.reply.state == "committed":
         acked = True
-    pushed = ' PUSH_PROMISE' if 'h2-pushed-stream' in f.metadata else ''
     d = dict(
         focus=focus,
         extended=extended,
-        max_url_len=max_url_len,
+        two_line=extended or cols < 100,
+        cols=cols,
         intercepted=f.intercepted,
         acked=acked,
         req_timestamp=f.request.timestamp_start,
         req_is_replay=f.request.is_replay,
-        req_method=f.request.method + pushed,
+        req_method=f.request.method,
+        req_promise='h2-pushed-stream' in f.metadata,
         req_url=f.request.pretty_url if hostheader else f.request.url,
+        req_scheme=f.request.scheme,
+        req_host=f.request.pretty_host if hostheader else f.request.host,
+        req_path=f.request.path,
         req_http_version=f.request.http_version,
         err_msg=f.error.msg if f.error else None,
         marked=f.marked,
     )
     if f.response:
         if f.response.raw_content:
+            content_len = len(f.response.raw_content)
             contentdesc = human.pretty_size(len(f.response.raw_content))
         elif f.response.raw_content is None:
+            content_len = -1
             contentdesc = "[content missing]"
         else:
+            content_len = -2
             contentdesc = "[no content]"
-        duration = 0
+
+        duration = None
         if f.response.timestamp_end and f.request.timestamp_start:
             duration = f.response.timestamp_end - f.request.timestamp_start
-        roundtrip = human.pretty_duration(duration)
 
         d.update(dict(
             resp_code=f.response.status_code,
             resp_reason=f.response.reason,
             resp_is_replay=f.response.is_replay,
+            resp_len=content_len,
+            resp_ctype=f.response.headers.get("content-type"),
             resp_clen=contentdesc,
-            roundtrip=roundtrip,
+            duration=duration,
         ))
 
-        t = f.response.headers.get("content-type")
-        if t:
-            d["resp_ctype"] = t.split(";")[0]
-        else:
-            d["resp_ctype"] = ""
-
-    return raw_format_flow(tuple(sorted(d.items())))
+    if ((layout == 'default' and cols < 100) or layout == "list"):
+        return raw_format_list(tuple(sorted(d.items())))
+    else:
+        return raw_format_table(tuple(sorted(d.items())))
diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py
index a40cdeaa..b6602413 100644
--- a/mitmproxy/tools/console/consoleaddons.py
+++ b/mitmproxy/tools/console/consoleaddons.py
@@ -37,6 +37,12 @@ console_layouts = [
     "horizontal",
 ]
 
+console_flowlist_layout = [
+    "default",
+    "table",
+    "list"
+]
+
 
 class UnsupportedLog:
     """
@@ -114,6 +120,13 @@ class ConsoleAddon:
             "Console mouse interaction."
         )
 
+        loader.add_option(
+            "console_flowlist_layout",
+            str, "default",
+            "Set the flowlist layout",
+            choices=sorted(console_flowlist_layout)
+        )
+
     @command.command("console.layout.options")
     def layout_options(self) -> typing.Sequence[str]:
         """
@@ -428,7 +441,12 @@ class ConsoleAddon:
             message.content = c.rstrip(b"\n")
         elif part == "set-cookies":
             self.master.switch_view("edit_focus_setcookies")
-        elif part in ["url", "method", "status_code", "reason"]:
+        elif part == "url":
+            url = flow.request.url.encode()
+            edited_url = self.master.spawn_editor(url)
+            url = edited_url.rstrip(b"\n")
+            flow.request.url = url.decode()
+        elif part in ["method", "status_code", "reason"]:
             self.master.commands.execute(
                 "console.command flow.set @focus %s " % part
             )
diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py
index e947a582..9650c0d3 100644
--- a/mitmproxy/tools/console/flowlist.py
+++ b/mitmproxy/tools/console/flowlist.py
@@ -18,7 +18,8 @@ class FlowItem(urwid.WidgetWrap):
             self.flow,
             self.flow is self.master.view.focus.flow,
             hostheader=self.master.options.showhost,
-            max_url_len=cols,
+            cols=cols,
+            layout=self.master.options.console_flowlist_layout
         )
 
     def selectable(self):
@@ -84,6 +85,10 @@ class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget):
     ) -> None:
         self.master: "mitmproxy.tools.console.master.ConsoleMaster" = master
         super().__init__(FlowListWalker(master))
+        self.master.options.subscribe(
+            self.set_flowlist_layout,
+            ["console_flowlist_layout"]
+        )
 
     def keypress(self, size, key):
         if key == "m_start":
@@ -96,3 +101,6 @@ class FlowListBox(urwid.ListBox, layoutwidget.LayoutWidget):
 
     def view_changed(self):
         self.body.view_changed()
+
+    def set_flowlist_layout(self, opts, updated):
+        self.master.ui.clear()
diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py
index b4e3876f..807c9714 100644
--- a/mitmproxy/tools/console/flowview.py
+++ b/mitmproxy/tools/console/flowview.py
@@ -38,7 +38,8 @@ class FlowViewHeader(urwid.WidgetWrap):
                 False,
                 extended=True,
                 hostheader=self.master.options.showhost,
-                max_url_len=cols,
+                cols=cols,
+                layout=self.master.options.console_flowlist_layout
             )
         else:
             self._w = urwid.Pile([])
diff --git a/mitmproxy/tools/console/grideditor/base.py b/mitmproxy/tools/console/grideditor/base.py
index 3badf1a6..64b6e5d5 100644
--- a/mitmproxy/tools/console/grideditor/base.py
+++ b/mitmproxy/tools/console/grideditor/base.py
@@ -254,7 +254,7 @@ FIRST_WIDTH_MAX = 40
 
 
 class BaseGridEditor(urwid.WidgetWrap):
-    title = ""
+    title: str = ""
     keyctx = "grideditor"
 
     def __init__(
@@ -402,8 +402,8 @@ class BaseGridEditor(urwid.WidgetWrap):
 
 
 class GridEditor(BaseGridEditor):
-    title: str = None
-    columns: typing.Sequence[Column] = None
+    title = ""
+    columns: typing.Sequence[Column] = ()
     keyctx = "grideditor"
 
     def __init__(
diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py
index 61fcf6b4..b4d59384 100644
--- a/mitmproxy/tools/console/grideditor/editors.py
+++ b/mitmproxy/tools/console/grideditor/editors.py
@@ -107,7 +107,7 @@ class CookieAttributeEditor(base.FocusEditor):
         col_text.Column("Name"),
         col_text.Column("Value"),
     ]
-    grideditor: base.BaseGridEditor = None
+    grideditor: base.BaseGridEditor
 
     def data_in(self, data):
         return [(k, v or "") for k, v in data]
@@ -169,7 +169,7 @@ class SetCookieEditor(base.FocusEditor):
 
 
 class OptionsEditor(base.GridEditor, layoutwidget.LayoutWidget):
-    title: str = None
+    title = ""
     columns = [
         col_text.Column("")
     ]
@@ -189,7 +189,7 @@ class OptionsEditor(base.GridEditor, layoutwidget.LayoutWidget):
 
 
 class DataViewer(base.GridEditor, layoutwidget.LayoutWidget):
-    title: str = None
+    title = ""
 
     def __init__(
             self,
diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py
index dd15a2f5..6ab9ba5a 100644
--- a/mitmproxy/tools/console/master.py
+++ b/mitmproxy/tools/console/master.py
@@ -120,7 +120,7 @@ class ConsoleMaster(master.Master):
         with open(fd, "w" if text else "wb") as f:
             f.write(data)
         # if no EDITOR is set, assume 'vi'
-        c = os.environ.get("EDITOR") or "vi"
+        c = os.environ.get("MITMPROXY_EDITOR") or os.environ.get("EDITOR") or "vi"
         cmd = shlex.split(c)
         cmd.append(name)
         with self.uistopped():
@@ -159,7 +159,7 @@ class ConsoleMaster(master.Master):
                 shell = True
         if not cmd:
             # hm which one should get priority?
-            c = os.environ.get("PAGER") or os.environ.get("EDITOR")
+            c = os.environ.get("MITMPROXY_EDITOR") or os.environ.get("PAGER") or os.environ.get("EDITOR")
             if not c:
                 c = "less"
             cmd = shlex.split(c)
diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py
index 7930c4a3..6033ff25 100644
--- a/mitmproxy/tools/console/palettes.py
+++ b/mitmproxy/tools/console/palettes.py
@@ -22,7 +22,12 @@ class Palette:
         'option_selected_key',
 
         # List and Connections
-        'method', 'focus',
+        'method',
+        'method_get', 'method_post', 'method_delete', 'method_other', 'method_head', 'method_put', 'method_http2_push',
+        'scheme_http', 'scheme_https', 'scheme_other',
+        'url_punctuation', 'url_domain', 'url_filename', 'url_extension', 'url_query_key', 'url_query_value',
+        'content_none', 'content_text', 'content_script', 'content_media', 'content_data', 'content_raw', 'content_other',
+        'focus',
         'code_200', 'code_300', 'code_400', 'code_500', 'code_other',
         'error', "warn", "alert",
         'header', 'highlight', 'intercept', 'replay', 'mark',
@@ -36,7 +41,8 @@ class Palette:
         # Commander
         'commander_command', 'commander_invalid', 'commander_hint'
     ]
-    high: typing.Mapping[str, typing.Sequence[str]] = None
+    _fields.extend(['gradient_%02d' % i for i in range(100)])
+    high: typing.Optional[typing.Mapping[str, typing.Sequence[str]]] = None
 
     def palette(self, transparent):
         l = []
@@ -68,6 +74,27 @@ class Palette:
         return l
 
 
+def gen_gradient(palette, cols):
+    for i in range(100):
+        palette['gradient_%02d' % i] = (cols[i * len(cols) // 100], 'default')
+
+
+def gen_rgb_gradient(palette, cols):
+    parts = len(cols) - 1
+    for i in range(100):
+        p = i / 100
+        idx = int(p * parts)
+        t0 = cols[idx]
+        t1 = cols[idx + 1]
+        pp = p * parts % 1
+        t = (
+            round(t0[0] + (t1[0] - t0[0]) * pp),
+            round(t0[1] + (t1[1] - t0[1]) * pp),
+            round(t0[2] + (t1[2] - t0[2]) * pp),
+        )
+        palette['gradient_%02d' % i] = ("#%x%x%x" % t, 'default')
+
+
 class LowDark(Palette):
 
     """
@@ -95,6 +122,33 @@ class LowDark(Palette):
 
         # List and Connections
         method = ('dark cyan', 'default'),
+        method_get = ('light green', 'default'),
+        method_post = ('brown', 'default'),
+        method_delete = ('light red', 'default'),
+        method_head = ('dark cyan', 'default'),
+        method_put = ('dark red', 'default'),
+        method_other = ('dark magenta', 'default'),
+        method_http2_push = ('dark gray', 'default'),
+
+        scheme_http = ('dark cyan', 'default'),
+        scheme_https = ('dark green', 'default'),
+        scheme_other = ('dark magenta', 'default'),
+
+        url_punctuation = ('light gray', 'default'),
+        url_domain = ('white', 'default'),
+        url_filename = ('dark cyan', 'default'),
+        url_extension = ('light gray', 'default'),
+        url_query_key = ('white', 'default'),
+        url_query_value = ('light gray', 'default'),
+
+        content_none = ('dark gray', 'default'),
+        content_text = ('light gray', 'default'),
+        content_script = ('dark green', 'default'),
+        content_media = ('light blue', 'default'),
+        content_data = ('brown', 'default'),
+        content_raw = ('dark red', 'default'),
+        content_other = ('dark magenta', 'default'),
+
         focus = ('yellow', 'default'),
 
         code_200 = ('dark green', 'default'),
@@ -127,6 +181,7 @@ class LowDark(Palette):
         commander_invalid = ('light red', 'default'),
         commander_hint = ('dark gray', 'default'),
     )
+    gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue'])
 
 
 class Dark(LowDark):
@@ -167,6 +222,33 @@ class LowLight(Palette):
 
         # List and Connections
         method = ('dark cyan', 'default'),
+        method_get = ('dark green', 'default'),
+        method_post = ('brown', 'default'),
+        method_head = ('dark cyan', 'default'),
+        method_put = ('light red', 'default'),
+        method_delete = ('dark red', 'default'),
+        method_other = ('light magenta', 'default'),
+        method_http2_push = ('light gray', 'default'),
+
+        scheme_http = ('dark cyan', 'default'),
+        scheme_https = ('light green', 'default'),
+        scheme_other = ('light magenta', 'default'),
+
+        url_punctuation = ('dark gray', 'default'),
+        url_domain = ('dark gray', 'default'),
+        url_filename = ('black', 'default'),
+        url_extension = ('dark gray', 'default'),
+        url_query_key = ('light blue', 'default'),
+        url_query_value = ('dark blue', 'default'),
+
+        content_none = ('black', 'default'),
+        content_text = ('dark gray', 'default'),
+        content_script = ('light green', 'default'),
+        content_media = ('light blue', 'default'),
+        content_data = ('brown', 'default'),
+        content_raw = ('light red', 'default'),
+        content_other = ('light magenta', 'default'),
+
         focus = ('black', 'default'),
 
         code_200 = ('dark green', 'default'),
@@ -198,6 +280,7 @@ class LowLight(Palette):
         commander_invalid = ('light red', 'default'),
         commander_hint = ('light gray', 'default'),
     )
+    gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue'])
 
 
 class Light(LowLight):
@@ -256,7 +339,27 @@ class SolarizedLight(LowLight):
         option_active_selected = (sol_orange, sol_base2),
 
         # List and Connections
-        method = (sol_cyan, 'default'),
+
+        method = ('dark cyan', 'default'),
+        method_get = (sol_green, 'default'),
+        method_post = (sol_orange, 'default'),
+        method_head = (sol_cyan, 'default'),
+        method_put = (sol_red, 'default'),
+        method_delete = (sol_red, 'default'),
+        method_other = (sol_magenta, 'default'),
+        method_http2_push = ('light gray', 'default'),
+
+        scheme_http = (sol_cyan, 'default'),
+        scheme_https = ('light green', 'default'),
+        scheme_other = ('light magenta', 'default'),
+
+        url_punctuation = ('dark gray', 'default'),
+        url_domain = ('dark gray', 'default'),
+        url_filename = ('black', 'default'),
+        url_extension = ('dark gray', 'default'),
+        url_query_key = (sol_blue, 'default'),
+        url_query_value = ('dark blue', 'default'),
+
         focus = (sol_base01, 'default'),
 
         code_200 = (sol_green, 'default'),
@@ -311,9 +414,28 @@ class SolarizedDark(LowDark):
         option_active_selected = (sol_orange, sol_base00),
 
         # List and Connections
-        method = (sol_cyan, 'default'),
         focus = (sol_base1, 'default'),
 
+        method = (sol_cyan, 'default'),
+        method_get = (sol_green, 'default'),
+        method_post = (sol_orange, 'default'),
+        method_delete = (sol_red, 'default'),
+        method_head = (sol_cyan, 'default'),
+        method_put = (sol_red, 'default'),
+        method_other = (sol_magenta, 'default'),
+        method_http2_push = (sol_base01, 'default'),
+
+        url_punctuation = ('h242', 'default'),
+        url_domain = ('h252', 'default'),
+        url_filename = ('h132', 'default'),
+        url_extension = ('h96', 'default'),
+        url_query_key = ('h37', 'default'),
+        url_query_value = ('h30', 'default'),
+
+        content_none = (sol_base01, 'default'),
+        content_text = (sol_base1, 'default'),
+        content_media = (sol_blue, 'default'),
+
         code_200 = (sol_green, 'default'),
         code_300 = (sol_blue, 'default'),
         code_400 = (sol_orange, 'default',),
@@ -342,6 +464,7 @@ class SolarizedDark(LowDark):
         commander_invalid = (sol_orange, 'default'),
         commander_hint = (sol_base00, 'default'),
     )
+    gen_rgb_gradient(high, [(15, 0, 0), (15, 15, 0), (0, 15, 0), (0, 15, 15), (0, 0, 15)])
 
 
 DEFAULT = "dark"
diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py
index 2d32f487..56f0674f 100644
--- a/mitmproxy/tools/console/statusbar.py
+++ b/mitmproxy/tools/console/statusbar.py
@@ -215,6 +215,10 @@ class StatusBar(urwid.WidgetWrap):
             r.append("[")
             r.append(("heading_key", "I"))
             r.append("gnore:%d]" % len(self.master.options.ignore_hosts))
+        elif self.master.options.allow_hosts:
+            r.append("[")
+            r.append(("heading_key", "A"))
+            r.append("llow:%d]" % len(self.master.options.allow_hosts))
         if self.master.options.tcp_hosts:
             r.append("[")
             r.append(("heading_key", "T"))
diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py
index 6e6b6223..a0803755 100644
--- a/mitmproxy/tools/web/app.py
+++ b/mitmproxy/tools/web/app.py
@@ -1,24 +1,26 @@
+import asyncio
 import hashlib
 import json
 import logging
 import os.path
 import re
 from io import BytesIO
-import asyncio
+from typing import ClassVar, Optional
 
-import mitmproxy.flow
 import tornado.escape
 import tornado.web
 import tornado.websocket
+
+import mitmproxy.flow
+import mitmproxy.tools.web.master  # noqa
 from mitmproxy import contentviews
 from mitmproxy import exceptions
 from mitmproxy import flowfilter
 from mitmproxy import http
 from mitmproxy import io
 from mitmproxy import log
-from mitmproxy import version
 from mitmproxy import optmanager
-import mitmproxy.tools.web.master # noqa
+from mitmproxy import version
 
 
 def flow_to_json(flow: mitmproxy.flow.Flow) -> dict:
@@ -49,6 +51,8 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict:
         f["error"] = flow.error.get_state()
 
     if isinstance(flow, http.HTTPFlow):
+        content_length: Optional[int]
+        content_hash: Optional[str]
         if flow.request:
             if flow.request.raw_content:
                 content_length = len(flow.request.raw_content)
@@ -108,6 +112,8 @@ class APIError(tornado.web.HTTPError):
 
 
 class RequestHandler(tornado.web.RequestHandler):
+    application: "Application"
+
     def write(self, chunk):
         # Writing arrays on the top level is ok nowadays.
         # http://flask.pocoo.org/docs/0.11/security/#json-security
@@ -190,7 +196,7 @@ class FilterHelp(RequestHandler):
 
 class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler):
     # raise an error if inherited class doesn't specify its own instance.
-    connections: set = None
+    connections: ClassVar[set]
 
     def open(self):
         self.connections.add(self)
@@ -210,7 +216,7 @@ class WebSocketEventBroadcaster(tornado.websocket.WebSocketHandler):
 
 
 class ClientConnection(WebSocketEventBroadcaster):
-    connections: set = set()
+    connections: ClassVar[set] = set()
 
 
 class Flows(RequestHandler):
@@ -432,7 +438,7 @@ class Settings(RequestHandler):
     def put(self):
         update = self.json
         option_whitelist = {
-            "intercept", "showhost", "upstream_cert",
+            "intercept", "showhost", "upstream_cert", "ssl_insecure",
             "rawtcp", "http2", "websocket", "anticache", "anticomp",
             "stickycookie", "stickyauth", "stream_large_bodies"
         }
@@ -473,7 +479,9 @@ class DnsRebind(RequestHandler):
 
 
 class Application(tornado.web.Application):
-    def __init__(self, master, debug):
+    master: "mitmproxy.tools.web.master.WebMaster"
+
+    def __init__(self, master: "mitmproxy.tools.web.master.WebMaster", debug: bool) -> None:
         self.master = master
         super().__init__(
             default_host="dns-rebind-protection",
diff --git a/mitmproxy/types.py b/mitmproxy/types.py
index f2a26b40..0634e4d7 100644
--- a/mitmproxy/types.py
+++ b/mitmproxy/types.py
@@ -423,7 +423,7 @@ class TypeManager:
         for t in types:
             self.typemap[t.typ] = t()
 
-    def get(self, t: type, default=None) -> _BaseType:
+    def get(self, t: typing.Optional[typing.Type], default=None) -> _BaseType:
         if type(t) in self.typemap:
             return self.typemap[type(t)]
         return self.typemap.get(t, default)
diff --git a/mitmproxy/utils/human.py b/mitmproxy/utils/human.py
index 5c02b072..3158a294 100644
--- a/mitmproxy/utils/human.py
+++ b/mitmproxy/utils/human.py
@@ -48,12 +48,14 @@ def parse_size(s: typing.Optional[str]) -> typing.Optional[int]:
     raise ValueError("Invalid size specification.")
 
 
-def pretty_duration(secs):
+def pretty_duration(secs: typing.Optional[float]) -> str:
     formatters = [
         (100, "{:.0f}s"),
         (10, "{:2.1f}s"),
         (1, "{:1.2f}s"),
     ]
+    if secs is None:
+        return ""
 
     for limit, formatter in formatters:
         if secs >= limit:
diff --git a/mitmproxy/utils/sliding_window.py b/mitmproxy/utils/sliding_window.py
index 0a65f5e4..cb31756d 100644
--- a/mitmproxy/utils/sliding_window.py
+++ b/mitmproxy/utils/sliding_window.py
@@ -1,5 +1,5 @@
 import itertools
-from typing import TypeVar, Iterable, Iterator, Tuple, Optional
+from typing import TypeVar, Iterable, Iterator, Tuple, Optional, List
 
 T = TypeVar('T')
 
@@ -18,7 +18,7 @@ def window(iterator: Iterable[T], behind: int = 0, ahead: int = 0) -> Iterator[T
         2 3 None
     """
     # TODO: move into utils
-    iters = list(itertools.tee(iterator, behind + 1 + ahead))
+    iters: List[Iterator[Optional[T]]] = list(itertools.tee(iterator, behind + 1 + ahead))
     for i in range(behind):
         iters[i] = itertools.chain((behind - i) * [None], iters[i])
     for i in range(ahead):
diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py
index 388c765f..6e399d8f 100644
--- a/mitmproxy/utils/strutils.py
+++ b/mitmproxy/utils/strutils.py
@@ -1,10 +1,10 @@
+import codecs
 import io
 import re
-import codecs
-from typing import AnyStr, Optional, cast, Iterable
+from typing import Iterable, Optional, Union, cast
 
 
-def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes]:
+def always_bytes(str_or_bytes: Union[str, bytes, None], *encode_args) -> Optional[bytes]:
     if isinstance(str_or_bytes, bytes) or str_or_bytes is None:
         return cast(Optional[bytes], str_or_bytes)
     elif isinstance(str_or_bytes, str):
@@ -13,13 +13,15 @@ def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes
         raise TypeError("Expected str or bytes, but got {}.".format(type(str_or_bytes).__name__))
 
 
-def always_str(str_or_bytes: Optional[AnyStr], *decode_args) -> Optional[str]:
+def always_str(str_or_bytes: Union[str, bytes, None], *decode_args) -> Optional[str]:
     """
     Returns,
         str_or_bytes unmodified, if
     """
-    if isinstance(str_or_bytes, str) or str_or_bytes is None:
-        return cast(Optional[str], str_or_bytes)
+    if str_or_bytes is None:
+        return None
+    if isinstance(str_or_bytes, str):
+        return cast(str, str_or_bytes)
     elif isinstance(str_or_bytes, bytes):
         return str_or_bytes.decode(*decode_args)
     else:
@@ -39,7 +41,6 @@ _control_char_trans_newline = _control_char_trans.copy()
 for x in ("\r", "\n", "\t"):
     del _control_char_trans_newline[ord(x)]
 
-
 _control_char_trans = str.maketrans(_control_char_trans)
 _control_char_trans_newline = str.maketrans(_control_char_trans_newline)
 
diff --git a/mitmproxy/version.py b/mitmproxy/version.py
index b40fae8b..363a4bf6 100644
--- a/mitmproxy/version.py
+++ b/mitmproxy/version.py
@@ -25,9 +25,9 @@ def get_dev_version() -> str:
             stderr=subprocess.STDOUT,
             cwd=here,
         )
-        last_tag, tag_dist, commit = git_describe.decode().strip().rsplit("-", 2)
+        last_tag, tag_dist_str, commit = git_describe.decode().strip().rsplit("-", 2)
         commit = commit.lstrip("g")[:7]
-        tag_dist = int(tag_dist)
+        tag_dist = int(tag_dist_str)
     except Exception:
         pass
     else: