Make mypy succeed with imports on master.py

We get little benefit from our mypy QA checks at the moment, because we skip
imports. This patch is what's needed to make mypy succeed with imports on a
single file: master.py

It also updates mypy to the current version, and enables a QA check.

Mypy bugs I encountered:

dict.update with kwargs not supported:

https://github.com/python/mypy/issues/1031

property setters and getters must be adjacent:

https://github.com/python/mypy/issues/1465

18 files changed, 116 insertions, 105 deletions

diff --git a/mitmproxy/ctx.py b/mitmproxy/ctx.py
index adae8cf6..7b5231e6 100644
--- a/mitmproxy/ctx.py
+++ b/mitmproxy/ctx.py
@@ -1,2 +1,4 @@
+import mitmproxy.master  # noqa
+import mitmproxy.log  # noqa
 master = None  # type: "mitmproxy.master.Master"
 log = None  # type: "mitmproxy.log.Log"
diff --git a/mitmproxy/http.py b/mitmproxy/http.py
index c6b17533..c09778fe 100644
--- a/mitmproxy/http.py
+++ b/mitmproxy/http.py
@@ -52,9 +52,7 @@ class HTTPRequest(http.Request):
 
     def get_state(self):
         state = super().get_state()
-        state.update(
-            is_replay=self.is_replay
-        )
+        state["is_replay"] = self.is_replay
         return state
 
     def set_state(self, state):
@@ -167,11 +165,12 @@ class HTTPFlow(flow.Flow):
         """ What mode was the proxy layer in when receiving this request? """
 
     _stateobject_attributes = flow.Flow._stateobject_attributes.copy()
-    _stateobject_attributes.update(
+    # mypy doesn't support update with kwargs
+    _stateobject_attributes.update(dict(
         request=HTTPRequest,
         response=HTTPResponse,
         mode=str
-    )
+    ))
 
     def __repr__(self):
         s = "<HTTPFlow"
@@ -223,8 +222,7 @@ def make_error_response(
         status_code=status_code,
         reason=reason,
         message=html.escape(message),
-    )
-    body = body.encode("utf8", "replace")
+    ).encode("utf8", "replace")
 
     if not headers:
         headers = http.Headers(
diff --git a/mitmproxy/net/http/encoding.py b/mitmproxy/net/http/encoding.py
index 5da07099..8cb96e5c 100644
--- a/mitmproxy/net/http/encoding.py
+++ b/mitmproxy/net/http/encoding.py
@@ -10,18 +10,22 @@ import gzip
 import zlib
 import brotli
 
-from typing import Union
+from typing import Union, Optional, AnyStr  # noqa
 
 
 # We have a shared single-element cache for encoding and decoding.
 # This is quite useful in practice, e.g.
 # flow.request.content = flow.request.content.replace(b"foo", b"bar")
 # does not require an .encode() call if content does not contain b"foo"
-CachedDecode = collections.namedtuple("CachedDecode", "encoded encoding errors decoded")
+CachedDecode = collections.namedtuple(
+    "CachedDecode", "encoded encoding errors decoded"
+)
 _cache = CachedDecode(None, None, None, None)
 
 
-def decode(encoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]:
+def decode(
+    encoded: Optional[bytes], encoding: str, errors: str='strict'
+) -> Optional[AnyStr]:
     """
     Decode the given input object
 
@@ -62,7 +66,7 @@ def decode(encoded: Union[str, bytes], encoding: str, errors: str='strict') -> U
         ))
 
 
-def encode(decoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]:
+def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optional[AnyStr]:
     """
     Encode the given input object
 
diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py
index 506674d6..1040c6ce 100644
--- a/mitmproxy/net/http/message.py
+++ b/mitmproxy/net/http/message.py
@@ -1,5 +1,5 @@
 import re
-from typing import Optional
+from typing import Optional, Union  # noqa
 
 from mitmproxy.utils import strutils
 from mitmproxy.net.http import encoding
@@ -8,6 +8,8 @@ from mitmproxy.net.http import headers
 
 
 class MessageData(serializable.Serializable):
+    content = None  # type: bytes
+
     def __eq__(self, other):
         if isinstance(other, MessageData):
             return self.__dict__ == other.__dict__
@@ -31,6 +33,8 @@ class MessageData(serializable.Serializable):
 
 
 class Message(serializable.Serializable):
+    data = None  # type: MessageData
+
     def __eq__(self, other):
         if isinstance(other, Message):
             return self.data == other.data
@@ -159,6 +163,7 @@ class Message(serializable.Serializable):
         ct = headers.parse_content_type(self.headers.get("content-type", ""))
         if ct:
             return ct[2].get("charset")
+        return None
 
     def _guess_encoding(self) -> str:
         enc = self._get_content_type_charset()
diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py
index 90a1f924..6f366a4f 100644
--- a/mitmproxy/net/http/request.py
+++ b/mitmproxy/net/http/request.py
@@ -82,8 +82,8 @@ class Request(message.Message):
             cls,
             method: str,
             url: str,
-            content: AnyStr = b"",
-            headers: Union[Dict[AnyStr, AnyStr], Iterable[Tuple[bytes, bytes]]] = ()
+            content: Union[bytes, str] = "",
+            headers: Union[Dict[str, AnyStr], Iterable[Tuple[bytes, bytes]]] = ()
     ):
         """
         Simplified API for creating request objects.
@@ -327,6 +327,15 @@ class Request(message.Message):
             return "%s:%d" % (self.pretty_host, self.port)
         return mitmproxy.net.http.url.unparse(self.scheme, self.pretty_host, self.port, self.path)
 
+    def _get_query(self):
+        query = urllib.parse.urlparse(self.url).query
+        return tuple(mitmproxy.net.http.url.decode(query))
+
+    def _set_query(self, query_data):
+        query = mitmproxy.net.http.url.encode(query_data)
+        _, _, path, params, _, fragment = urllib.parse.urlparse(self.url)
+        self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment])
+
     @property
     def query(self) -> multidict.MultiDictView:
         """
@@ -337,19 +346,17 @@ class Request(message.Message):
             self._set_query
         )
 
-    def _get_query(self):
-        query = urllib.parse.urlparse(self.url).query
-        return tuple(mitmproxy.net.http.url.decode(query))
-
-    def _set_query(self, query_data):
-        query = mitmproxy.net.http.url.encode(query_data)
-        _, _, path, params, _, fragment = urllib.parse.urlparse(self.url)
-        self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment])
-
     @query.setter
     def query(self, value):
         self._set_query(value)
 
+    def _get_cookies(self):
+        h = self.headers.get_all("Cookie")
+        return tuple(cookies.parse_cookie_headers(h))
+
+    def _set_cookies(self, value):
+        self.headers["cookie"] = cookies.format_cookie_header(value)
+
     @property
     def cookies(self) -> multidict.MultiDictView:
         """
@@ -362,13 +369,6 @@ class Request(message.Message):
             self._set_cookies
         )
 
-    def _get_cookies(self):
-        h = self.headers.get_all("Cookie")
-        return tuple(cookies.parse_cookie_headers(h))
-
-    def _set_cookies(self, value):
-        self.headers["cookie"] = cookies.format_cookie_header(value)
-
     @cookies.setter
     def cookies(self, value):
         self._set_cookies(value)
@@ -426,20 +426,6 @@ class Request(message.Message):
                 )
             )
 
-    @property
-    def urlencoded_form(self):
-        """
-        The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
-        An empty multidict.MultiDictView if the content-type indicates non-form data
-        or the content could not be parsed.
-
-        Starting with mitmproxy 1.0, key and value are strings.
-        """
-        return multidict.MultiDictView(
-            self._get_urlencoded_form,
-            self._set_urlencoded_form
-        )
-
     def _get_urlencoded_form(self):
         is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower()
         if is_valid_content_type:
@@ -457,24 +443,24 @@ class Request(message.Message):
         self.headers["content-type"] = "application/x-www-form-urlencoded"
         self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode()
 
-    @urlencoded_form.setter
-    def urlencoded_form(self, value):
-        self._set_urlencoded_form(value)
-
     @property
-    def multipart_form(self):
+    def urlencoded_form(self):
         """
-        The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
+        The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
         An empty multidict.MultiDictView if the content-type indicates non-form data
         or the content could not be parsed.
 
-        Key and value are bytes.
+        Starting with mitmproxy 1.0, key and value are strings.
         """
         return multidict.MultiDictView(
-            self._get_multipart_form,
-            self._set_multipart_form
+            self._get_urlencoded_form,
+            self._set_urlencoded_form
         )
 
+    @urlencoded_form.setter
+    def urlencoded_form(self, value):
+        self._set_urlencoded_form(value)
+
     def _get_multipart_form(self):
         is_valid_content_type = "multipart/form-data" in self.headers.get("content-type", "").lower()
         if is_valid_content_type:
@@ -487,6 +473,20 @@ class Request(message.Message):
     def _set_multipart_form(self, value):
         raise NotImplementedError()
 
+    @property
+    def multipart_form(self):
+        """
+        The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object.
+        An empty multidict.MultiDictView if the content-type indicates non-form data
+        or the content could not be parsed.
+
+        Key and value are bytes.
+        """
+        return multidict.MultiDictView(
+            self._get_multipart_form,
+            self._set_multipart_form
+        )
+
     @multipart_form.setter
     def multipart_form(self, value):
         self._set_multipart_form(value)
diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py
index 53c9c1ca..8edd43b8 100644
--- a/mitmproxy/net/http/response.py
+++ b/mitmproxy/net/http/response.py
@@ -69,8 +69,8 @@ class Response(message.Message):
     def make(
             cls,
             status_code: int=200,
-            content: AnyStr=b"",
-            headers: Union[Dict[AnyStr, AnyStr], Iterable[Tuple[bytes, bytes]]]=()
+            content: Union[bytes, str]=b"",
+            headers: Union[Dict[str, AnyStr], Iterable[Tuple[bytes, bytes]]]=()
     ):
         """
         Simplified API for creating response objects.
@@ -129,6 +129,17 @@ class Response(message.Message):
     def reason(self, reason):
         self.data.reason = strutils.always_bytes(reason, "ISO-8859-1", "surrogateescape")
 
+    def _get_cookies(self):
+        h = self.headers.get_all("set-cookie")
+        return tuple(cookies.parse_set_cookie_headers(h))
+
+    def _set_cookies(self, value):
+        cookie_headers = []
+        for k, v in value:
+            header = cookies.format_set_cookie_header([(k, v[0], v[1])])
+            cookie_headers.append(header)
+        self.headers.set_all("set-cookie", cookie_headers)
+
     @property
     def cookies(self) -> multidict.MultiDictView:
         """
@@ -146,17 +157,6 @@ class Response(message.Message):
             self._set_cookies
         )
 
-    def _get_cookies(self):
-        h = self.headers.get_all("set-cookie")
-        return tuple(cookies.parse_set_cookie_headers(h))
-
-    def _set_cookies(self, value):
-        cookie_headers = []
-        for k, v in value:
-            header = cookies.format_set_cookie_header([(k, v[0], v[1])])
-            cookie_headers.append(header)
-        self.headers.set_all("set-cookie", cookie_headers)
-
     @cookies.setter
     def cookies(self, value):
         self._set_cookies(value)
diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py
index 86ce9764..f2c8c473 100644
--- a/mitmproxy/net/http/url.py
+++ b/mitmproxy/net/http/url.py
@@ -1,4 +1,4 @@
-import urllib
+import urllib.parse
 from typing import Sequence
 from typing import Tuple
 
diff --git a/mitmproxy/options.py b/mitmproxy/options.py
index 5b84ac93..70392803 100644
--- a/mitmproxy/options.py
+++ b/mitmproxy/options.py
@@ -332,7 +332,7 @@ class Options(optmanager.OptManager):
             Set supported SSL/TLS versions for client connections. SSLv2, SSLv3
             and 'all' are INSECURE. Defaults to secure, which is TLS1.0+.
             """,
-            choices=tcp.sslversion_choices.keys(),
+            choices=list(tcp.sslversion_choices.keys()),
         )
         self.add_option(
             "ssl_version_server", str, "secure",
@@ -340,7 +340,7 @@ class Options(optmanager.OptManager):
             Set supported SSL/TLS versions for server connections. SSLv2, SSLv3
             and 'all' are INSECURE. Defaults to secure, which is TLS1.0+.
             """,
-            choices=tcp.sslversion_choices.keys(),
+            choices=list(tcp.sslversion_choices.keys()),
         )
         self.add_option(
             "ssl_insecure", bool, False,
diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py
index 8417ebad..b809d89a 100644
--- a/mitmproxy/proxy/config.py
+++ b/mitmproxy/proxy/config.py
@@ -37,11 +37,11 @@ class ProxyConfig:
     def __init__(self, options: moptions.Options) -> None:
         self.options = options
 
-        self.check_ignore = None
-        self.check_tcp = None
-        self.certstore = None
-        self.client_certs = None
-        self.openssl_verification_mode_server = None
+        self.check_ignore = None  # type: HostMatcher
+        self.check_tcp = None  # type: HostMatcher
+        self.certstore = None  # type: certs.CertStore
+        self.client_certs = None  # type: str
+        self.openssl_verification_mode_server = None  # type: int
         self.configure(options, set(options.keys()))
         options.changed.connect(self.configure)
 
diff --git a/mitmproxy/proxy/protocol/base.py b/mitmproxy/proxy/protocol/base.py
index b10bb8f5..7c0f78ae 100644
--- a/mitmproxy/proxy/protocol/base.py
+++ b/mitmproxy/proxy/protocol/base.py
@@ -1,5 +1,7 @@
 from mitmproxy import exceptions
 from mitmproxy import connections
+from mitmproxy import controller  # noqa
+from mitmproxy.proxy import config  # noqa
 
 
 class _LayerCodeCompletion:
@@ -12,14 +14,10 @@ class _LayerCodeCompletion:
         super().__init__(**mixin_args)
         if True:
             return
-        self.config = None
-        """@type: mitmproxy.proxy.ProxyConfig"""
-        self.client_conn = None
-        """@type: mitmproxy.connections.ClientConnection"""
-        self.server_conn = None
-        """@type: mitmproxy.connections.ServerConnection"""
-        self.channel = None
-        """@type: mitmproxy.controller.Channel"""
+        self.config = None  # type: config.ProxyConfig
+        self.client_conn = None  # type: connections.ClientConnection
+        self.server_conn = None  # type: connections.ServerConnection
+        self.channel = None  # type: controller.Channel
         self.ctx = None
         """@type: mitmproxy.proxy.protocol.Layer"""
 
diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py
index 16692234..9f783bc3 100644
--- a/mitmproxy/proxy/server.py
+++ b/mitmproxy/proxy/server.py
@@ -3,10 +3,11 @@ import traceback
 
 from mitmproxy import exceptions
 from mitmproxy import connections
+from mitmproxy import controller  # noqa
 from mitmproxy import http
 from mitmproxy import log
 from mitmproxy import platform
-from mitmproxy.proxy import ProxyConfig
+from mitmproxy.proxy import config
 from mitmproxy.proxy import modes
 from mitmproxy.proxy import root_context
 from mitmproxy.net import tcp
@@ -34,7 +35,7 @@ class ProxyServer(tcp.TCPServer):
     allow_reuse_address = True
     bound = True
 
-    def __init__(self, config: ProxyConfig):
+    def __init__(self, config: config.ProxyConfig) -> None:
         """
             Raises ServerException if there's a startup problem.
         """
@@ -49,7 +50,7 @@ class ProxyServer(tcp.TCPServer):
             raise exceptions.ServerException(
                 'Error starting proxy server: ' + repr(e)
             ) from e
-        self.channel = None
+        self.channel = None  # type: controller.Channel
 
     def set_channel(self, channel):
         self.channel = channel
@@ -67,8 +68,7 @@ class ProxyServer(tcp.TCPServer):
 class ConnectionHandler:
 
     def __init__(self, client_conn, client_address, config, channel):
-        self.config = config
-        """@type: mitmproxy.proxy.config.ProxyConfig"""
+        self.config = config  # type: config.ProxyConfig
         self.client_conn = connections.ClientConnection(
             client_conn,
             client_address,
diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py
index 14159001..a0deaec9 100644
--- a/mitmproxy/stateobject.py
+++ b/mitmproxy/stateobject.py
@@ -1,5 +1,6 @@
 from typing import Any
 from typing import List
+from typing import MutableMapping  # noqa
 
 from mitmproxy.types import serializable
 
@@ -19,7 +20,7 @@ class StateObject(serializable.Serializable):
     or StateObject instances themselves.
     """
 
-    _stateobject_attributes = None
+    _stateobject_attributes = None  # type: MutableMapping[str, Any]
     """
     An attribute-name -> class-or-type dict containing all attributes that
     should be serialized. If the attribute is a class, it must implement the
diff --git a/mitmproxy/tcp.py b/mitmproxy/tcp.py
index 067fbfe3..fe9f217b 100644
--- a/mitmproxy/tcp.py
+++ b/mitmproxy/tcp.py
@@ -41,9 +41,7 @@ class TCPFlow(flow.Flow):
         self.messages = []  # type: List[TCPMessage]
 
     _stateobject_attributes = flow.Flow._stateobject_attributes.copy()
-    _stateobject_attributes.update(
-        messages=List[TCPMessage]
-    )
+    _stateobject_attributes["messages"] = List[TCPMessage]
 
     def __repr__(self):
         return "<TCPFlow ({} messages)>".format(len(self.messages))
diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py
index 29465615..1b90c2e5 100644
--- a/mitmproxy/utils/strutils.py
+++ b/mitmproxy/utils/strutils.py
@@ -1,11 +1,11 @@
 import re
 import codecs
-from typing import AnyStr, Optional
+from typing import AnyStr, Optional, cast
 
 
 def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes]:
     if isinstance(str_or_bytes, bytes) or str_or_bytes is None:
-        return str_or_bytes
+        return cast(Optional[bytes], str_or_bytes)
     elif isinstance(str_or_bytes, str):
         return str_or_bytes.encode(*encode_args)
     else:
@@ -18,7 +18,7 @@ def always_str(str_or_bytes: Optional[AnyStr], *decode_args) -> Optional[str]:
         str_or_bytes unmodified, if
     """
     if isinstance(str_or_bytes, str) or str_or_bytes is None:
-        return str_or_bytes
+        return cast(Optional[str], str_or_bytes)
     elif isinstance(str_or_bytes, bytes):
         return str_or_bytes.decode(*decode_args)
     else:
diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py
index bdd83ee6..e8e2121e 100644
--- a/mitmproxy/utils/typecheck.py
+++ b/mitmproxy/utils/typecheck.py
@@ -25,10 +25,10 @@ def check_type(name: str, value: typing.Any, typeinfo: type) -> None:
 
     if typename.startswith("typing.Union"):
         try:
-            types = typeinfo.__args__
+            types = typeinfo.__args__  # type: ignore
         except AttributeError:
             # Python 3.5.x
-            types = typeinfo.__union_params__
+            types = typeinfo.__union_params__  # type: ignore
 
         for T in types:
             try:
@@ -40,10 +40,10 @@ def check_type(name: str, value: typing.Any, typeinfo: type) -> None:
         raise e
     elif typename.startswith("typing.Tuple"):
         try:
-            types = typeinfo.__args__
+            types = typeinfo.__args__  # type: ignore
         except AttributeError:
             # Python 3.5.x
-            types = typeinfo.__tuple_params__
+            types = typeinfo.__tuple_params__  # type: ignore
 
         if not isinstance(value, (tuple, list)):
             raise e
@@ -54,10 +54,10 @@ def check_type(name: str, value: typing.Any, typeinfo: type) -> None:
         return
     elif typename.startswith("typing.Sequence"):
         try:
-            T = typeinfo.__args__[0]
+            T = typeinfo.__args__[0]  # type: ignore
         except AttributeError:
             # Python 3.5.0
-            T = typeinfo.__parameters__[0]
+            T = typeinfo.__parameters__[0]  # type: ignore
 
         if not isinstance(value, (tuple, list)):
             raise e
diff --git a/mitmproxy/websocket.py b/mitmproxy/websocket.py
index 5d76aafc..2efa7ad1 100644
--- a/mitmproxy/websocket.py
+++ b/mitmproxy/websocket.py
@@ -8,11 +8,13 @@ from mitmproxy.utils import strutils
 
 
 class WebSocketMessage(serializable.Serializable):
-    def __init__(self, type: int, from_client: bool, content: bytes, timestamp: Optional[int]=None):
+    def __init__(
+        self, type: int, from_client: bool, content: bytes, timestamp: Optional[int]=None
+    ) -> None:
         self.type = type
         self.from_client = from_client
         self.content = content
-        self.timestamp = timestamp or time.time()  # type: int
+        self.timestamp = timestamp or int(time.time())  # type: int
 
     @classmethod
     def from_state(cls, state):
@@ -62,7 +64,8 @@ class WebSocketFlow(flow.Flow):
         self.handshake_flow = handshake_flow
 
     _stateobject_attributes = flow.Flow._stateobject_attributes.copy()
-    _stateobject_attributes.update(
+    # mypy doesn't support update with kwargs
+    _stateobject_attributes.update(dict(
         messages=List[WebSocketMessage],
         close_sender=str,
         close_code=str,
@@ -77,7 +80,7 @@ class WebSocketFlow(flow.Flow):
         # Do not include handshake_flow, to prevent recursive serialization!
         # Since mitmproxy-console currently only displays HTTPFlows,
         # dumping the handshake_flow will include the WebSocketFlow too.
-    )
+    ))
 
     @classmethod
     def from_state(cls, state):
diff --git a/setup.py b/setup.py
index 9ea69f67..ec28eded 100644
--- a/setup.py
+++ b/setup.py
@@ -96,7 +96,7 @@ setup(
         'dev': [
             "Flask>=0.10.1, <0.13",
             "flake8>=3.2.1, <3.4",
-            "mypy>=0.471, <0.502",
+            "mypy>=0.501, <0.502",
             "rstcheck>=2.2, <4.0",
             "tox>=2.3, <3",
             "pytest>=3, <3.1",
diff --git a/tox.ini b/tox.ini
index a9904e87..a1ed53f7 100644
--- a/tox.ini
+++ b/tox.ini
@@ -36,6 +36,8 @@ commands =
     mitmproxy/tools/dump.py \
     mitmproxy/tools/web/ \
     mitmproxy/contentviews/
+  mypy --ignore-missing-imports \
+    mitmproxy/master.py
 
 [testenv:individual_coverage]
 deps =