diff options
| -rw-r--r-- | mitmproxy/ctx.py | 2 | ||||
| -rw-r--r-- | mitmproxy/http.py | 12 | ||||
| -rw-r--r-- | mitmproxy/net/http/encoding.py | 12 | ||||
| -rw-r--r-- | mitmproxy/net/http/message.py | 7 | ||||
| -rw-r--r-- | mitmproxy/net/http/request.py | 82 | ||||
| -rw-r--r-- | mitmproxy/net/http/response.py | 26 | ||||
| -rw-r--r-- | mitmproxy/net/http/url.py | 2 | ||||
| -rw-r--r-- | mitmproxy/options.py | 4 | ||||
| -rw-r--r-- | mitmproxy/proxy/config.py | 10 | ||||
| -rw-r--r-- | mitmproxy/proxy/protocol/base.py | 14 | ||||
| -rw-r--r-- | mitmproxy/proxy/server.py | 10 | ||||
| -rw-r--r-- | mitmproxy/stateobject.py | 3 | ||||
| -rw-r--r-- | mitmproxy/tcp.py | 4 | ||||
| -rw-r--r-- | mitmproxy/utils/strutils.py | 6 | ||||
| -rw-r--r-- | mitmproxy/utils/typecheck.py | 12 | ||||
| -rw-r--r-- | mitmproxy/websocket.py | 11 | ||||
| -rw-r--r-- | setup.py | 2 | ||||
| -rw-r--r-- | test/mitmproxy/test_taddons.py | 1 | ||||
| -rw-r--r-- | tox.ini | 2 |
19 files changed, 116 insertions, 106 deletions
diff --git a/mitmproxy/ctx.py b/mitmproxy/ctx.py index adae8cf6..7b5231e6 100644 --- a/mitmproxy/ctx.py +++ b/mitmproxy/ctx.py @@ -1,2 +1,4 @@ +import mitmproxy.master # noqa +import mitmproxy.log # noqa master = None # type: "mitmproxy.master.Master" log = None # type: "mitmproxy.log.Log" diff --git a/mitmproxy/http.py b/mitmproxy/http.py index c6b17533..c09778fe 100644 --- a/mitmproxy/http.py +++ b/mitmproxy/http.py @@ -52,9 +52,7 @@ class HTTPRequest(http.Request): def get_state(self): state = super().get_state() - state.update( - is_replay=self.is_replay - ) + state["is_replay"] = self.is_replay return state def set_state(self, state): @@ -167,11 +165,12 @@ class HTTPFlow(flow.Flow): """ What mode was the proxy layer in when receiving this request? """ _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - _stateobject_attributes.update( + # mypy doesn't support update with kwargs + _stateobject_attributes.update(dict( request=HTTPRequest, response=HTTPResponse, mode=str - ) + )) def __repr__(self): s = "<HTTPFlow" @@ -223,8 +222,7 @@ def make_error_response( status_code=status_code, reason=reason, message=html.escape(message), - ) - body = body.encode("utf8", "replace") + ).encode("utf8", "replace") if not headers: headers = http.Headers( diff --git a/mitmproxy/net/http/encoding.py b/mitmproxy/net/http/encoding.py index 5da07099..8cb96e5c 100644 --- a/mitmproxy/net/http/encoding.py +++ b/mitmproxy/net/http/encoding.py @@ -10,18 +10,22 @@ import gzip import zlib import brotli -from typing import Union +from typing import Union, Optional, AnyStr # noqa # We have a shared single-element cache for encoding and decoding. # This is quite useful in practice, e.g. # flow.request.content = flow.request.content.replace(b"foo", b"bar") # does not require an .encode() call if content does not contain b"foo" -CachedDecode = collections.namedtuple("CachedDecode", "encoded encoding errors decoded") +CachedDecode = collections.namedtuple( + "CachedDecode", "encoded encoding errors decoded" +) _cache = CachedDecode(None, None, None, None) -def decode(encoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]: +def decode( + encoded: Optional[bytes], encoding: str, errors: str='strict' +) -> Optional[AnyStr]: """ Decode the given input object @@ -62,7 +66,7 @@ def decode(encoded: Union[str, bytes], encoding: str, errors: str='strict') -> U )) -def encode(decoded: Union[str, bytes], encoding: str, errors: str='strict') -> Union[str, bytes]: +def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optional[AnyStr]: """ Encode the given input object diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index 506674d6..1040c6ce 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -1,5 +1,5 @@ import re -from typing import Optional +from typing import Optional, Union # noqa from mitmproxy.utils import strutils from mitmproxy.net.http import encoding @@ -8,6 +8,8 @@ from mitmproxy.net.http import headers class MessageData(serializable.Serializable): + content = None # type: bytes + def __eq__(self, other): if isinstance(other, MessageData): return self.__dict__ == other.__dict__ @@ -31,6 +33,8 @@ class MessageData(serializable.Serializable): class Message(serializable.Serializable): + data = None # type: MessageData + def __eq__(self, other): if isinstance(other, Message): return self.data == other.data @@ -159,6 +163,7 @@ class Message(serializable.Serializable): ct = headers.parse_content_type(self.headers.get("content-type", "")) if ct: return ct[2].get("charset") + return None def _guess_encoding(self) -> str: enc = self._get_content_type_charset() diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 90a1f924..6f366a4f 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -82,8 +82,8 @@ class Request(message.Message): cls, method: str, url: str, - content: AnyStr = b"", - headers: Union[Dict[AnyStr, AnyStr], Iterable[Tuple[bytes, bytes]]] = () + content: Union[bytes, str] = "", + headers: Union[Dict[str, AnyStr], Iterable[Tuple[bytes, bytes]]] = () ): """ Simplified API for creating request objects. @@ -327,6 +327,15 @@ class Request(message.Message): return "%s:%d" % (self.pretty_host, self.port) return mitmproxy.net.http.url.unparse(self.scheme, self.pretty_host, self.port, self.path) + def _get_query(self): + query = urllib.parse.urlparse(self.url).query + return tuple(mitmproxy.net.http.url.decode(query)) + + def _set_query(self, query_data): + query = mitmproxy.net.http.url.encode(query_data) + _, _, path, params, _, fragment = urllib.parse.urlparse(self.url) + self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment]) + @property def query(self) -> multidict.MultiDictView: """ @@ -337,19 +346,17 @@ class Request(message.Message): self._set_query ) - def _get_query(self): - query = urllib.parse.urlparse(self.url).query - return tuple(mitmproxy.net.http.url.decode(query)) - - def _set_query(self, query_data): - query = mitmproxy.net.http.url.encode(query_data) - _, _, path, params, _, fragment = urllib.parse.urlparse(self.url) - self.path = urllib.parse.urlunparse(["", "", path, params, query, fragment]) - @query.setter def query(self, value): self._set_query(value) + def _get_cookies(self): + h = self.headers.get_all("Cookie") + return tuple(cookies.parse_cookie_headers(h)) + + def _set_cookies(self, value): + self.headers["cookie"] = cookies.format_cookie_header(value) + @property def cookies(self) -> multidict.MultiDictView: """ @@ -362,13 +369,6 @@ class Request(message.Message): self._set_cookies ) - def _get_cookies(self): - h = self.headers.get_all("Cookie") - return tuple(cookies.parse_cookie_headers(h)) - - def _set_cookies(self, value): - self.headers["cookie"] = cookies.format_cookie_header(value) - @cookies.setter def cookies(self, value): self._set_cookies(value) @@ -426,20 +426,6 @@ class Request(message.Message): ) ) - @property - def urlencoded_form(self): - """ - The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. - An empty multidict.MultiDictView if the content-type indicates non-form data - or the content could not be parsed. - - Starting with mitmproxy 1.0, key and value are strings. - """ - return multidict.MultiDictView( - self._get_urlencoded_form, - self._set_urlencoded_form - ) - def _get_urlencoded_form(self): is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower() if is_valid_content_type: @@ -457,24 +443,24 @@ class Request(message.Message): self.headers["content-type"] = "application/x-www-form-urlencoded" self.content = mitmproxy.net.http.url.encode(form_data, self.content.decode()).encode() - @urlencoded_form.setter - def urlencoded_form(self, value): - self._set_urlencoded_form(value) - @property - def multipart_form(self): + def urlencoded_form(self): """ - The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. + The URL-encoded form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. An empty multidict.MultiDictView if the content-type indicates non-form data or the content could not be parsed. - Key and value are bytes. + Starting with mitmproxy 1.0, key and value are strings. """ return multidict.MultiDictView( - self._get_multipart_form, - self._set_multipart_form + self._get_urlencoded_form, + self._set_urlencoded_form ) + @urlencoded_form.setter + def urlencoded_form(self, value): + self._set_urlencoded_form(value) + def _get_multipart_form(self): is_valid_content_type = "multipart/form-data" in self.headers.get("content-type", "").lower() if is_valid_content_type: @@ -487,6 +473,20 @@ class Request(message.Message): def _set_multipart_form(self, value): raise NotImplementedError() + @property + def multipart_form(self): + """ + The multipart form data as an :py:class:`~mitmproxy.net.multidict.MultiDictView` object. + An empty multidict.MultiDictView if the content-type indicates non-form data + or the content could not be parsed. + + Key and value are bytes. + """ + return multidict.MultiDictView( + self._get_multipart_form, + self._set_multipart_form + ) + @multipart_form.setter def multipart_form(self, value): self._set_multipart_form(value) diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py index 53c9c1ca..8edd43b8 100644 --- a/mitmproxy/net/http/response.py +++ b/mitmproxy/net/http/response.py @@ -69,8 +69,8 @@ class Response(message.Message): def make( cls, status_code: int=200, - content: AnyStr=b"", - headers: Union[Dict[AnyStr, AnyStr], Iterable[Tuple[bytes, bytes]]]=() + content: Union[bytes, str]=b"", + headers: Union[Dict[str, AnyStr], Iterable[Tuple[bytes, bytes]]]=() ): """ Simplified API for creating response objects. @@ -129,6 +129,17 @@ class Response(message.Message): def reason(self, reason): self.data.reason = strutils.always_bytes(reason, "ISO-8859-1", "surrogateescape") + def _get_cookies(self): + h = self.headers.get_all("set-cookie") + return tuple(cookies.parse_set_cookie_headers(h)) + + def _set_cookies(self, value): + cookie_headers = [] + for k, v in value: + header = cookies.format_set_cookie_header([(k, v[0], v[1])]) + cookie_headers.append(header) + self.headers.set_all("set-cookie", cookie_headers) + @property def cookies(self) -> multidict.MultiDictView: """ @@ -146,17 +157,6 @@ class Response(message.Message): self._set_cookies ) - def _get_cookies(self): - h = self.headers.get_all("set-cookie") - return tuple(cookies.parse_set_cookie_headers(h)) - - def _set_cookies(self, value): - cookie_headers = [] - for k, v in value: - header = cookies.format_set_cookie_header([(k, v[0], v[1])]) - cookie_headers.append(header) - self.headers.set_all("set-cookie", cookie_headers) - @cookies.setter def cookies(self, value): self._set_cookies(value) diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index 86ce9764..f2c8c473 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -1,4 +1,4 @@ -import urllib +import urllib.parse from typing import Sequence from typing import Tuple diff --git a/mitmproxy/options.py b/mitmproxy/options.py index 5b84ac93..70392803 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -332,7 +332,7 @@ class Options(optmanager.OptManager): Set supported SSL/TLS versions for client connections. SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+. """, - choices=tcp.sslversion_choices.keys(), + choices=list(tcp.sslversion_choices.keys()), ) self.add_option( "ssl_version_server", str, "secure", @@ -340,7 +340,7 @@ class Options(optmanager.OptManager): Set supported SSL/TLS versions for server connections. SSLv2, SSLv3 and 'all' are INSECURE. Defaults to secure, which is TLS1.0+. """, - choices=tcp.sslversion_choices.keys(), + choices=list(tcp.sslversion_choices.keys()), ) self.add_option( "ssl_insecure", bool, False, diff --git a/mitmproxy/proxy/config.py b/mitmproxy/proxy/config.py index 8417ebad..b809d89a 100644 --- a/mitmproxy/proxy/config.py +++ b/mitmproxy/proxy/config.py @@ -37,11 +37,11 @@ class ProxyConfig: def __init__(self, options: moptions.Options) -> None: self.options = options - self.check_ignore = None - self.check_tcp = None - self.certstore = None - self.client_certs = None - self.openssl_verification_mode_server = None + self.check_ignore = None # type: HostMatcher + self.check_tcp = None # type: HostMatcher + self.certstore = None # type: certs.CertStore + self.client_certs = None # type: str + self.openssl_verification_mode_server = None # type: int self.configure(options, set(options.keys())) options.changed.connect(self.configure) diff --git a/mitmproxy/proxy/protocol/base.py b/mitmproxy/proxy/protocol/base.py index b10bb8f5..7c0f78ae 100644 --- a/mitmproxy/proxy/protocol/base.py +++ b/mitmproxy/proxy/protocol/base.py @@ -1,5 +1,7 @@ from mitmproxy import exceptions from mitmproxy import connections +from mitmproxy import controller # noqa +from mitmproxy.proxy import config # noqa class _LayerCodeCompletion: @@ -12,14 +14,10 @@ class _LayerCodeCompletion: super().__init__(**mixin_args) if True: return - self.config = None - """@type: mitmproxy.proxy.ProxyConfig""" - self.client_conn = None - """@type: mitmproxy.connections.ClientConnection""" - self.server_conn = None - """@type: mitmproxy.connections.ServerConnection""" - self.channel = None - """@type: mitmproxy.controller.Channel""" + self.config = None # type: config.ProxyConfig + self.client_conn = None # type: connections.ClientConnection + self.server_conn = None # type: connections.ServerConnection + self.channel = None # type: controller.Channel self.ctx = None """@type: mitmproxy.proxy.protocol.Layer""" diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 16692234..9f783bc3 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -3,10 +3,11 @@ import traceback from mitmproxy import exceptions from mitmproxy import connections +from mitmproxy import controller # noqa from mitmproxy import http from mitmproxy import log from mitmproxy import platform -from mitmproxy.proxy import ProxyConfig +from mitmproxy.proxy import config from mitmproxy.proxy import modes from mitmproxy.proxy import root_context from mitmproxy.net import tcp @@ -34,7 +35,7 @@ class ProxyServer(tcp.TCPServer): allow_reuse_address = True bound = True - def __init__(self, config: ProxyConfig): + def __init__(self, config: config.ProxyConfig) -> None: """ Raises ServerException if there's a startup problem. """ @@ -49,7 +50,7 @@ class ProxyServer(tcp.TCPServer): raise exceptions.ServerException( 'Error starting proxy server: ' + repr(e) ) from e - self.channel = None + self.channel = None # type: controller.Channel def set_channel(self, channel): self.channel = channel @@ -67,8 +68,7 @@ class ProxyServer(tcp.TCPServer): class ConnectionHandler: def __init__(self, client_conn, client_address, config, channel): - self.config = config - """@type: mitmproxy.proxy.config.ProxyConfig""" + self.config = config # type: config.ProxyConfig self.client_conn = connections.ClientConnection( client_conn, client_address, diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index 14159001..a0deaec9 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -1,5 +1,6 @@ from typing import Any from typing import List +from typing import MutableMapping # noqa from mitmproxy.types import serializable @@ -19,7 +20,7 @@ class StateObject(serializable.Serializable): or StateObject instances themselves. """ - _stateobject_attributes = None + _stateobject_attributes = None # type: MutableMapping[str, Any] """ An attribute-name -> class-or-type dict containing all attributes that should be serialized. If the attribute is a class, it must implement the diff --git a/mitmproxy/tcp.py b/mitmproxy/tcp.py index 067fbfe3..fe9f217b 100644 --- a/mitmproxy/tcp.py +++ b/mitmproxy/tcp.py @@ -41,9 +41,7 @@ class TCPFlow(flow.Flow): self.messages = [] # type: List[TCPMessage] _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - _stateobject_attributes.update( - messages=List[TCPMessage] - ) + _stateobject_attributes["messages"] = List[TCPMessage] def __repr__(self): return "<TCPFlow ({} messages)>".format(len(self.messages)) diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py index 29465615..1b90c2e5 100644 --- a/mitmproxy/utils/strutils.py +++ b/mitmproxy/utils/strutils.py @@ -1,11 +1,11 @@ import re import codecs -from typing import AnyStr, Optional +from typing import AnyStr, Optional, cast def always_bytes(str_or_bytes: Optional[AnyStr], *encode_args) -> Optional[bytes]: if isinstance(str_or_bytes, bytes) or str_or_bytes is None: - return str_or_bytes + return cast(Optional[bytes], str_or_bytes) elif isinstance(str_or_bytes, str): return str_or_bytes.encode(*encode_args) else: @@ -18,7 +18,7 @@ def always_str(str_or_bytes: Optional[AnyStr], *decode_args) -> Optional[str]: str_or_bytes unmodified, if """ if isinstance(str_or_bytes, str) or str_or_bytes is None: - return str_or_bytes + return cast(Optional[str], str_or_bytes) elif isinstance(str_or_bytes, bytes): return str_or_bytes.decode(*decode_args) else: diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py index bdd83ee6..e8e2121e 100644 --- a/mitmproxy/utils/typecheck.py +++ b/mitmproxy/utils/typecheck.py @@ -25,10 +25,10 @@ def check_type(name: str, value: typing.Any, typeinfo: type) -> None: if typename.startswith("typing.Union"): try: - types = typeinfo.__args__ + types = typeinfo.__args__ # type: ignore except AttributeError: # Python 3.5.x - types = typeinfo.__union_params__ + types = typeinfo.__union_params__ # type: ignore for T in types: try: @@ -40,10 +40,10 @@ def check_type(name: str, value: typing.Any, typeinfo: type) -> None: raise e elif typename.startswith("typing.Tuple"): try: - types = typeinfo.__args__ + types = typeinfo.__args__ # type: ignore except AttributeError: # Python 3.5.x - types = typeinfo.__tuple_params__ + types = typeinfo.__tuple_params__ # type: ignore if not isinstance(value, (tuple, list)): raise e @@ -54,10 +54,10 @@ def check_type(name: str, value: typing.Any, typeinfo: type) -> None: return elif typename.startswith("typing.Sequence"): try: - T = typeinfo.__args__[0] + T = typeinfo.__args__[0] # type: ignore except AttributeError: # Python 3.5.0 - T = typeinfo.__parameters__[0] + T = typeinfo.__parameters__[0] # type: ignore if not isinstance(value, (tuple, list)): raise e diff --git a/mitmproxy/websocket.py b/mitmproxy/websocket.py index 5d76aafc..2efa7ad1 100644 --- a/mitmproxy/websocket.py +++ b/mitmproxy/websocket.py @@ -8,11 +8,13 @@ from mitmproxy.utils import strutils class WebSocketMessage(serializable.Serializable): - def __init__(self, type: int, from_client: bool, content: bytes, timestamp: Optional[int]=None): + def __init__( + self, type: int, from_client: bool, content: bytes, timestamp: Optional[int]=None + ) -> None: self.type = type self.from_client = from_client self.content = content - self.timestamp = timestamp or time.time() # type: int + self.timestamp = timestamp or int(time.time()) # type: int @classmethod def from_state(cls, state): @@ -62,7 +64,8 @@ class WebSocketFlow(flow.Flow): self.handshake_flow = handshake_flow _stateobject_attributes = flow.Flow._stateobject_attributes.copy() - _stateobject_attributes.update( + # mypy doesn't support update with kwargs + _stateobject_attributes.update(dict( messages=List[WebSocketMessage], close_sender=str, close_code=str, @@ -77,7 +80,7 @@ class WebSocketFlow(flow.Flow): # Do not include handshake_flow, to prevent recursive serialization! # Since mitmproxy-console currently only displays HTTPFlows, # dumping the handshake_flow will include the WebSocketFlow too. - ) + )) @classmethod def from_state(cls, state): @@ -96,7 +96,7 @@ setup( 'dev': [ "Flask>=0.10.1, <0.13", "flake8>=3.2.1, <3.4", - "mypy>=0.471, <0.502", + "mypy>=0.501, <0.502", "rstcheck>=2.2, <4.0", "tox>=2.3, <3", "pytest>=3, <3.1", diff --git a/test/mitmproxy/test_taddons.py b/test/mitmproxy/test_taddons.py index 1e42141c..42371cfe 100644 --- a/test/mitmproxy/test_taddons.py +++ b/test/mitmproxy/test_taddons.py @@ -1,4 +1,3 @@ - from mitmproxy.test import taddons from mitmproxy import ctx @@ -36,6 +36,8 @@ commands = mitmproxy/tools/dump.py \ mitmproxy/tools/web/ \ mitmproxy/contentviews/ + mypy --ignore-missing-imports \ + mitmproxy/master.py [testenv:individual_coverage] deps = |