diff options
43 files changed, 215 insertions, 132 deletions
diff --git a/.travis.yml b/.travis.yml index dca567bf..20afc279 100644 --- a/.travis.yml +++ b/.travis.yml @@ -1,4 +1,3 @@ -sudo: false language: python branches: @@ -34,7 +33,6 @@ matrix: - python: 3.7 env: TOXENV=py37 dist: xenial - sudo: true # required workaround for https://github.com/travis-ci/travis-ci/issues/9815 - language: node_js node_js: "node" before_install: @@ -106,7 +106,7 @@ For speedier testing, we recommend you run `pytest`_ directly on individual test .. code-block:: bash cd test/mitmproxy/addons - pytest --cov mitmproxy.addons.anticache --looponfail test_anticache.py + pytest --cov mitmproxy.addons.anticache --cov-report term-missing --looponfail test_anticache.py As pytest does not check the code style, you probably want to run ``tox -e lint`` before committing your changes. diff --git a/docs/src/content/addons-scripting.md b/docs/src/content/addons-scripting.md index 4e9916ca..6a18eaf4 100644 --- a/docs/src/content/addons-scripting.md +++ b/docs/src/content/addons-scripting.md @@ -27,6 +27,6 @@ You can look at the [http][] module, or the [Request][], and [Response][] classes for other attributes that you can use when scripting. -[http][]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/http.py +[http]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/http.py [Request]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/net/http/request.py [Response]: https://github.com/mitmproxy/mitmproxy/blob/master/mitmproxy/net/http/response.py diff --git a/docs/src/content/concepts-protocols.md b/docs/src/content/concepts-protocols.md index fc056545..c79274bf 100644 --- a/docs/src/content/concepts-protocols.md +++ b/docs/src/content/concepts-protocols.md @@ -36,7 +36,7 @@ mitmproxy currently does not support HTTP/2 Cleartext (h2c) since none of the major browser vendors have implemented it. Some websites are still having problems with correct HTTP/2 support in their -webservers and can cause errors, dropped connectiones, or simply no response at +webservers and can cause errors, dropped connections, or simply no response at all. We are trying to be as tolerant and forgiving as possible with the types of data we send and receive, but [some](https://github.com/mitmproxy/mitmproxy/issues/1745) diff --git a/docs/src/content/howto-transparent.md b/docs/src/content/howto-transparent.md index ae36f579..3915e4b7 100644 --- a/docs/src/content/howto-transparent.md +++ b/docs/src/content/howto-transparent.md @@ -124,7 +124,7 @@ doas pfctl -e You probably want a command like this: {{< highlight bash >}} -mitmproxy --mode transparent --showhost +mitmproxy --mode transparent --listen-host 127.0.0.1 --showhost {{< / highlight >}} The `--mode transparent` option turns on transparent mode, and the `--showhost` argument tells @@ -229,7 +229,7 @@ for more. ### Work-around to redirect traffic originating from the machine itself -Follow the steps **1, 2** as above. In step **3** change the contents of the file **pf.conf** to +Follow steps **1, 2** as above, but in step **2** change the contents of the file **pf.conf** to {{< highlight none >}} #The ports to redirect to proxy @@ -257,7 +257,7 @@ rdr pass proto tcp from any to any port $redir_ports -> $tproxy pass out route-to (lo0 127.0.0.1) proto tcp from any to any port $redir_ports user $redir_users {{< / highlight >}} -Follow steps **4-6** above. This will redirect the packets from all users other than `nobody` on the machine to mitmproxy. To avoid circularity, run mitmproxy as the user `nobody`. Hence step **7** should look like: +Follow steps **3-5** above. This will redirect the packets from all users other than `nobody` on the machine to mitmproxy. To avoid circularity, run mitmproxy as the user `nobody`. Hence step **6** should look like: {{< highlight bash >}} sudo -u nobody mitmproxy --mode transparent --showhost diff --git a/docs/src/content/howto-wireshark-tls.md b/docs/src/content/howto-wireshark-tls.md index 588223ac..a55d177b 100644 --- a/docs/src/content/howto-wireshark-tls.md +++ b/docs/src/content/howto-wireshark-tls.md @@ -7,7 +7,7 @@ menu: # Wireshark and SSL/TLS Master Secrets -The SSL/SSL master keys can be logged by mitmproxy so that external programs can +The SSL/TLS master keys can be logged by mitmproxy so that external programs can decrypt SSL/TLS connections both from and to the proxy. Recent versions of Wireshark can use these log files to decrypt packets. See the [Wireshark wiki](https://wiki.wireshark.org/SSL#Using_the_.28Pre.29-Master-Secret) for more information. diff --git a/docs/src/content/tute-highscores.md b/docs/src/content/tute-highscores.md index f5cbd7bc..2d03076d 100644 --- a/docs/src/content/tute-highscores.md +++ b/docs/src/content/tute-highscores.md @@ -67,7 +67,7 @@ timestamp. Looks pretty simple to mess with. Lets edit the score submission. First, select it in mitmproxy, then press <span data-role="kbd">enter</span> to view it. Make sure you're -viewing the request, not the response -you can use +viewing the request, not the response - you can use <span data-role="kbd">tab</span> to flick between the two. Now press <span data-role="kbd">e</span> for edit. You'll be prompted for the part of the request you want to change - press <span data-role="kbd">r</span> diff --git a/examples/addons/commands-paths.py b/examples/addons/commands-paths.py index f37a0fbc..4d9535b9 100644 --- a/examples/addons/commands-paths.py +++ b/examples/addons/commands-paths.py @@ -20,9 +20,9 @@ class MyAddon: for f in flows: totals[f.request.host] = totals.setdefault(f.request.host, 0) + 1 - fp = open(path, "w+") - for cnt, dom in sorted([(v, k) for (k, v) in totals.items()]): - fp.write("%s: %s\n" % (cnt, dom)) + with open(path, "w+") as fp: + for cnt, dom in sorted([(v, k) for (k, v) in totals.items()]): + fp.write("%s: %s\n" % (cnt, dom)) ctx.log.alert("done") diff --git a/examples/complex/sslstrip.py b/examples/complex/sslstrip.py index c862536f..69b9ea9e 100644 --- a/examples/complex/sslstrip.py +++ b/examples/complex/sslstrip.py @@ -38,7 +38,7 @@ def response(flow: http.HTTPFlow) -> None: flow.response.content = flow.response.content.replace(b'https://', b'http://') # strip meta tag upgrade-insecure-requests in response body - csp_meta_tag_pattern = b'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>' + csp_meta_tag_pattern = br'<meta.*http-equiv=["\']Content-Security-Policy[\'"].*upgrade-insecure-requests.*?>' flow.response.content = re.sub(csp_meta_tag_pattern, b'', flow.response.content, flags=re.IGNORECASE) # strip links in 'Location' header @@ -52,7 +52,7 @@ def response(flow: http.HTTPFlow) -> None: # strip upgrade-insecure-requests in Content-Security-Policy header if re.search('upgrade-insecure-requests', flow.response.headers.get('Content-Security-Policy', ''), flags=re.IGNORECASE): csp = flow.response.headers['Content-Security-Policy'] - flow.response.headers['Content-Security-Policy'] = re.sub('upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) + flow.response.headers['Content-Security-Policy'] = re.sub(r'upgrade-insecure-requests[;\s]*', '', csp, flags=re.IGNORECASE) # strip secure flag from 'Set-Cookie' headers cookies = flow.response.headers.get_all('Set-Cookie') diff --git a/examples/complex/xss_scanner.py b/examples/complex/xss_scanner.py index cdaaf478..97e94ed4 100755 --- a/examples/complex/xss_scanner.py +++ b/examples/complex/xss_scanner.py @@ -1,4 +1,4 @@ -""" +r""" __ __ _____ _____ _____ \ \ / // ____/ ____| / ____| diff --git a/mitmproxy/addonmanager.py b/mitmproxy/addonmanager.py index 4214d6ea..8a565a73 100644 --- a/mitmproxy/addonmanager.py +++ b/mitmproxy/addonmanager.py @@ -184,7 +184,7 @@ class AddonManager: raise exceptions.AddonManagerError("No such addon: %s" % n) self.chain = [i for i in self.chain if i is not a] del self.lookup[_get_name(a)] - self.invoke_addon(a, "done") + self.invoke_addon(addon, "done") def __len__(self): return len(self.chain) diff --git a/mitmproxy/addons/session.py b/mitmproxy/addons/session.py index 63e382ec..f9073c3e 100644 --- a/mitmproxy/addons/session.py +++ b/mitmproxy/addons/session.py @@ -87,8 +87,8 @@ class SessionDB: def _create_session(self): script_path = pkg_data.path("io/sql/session_create.sql") - qry = open(script_path, 'r').read() - self.con.executescript(qry) + with open(script_path, 'r') as qry: + self.con.executescript(qry.read()) self.con.commit() @staticmethod diff --git a/mitmproxy/contentviews/css.py b/mitmproxy/contentviews/css.py index cbe8ce62..44b33761 100644 --- a/mitmproxy/contentviews/css.py +++ b/mitmproxy/contentviews/css.py @@ -16,7 +16,7 @@ A custom CSS prettifier. Compared to other prettifiers, its main features are: CSS_SPECIAL_AREAS = ( "'" + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + "'", '"' + strutils.SINGLELINE_CONTENT + strutils.NO_ESCAPE + '"', - r"/\*" + strutils.MULTILINE_CONTENT + "\*/", + r"/\*" + strutils.MULTILINE_CONTENT + r"\*/", "//" + strutils.SINGLELINE_CONTENT + "$" ) CSS_SPECIAL_CHARS = "{};:" diff --git a/mitmproxy/contentviews/javascript.py b/mitmproxy/contentviews/javascript.py index 1440ea5d..b5f09150 100644 --- a/mitmproxy/contentviews/javascript.py +++ b/mitmproxy/contentviews/javascript.py @@ -10,9 +10,9 @@ SPECIAL_AREAS = ( r"'" + strutils.MULTILINE_CONTENT_LINE_CONTINUATION + strutils.NO_ESCAPE + "'", r'"' + strutils.MULTILINE_CONTENT_LINE_CONTINUATION + strutils.NO_ESCAPE + '"', r'`' + strutils.MULTILINE_CONTENT + strutils.NO_ESCAPE + '`', - r"/\*" + strutils.MULTILINE_CONTENT + "\*/", + r"/\*" + strutils.MULTILINE_CONTENT + r"\*/", r"//" + strutils.SINGLELINE_CONTENT + "$", - r"for\(" + strutils.SINGLELINE_CONTENT + "\)", + r"for\(" + strutils.SINGLELINE_CONTENT + r"\)", ) diff --git a/mitmproxy/contentviews/xml_html.py b/mitmproxy/contentviews/xml_html.py index 658fbcd7..00a62a15 100644 --- a/mitmproxy/contentviews/xml_html.py +++ b/mitmproxy/contentviews/xml_html.py @@ -18,7 +18,7 @@ The implementation is split into two main parts: tokenization and formatting of """ # http://www.xml.com/pub/a/2001/07/25/namingparts.html - this is close enough for what we do. -REGEX_TAG = re.compile("[a-zA-Z0-9._:\-]+(?!=)") +REGEX_TAG = re.compile(r"[a-zA-Z0-9._:\-]+(?!=)") # https://www.w3.org/TR/html5/syntax.html#void-elements HTML_VOID_ELEMENTS = { "area", "base", "br", "col", "embed", "hr", "img", "input", "keygen", "link", "meta", "param", diff --git a/mitmproxy/contrib/kaitaistruct/exif_be.py b/mitmproxy/contrib/kaitaistruct/exif_be.py index 8a6e7a2b..88ce4e54 100644 --- a/mitmproxy/contrib/kaitaistruct/exif_be.py +++ b/mitmproxy/contrib/kaitaistruct/exif_be.py @@ -1,12 +1,8 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -import array -import struct -import zlib -from enum import Enum from pkg_resources import parse_version - from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +from enum import Enum if parse_version(ks_version) < parse_version('0.7'): @@ -17,6 +13,9 @@ class ExifBe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.version = self._io.read_u2be() self.ifd0_ofs = self._io.read_u4be() @@ -25,6 +24,9 @@ class ExifBe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.num_fields = self._io.read_u2be() self.fields = [None] * (self.num_fields) for i in range(self.num_fields): @@ -54,6 +56,9 @@ class ExifBe(KaitaiStruct): word = 3 dword = 4 rational = 5 + undefined = 7 + slong = 9 + srational = 10 class TagEnum(Enum): image_width = 256 @@ -518,6 +523,9 @@ class ExifBe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.tag = self._root.IfdField.TagEnum(self._io.read_u2be()) self.field_type = self._root.IfdField.FieldTypeEnum(self._io.read_u2be()) self.length = self._io.read_u4be() @@ -552,7 +560,7 @@ class ExifBe(KaitaiStruct): if hasattr(self, '_m_data'): return self._m_data if hasattr(self, '_m_data') else None - if not self.is_immediate_data: + if not (self.is_immediate_data): io = self._root._io _pos = io.pos() io.seek(self.ofs_or_data) diff --git a/mitmproxy/contrib/kaitaistruct/exif_le.py b/mitmproxy/contrib/kaitaistruct/exif_le.py index 84e53a38..e25a2fc9 100644 --- a/mitmproxy/contrib/kaitaistruct/exif_le.py +++ b/mitmproxy/contrib/kaitaistruct/exif_le.py @@ -1,12 +1,8 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -import array -import struct -import zlib -from enum import Enum from pkg_resources import parse_version - from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +from enum import Enum if parse_version(ks_version) < parse_version('0.7'): @@ -17,6 +13,9 @@ class ExifLe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.version = self._io.read_u2le() self.ifd0_ofs = self._io.read_u4le() @@ -25,6 +24,9 @@ class ExifLe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.num_fields = self._io.read_u2le() self.fields = [None] * (self.num_fields) for i in range(self.num_fields): @@ -54,6 +56,9 @@ class ExifLe(KaitaiStruct): word = 3 dword = 4 rational = 5 + undefined = 7 + slong = 9 + srational = 10 class TagEnum(Enum): image_width = 256 @@ -518,6 +523,9 @@ class ExifLe(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self + self._read() + + def _read(self): self.tag = self._root.IfdField.TagEnum(self._io.read_u2le()) self.field_type = self._root.IfdField.FieldTypeEnum(self._io.read_u2le()) self.length = self._io.read_u4le() @@ -552,7 +560,7 @@ class ExifLe(KaitaiStruct): if hasattr(self, '_m_data'): return self._m_data if hasattr(self, '_m_data') else None - if not self.is_immediate_data: + if not (self.is_immediate_data): io = self._root._io _pos = io.pos() io.seek(self.ofs_or_data) diff --git a/mitmproxy/contrib/wbxml/ASCommandResponse.py b/mitmproxy/contrib/wbxml/ASCommandResponse.py index 2d60eb2d..34755cbe 100644 --- a/mitmproxy/contrib/wbxml/ASCommandResponse.py +++ b/mitmproxy/contrib/wbxml/ASCommandResponse.py @@ -63,8 +63,9 @@ if __name__ == "__main__": listOfSamples = os.listdir(samplesDir) for filename in listOfSamples: - byteWBXML = open(samplesDir + os.sep + filename, "rb").read() - + with open(samplesDir + os.sep + filename, "rb") as f: + byteWBXML = f.read() + logging.info("-"*100) logging.info(filename) logging.info("-"*100) diff --git a/mitmproxy/net/check.py b/mitmproxy/net/check.py index aaea851f..a19ad6fe 100644 --- a/mitmproxy/net/check.py +++ b/mitmproxy/net/check.py @@ -2,7 +2,7 @@ import ipaddress import re # Allow underscore in host name -_label_valid = re.compile(b"(?!-)[A-Z\d\-_]{1,63}(?<!-)$", re.IGNORECASE) +_label_valid = re.compile(br"(?!-)[A-Z\d\-_]{1,63}(?<!-)$", re.IGNORECASE) def is_valid_host(host: bytes) -> bool: diff --git a/mitmproxy/net/http/cookies.py b/mitmproxy/net/http/cookies.py index 1472ab55..2745701f 100644 --- a/mitmproxy/net/http/cookies.py +++ b/mitmproxy/net/http/cookies.py @@ -304,7 +304,7 @@ def refresh_set_cookie_header(c: str, delta: int) -> str: e = email.utils.parsedate_tz(attrs["expires"]) if e: f = email.utils.mktime_tz(e) + delta - attrs.set_all("expires", [email.utils.formatdate(f)]) + attrs.set_all("expires", [email.utils.formatdate(f, usegmt=True)]) else: # This can happen when the expires tag is invalid. # reddit.com sends a an expires tag like this: "Thu, 31 Dec diff --git a/mitmproxy/net/http/response.py b/mitmproxy/net/http/response.py index 48527d63..9491fc03 100644 --- a/mitmproxy/net/http/response.py +++ b/mitmproxy/net/http/response.py @@ -186,7 +186,7 @@ class Response(message.Message): d = parsedate_tz(self.headers[i]) if d: new = mktime_tz(d) + delta - self.headers[i] = formatdate(new) + self.headers[i] = formatdate(new, usegmt=True) c = [] for set_cookie_header in self.headers.get_all("set-cookie"): try: diff --git a/mitmproxy/net/websockets/masker.py b/mitmproxy/net/websockets/masker.py index 47b1a688..6134e09e 100644 --- a/mitmproxy/net/websockets/masker.py +++ b/mitmproxy/net/websockets/masker.py @@ -1,3 +1,6 @@ +import sys + + class Masker: """ Data sent from the server must be masked to prevent malicious clients @@ -12,12 +15,13 @@ class Masker: self.offset = 0 def mask(self, offset, data): - result = bytearray(data) - for i in range(len(data)): - result[i] ^= self.key[offset % 4] - offset += 1 - result = bytes(result) - return result + datalen = len(data) + offset_mod = offset % 4 + data = int.from_bytes(data, sys.byteorder) + num_keys = (datalen + offset_mod + 3) // 4 + mask = int.from_bytes((self.key * num_keys)[offset_mod:datalen + + offset_mod], sys.byteorder) + return (data ^ mask).to_bytes(datalen, sys.byteorder) def __call__(self, data): ret = self.mask(self.offset, data) diff --git a/mitmproxy/optmanager.py b/mitmproxy/optmanager.py index 06e696c0..6e187b0d 100644 --- a/mitmproxy/optmanager.py +++ b/mitmproxy/optmanager.py @@ -320,7 +320,9 @@ class OptManager: update = {} for optname, optval in self.deferred.items(): if optname in self._options: - update[optname] = self.parse_setval(self._options[optname], optval) + if isinstance(optval, str): + optval = self.parse_setval(self._options[optname], optval) + update[optname] = optval self.update(**update) for k in update.keys(): del self.deferred[k] diff --git a/mitmproxy/platform/pf.py b/mitmproxy/platform/pf.py index bb5eb515..5e22ec31 100644 --- a/mitmproxy/platform/pf.py +++ b/mitmproxy/platform/pf.py @@ -11,7 +11,7 @@ def lookup(address, port, s): """ # We may get an ipv4-mapped ipv6 address here, e.g. ::ffff:127.0.0.1. # Those still appear as "127.0.0.1" in the table, so we need to strip the prefix. - address = re.sub("^::ffff:(?=\d+.\d+.\d+.\d+$)", "", address) + address = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", address) s = s.decode() spec = "%s:%s" % (address, port) for i in s.split("\n"): diff --git a/mitmproxy/platform/windows.py b/mitmproxy/platform/windows.py index b849afa5..19d9abd4 100644 --- a/mitmproxy/platform/windows.py +++ b/mitmproxy/platform/windows.py @@ -13,6 +13,7 @@ import typing import click import collections +import collections.abc import pydivert import pydivert.consts @@ -58,7 +59,7 @@ class Resolver: def original_addr(self, csock: socket.socket): ip, port = csock.getpeername()[:2] - ip = re.sub("^::ffff:(?=\d+.\d+.\d+.\d+$)", "", ip) + ip = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", ip) ip = ip.split("%", 1)[0] with self.lock: try: @@ -171,7 +172,7 @@ def MIB_TCPTABLE_OWNER_PID(size): TCP_TABLE_OWNER_PID_CONNECTIONS = 4 -class TcpConnectionTable(collections.Mapping): +class TcpConnectionTable(collections.abc.Mapping): DEFAULT_TABLE_SIZE = 4096 def __init__(self): diff --git a/mitmproxy/proxy/protocol/websocket.py b/mitmproxy/proxy/protocol/websocket.py index 0d1964a6..f5ac6a29 100644 --- a/mitmproxy/proxy/protocol/websocket.py +++ b/mitmproxy/proxy/protocol/websocket.py @@ -4,8 +4,9 @@ from OpenSSL import SSL import wsproto -from wsproto import events -from wsproto.connection import ConnectionType, WSConnection +from wsproto import events, WSConnection +from wsproto.connection import ConnectionType +from wsproto.events import AcceptConnection, CloseConnection, Message, Ping, Request from wsproto.extensions import PerMessageDeflate from mitmproxy import exceptions @@ -52,51 +53,52 @@ class WebSocketLayer(base.Layer): self.connections: dict[object, WSConnection] = {} - extensions = [] + client_extensions = [] + server_extensions = [] if 'Sec-WebSocket-Extensions' in handshake_flow.response.headers: if PerMessageDeflate.name in handshake_flow.response.headers['Sec-WebSocket-Extensions']: - extensions = [PerMessageDeflate()] - self.connections[self.client_conn] = WSConnection(ConnectionType.SERVER, - extensions=extensions) - self.connections[self.server_conn] = WSConnection(ConnectionType.CLIENT, - host=handshake_flow.request.host, - resource=handshake_flow.request.path, - extensions=extensions) - if extensions: - for conn in self.connections.values(): - conn.extensions[0].finalize(conn, handshake_flow.response.headers['Sec-WebSocket-Extensions']) - - data = self.connections[self.server_conn].bytes_to_send() - self.connections[self.client_conn].receive_bytes(data) + client_extensions = [PerMessageDeflate()] + server_extensions = [PerMessageDeflate()] + self.connections[self.client_conn] = WSConnection(ConnectionType.SERVER) + self.connections[self.server_conn] = WSConnection(ConnectionType.CLIENT) + + if client_extensions: + client_extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) + if server_extensions: + server_extensions[0].finalize(handshake_flow.response.headers['Sec-WebSocket-Extensions']) + + request = Request(extensions=client_extensions, host=handshake_flow.request.host, target=handshake_flow.request.path) + data = self.connections[self.server_conn].send(request) + self.connections[self.client_conn].receive_data(data) event = next(self.connections[self.client_conn].events()) - assert isinstance(event, events.ConnectionRequested) + assert isinstance(event, events.Request) - self.connections[self.client_conn].accept(event) - self.connections[self.server_conn].receive_bytes(self.connections[self.client_conn].bytes_to_send()) - assert isinstance(next(self.connections[self.server_conn].events()), events.ConnectionEstablished) + data = self.connections[self.client_conn].send(AcceptConnection(extensions=server_extensions)) + self.connections[self.server_conn].receive_data(data) + assert isinstance(next(self.connections[self.server_conn].events()), events.AcceptConnection) def _handle_event(self, event, source_conn, other_conn, is_server): - if isinstance(event, events.DataReceived): - return self._handle_data_received(event, source_conn, other_conn, is_server) - elif isinstance(event, events.PingReceived): - return self._handle_ping_received(event, source_conn, other_conn, is_server) - elif isinstance(event, events.PongReceived): - return self._handle_pong_received(event, source_conn, other_conn, is_server) - elif isinstance(event, events.ConnectionClosed): - return self._handle_connection_closed(event, source_conn, other_conn, is_server) + if isinstance(event, events.Message): + return self._handle_message(event, source_conn, other_conn, is_server) + elif isinstance(event, events.Ping): + return self._handle_ping(event, source_conn, other_conn, is_server) + elif isinstance(event, events.Pong): + return self._handle_pong(event, source_conn, other_conn, is_server) + elif isinstance(event, events.CloseConnection): + return self._handle_close_connection(event, source_conn, other_conn, is_server) # fail-safe for unhandled events return True # pragma: no cover - def _handle_data_received(self, event, source_conn, other_conn, is_server): + def _handle_message(self, event, source_conn, other_conn, is_server): fb = self.server_frame_buffer if is_server else self.client_frame_buffer fb.append(event.data) if event.message_finished: original_chunk_sizes = [len(f) for f in fb] - if isinstance(event, events.TextReceived): + if isinstance(event, events.TextMessage): message_type = wsproto.frame_protocol.Opcode.TEXT payload = ''.join(fb) else: @@ -127,19 +129,20 @@ class WebSocketLayer(base.Layer): yield (payload[i:i + chunk_size], True if i + chunk_size >= len(payload) else False) for chunk, final in get_chunk(websocket_message.content): - self.connections[other_conn].send_data(chunk, final) - other_conn.send(self.connections[other_conn].bytes_to_send()) + data = self.connections[other_conn].send(Message(data=chunk, message_finished=final)) + other_conn.send(data) if self.flow.stream: - self.connections[other_conn].send_data(event.data, event.message_finished) - other_conn.send(self.connections[other_conn].bytes_to_send()) + data = self.connections[other_conn].send(Message(data=event.data, message_finished=event.message_finished)) + other_conn.send(data) return True - def _handle_ping_received(self, event, source_conn, other_conn, is_server): - # PING is automatically answered with a PONG by wsproto - self.connections[other_conn].ping() - other_conn.send(self.connections[other_conn].bytes_to_send()) - source_conn.send(self.connections[source_conn].bytes_to_send()) + def _handle_ping(self, event, source_conn, other_conn, is_server): + # Use event.response to create the approprate Pong response + data = self.connections[other_conn].send(Ping()) + other_conn.send(data) + data = self.connections[source_conn].send(event.response()) + source_conn.send(data) self.log( "Ping Received from {}".format("server" if is_server else "client"), "info", @@ -147,7 +150,7 @@ class WebSocketLayer(base.Layer): ) return True - def _handle_pong_received(self, event, source_conn, other_conn, is_server): + def _handle_pong(self, event, source_conn, other_conn, is_server): self.log( "Pong Received from {}".format("server" if is_server else "client"), "info", @@ -155,14 +158,15 @@ class WebSocketLayer(base.Layer): ) return True - def _handle_connection_closed(self, event, source_conn, other_conn, is_server): + def _handle_close_connection(self, event, source_conn, other_conn, is_server): self.flow.close_sender = "server" if is_server else "client" self.flow.close_code = event.code self.flow.close_reason = event.reason - self.connections[other_conn].close(event.code, event.reason) - other_conn.send(self.connections[other_conn].bytes_to_send()) - source_conn.send(self.connections[source_conn].bytes_to_send()) + data = self.connections[other_conn].send(CloseConnection(code=event.code, reason=event.reason)) + other_conn.send(data) + data = self.connections[source_conn].send(event.response()) + source_conn.send(data) return False @@ -170,8 +174,7 @@ class WebSocketLayer(base.Layer): while True: try: payload = message_queue.get_nowait() - self.connections[endpoint].send_data(payload, final=True) - data = self.connections[endpoint].bytes_to_send() + data = self.connections[endpoint].send(Message(data=payload, message_finished=True)) endpoint.send(data) except queue.Empty: break @@ -197,8 +200,8 @@ class WebSocketLayer(base.Layer): is_server = (source_conn == self.server_conn) frame = websockets.Frame.from_file(source_conn.rfile) - self.connections[source_conn].receive_bytes(bytes(frame)) - source_conn.send(self.connections[source_conn].bytes_to_send()) + data = self.connections[source_conn].receive_data(bytes(frame)) + source_conn.send(data) if close_received: return diff --git a/mitmproxy/tools/_main.py b/mitmproxy/tools/_main.py index f1c763b2..b95d73ab 100644 --- a/mitmproxy/tools/_main.py +++ b/mitmproxy/tools/_main.py @@ -87,7 +87,7 @@ def run( arg_check.check() sys.exit(1) try: - opts.confdir = args.confdir + opts.set(*args.setoptions, defer=True) optmanager.load_paths( opts, os.path.join(opts.confdir, OPTIONS_FILE_NAME), @@ -110,7 +110,6 @@ def run( if args.commands: master.commands.dump() sys.exit(0) - opts.set(*args.setoptions, defer=True) if extra: opts.update(**extra(args)) diff --git a/mitmproxy/tools/cmdline.py b/mitmproxy/tools/cmdline.py index ad934ca2..21369a1f 100644 --- a/mitmproxy/tools/cmdline.py +++ b/mitmproxy/tools/cmdline.py @@ -1,8 +1,5 @@ import argparse -from mitmproxy.addons import core - - def common_options(parser, opts): parser.add_argument( '--version', @@ -21,12 +18,6 @@ def common_options(parser, opts): help="Show all commands and their signatures", ) parser.add_argument( - "--confdir", - type=str, dest="confdir", default=core.CONF_DIR, - metavar="PATH", - help="Path to the mitmproxy config directory" - ) - parser.add_argument( "--set", type=str, dest="setoptions", default=[], action="append", diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 49f5c247..58a83c0e 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -309,6 +309,9 @@ def raw_format_flow(f): methods = { 'GET': 'method_get', 'POST': 'method_post', + 'DELETE': 'method_delete', + 'HEAD': 'method_head', + 'PUT': 'method_put' } uc = methods.get(f["req_method"], "method_other") if f['extended']: diff --git a/mitmproxy/tools/console/help.py b/mitmproxy/tools/console/help.py index 1b4b9ac6..fb4e0051 100644 --- a/mitmproxy/tools/console/help.py +++ b/mitmproxy/tools/console/help.py @@ -91,9 +91,9 @@ class HelpView(tabs.Tabs, layoutwidget.LayoutWidget): ) ) examples = [ - ("google\.com", "Url containing \"google.com"), - ("~q ~b test", "Requests where body contains \"test\""), - ("!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."), + (r"google\.com", r"Url containing \"google.com"), + ("~q ~b test", r"Requests where body contains \"test\""), + (r"!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."), ] text.extend( common.format_keyvals(examples, indent=4) diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 405f1a6c..befe3044 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -23,7 +23,7 @@ class Palette: # List and Connections 'method', - 'method_get', 'method_post', 'method_other', 'method_http2_push', + 'method_get', 'method_post', 'method_delete', 'method_other', 'method_head', 'method_put', 'method_http2_push', 'scheme_http', 'scheme_https', 'scheme_other', 'url_punctuation', 'url_domain', 'url_filename', 'url_extension', 'url_query_key', 'url_query_value', 'content_none', 'content_text', 'content_script', 'content_media', 'content_data', 'content_raw', 'content_other', @@ -122,8 +122,11 @@ class LowDark(Palette): # List and Connections method = ('dark cyan', 'default'), - method_get = ('dark cyan', 'default'), - method_post = ('dark red', 'default'), + method_get = ('light green', 'default'), + method_post = ('brown', 'default'), + method_delete = ('light red', 'default'), + method_head = ('dark cyan', 'default'), + method_put = ('dark red', 'default'), method_other = ('dark magenta', 'default'), method_http2_push = ('dark gray', 'default'), @@ -131,6 +134,7 @@ class LowDark(Palette): scheme_https = ('dark green', 'default'), scheme_other = ('dark magenta', 'default'), + url_punctuation = ('light gray', 'default'), url_punctuation = ('dark gray', 'default'), url_domain = ('white', 'default'), url_filename = ('dark cyan', 'default'), @@ -219,6 +223,33 @@ class LowLight(Palette): # List and Connections method = ('dark cyan', 'default'), + method_get = ('dark green', 'default'), + method_post = ('brown', 'default'), + method_head = ('dark cyan', 'default'), + method_put = ('light red', 'default'), + method_delete = ('dark red', 'default'), + method_other = ('light magenta', 'default'), + method_http2_push = ('light gray','default'), + + scheme_http = ('dark cyan', 'default'), + scheme_https = ('light green', 'default'), + scheme_other = ('light magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('dark gray', 'default'), + url_filename = ('black', 'default'), + url_extension = ('dark gray', 'default'), + url_query_key = ('light blue', 'default'), + url_query_value = ('dark blue', 'default'), + + content_none = ('black', 'default'), + content_text = ('dark gray', 'default'), + content_script = ('light green', 'default'), + content_media = ('light blue', 'default'), + content_data = ('brown', 'default'), + content_raw = ('light red', 'default'), + content_other = ('light magenta', 'default'), + focus = ('black', 'default'), code_200 = ('dark green', 'default'), @@ -250,6 +281,7 @@ class LowLight(Palette): commander_invalid = ('light red', 'default'), commander_hint = ('light gray', 'default'), ) + gen_gradient(low, ['light red', 'yellow', 'light green', 'dark green', 'dark cyan', 'dark blue']) class Light(LowLight): @@ -308,7 +340,27 @@ class SolarizedLight(LowLight): option_active_selected = (sol_orange, sol_base2), # List and Connections - method = (sol_cyan, 'default'), + + method = ('dark cyan', 'default'), + method_get = (sol_green, 'default'), + method_post = (sol_orange, 'default'), + method_head = (sol_cyan, 'default'), + method_put = (sol_red, 'default'), + method_delete = (sol_red, 'default'), + method_other = (sol_magenta, 'default'), + method_http2_push = ('light gray','default'), + + scheme_http = (sol_cyan, 'default'), + scheme_https = ('light green', 'default'), + scheme_other = ('light magenta', 'default'), + + url_punctuation = ('dark gray', 'default'), + url_domain = ('dark gray', 'default'), + url_filename = ('black', 'default'), + url_extension = ('dark gray', 'default'), + url_query_key = (sol_blue, 'default'), + url_query_value = ('dark blue', 'default'), + focus = (sol_base01, 'default'), code_200 = (sol_green, 'default'), @@ -363,9 +415,16 @@ class SolarizedDark(LowDark): option_active_selected = (sol_orange, sol_base00), # List and Connections + focus = (sol_base1, 'default'), + method = (sol_cyan, 'default'), + method_get = (sol_green, 'default'), + method_post = (sol_orange, 'default'), + method_delete = (sol_red, 'default'), + method_head = (sol_cyan, 'default'), + method_put = (sol_red, 'default'), + method_other = (sol_magenta, 'default'), method_http2_push = (sol_base01, 'default'), - focus = (sol_base1, 'default'), url_punctuation = ('h242', 'default'), url_domain = ('h252', 'default'), diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index b72e0d77..6e6b6223 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -370,7 +370,7 @@ class FlowContent(RequestHandler): original_cd = message.headers.get("Content-Disposition", None) filename = None if original_cd: - filename = re.search('filename=([-\w" .()]+)', original_cd) + filename = re.search(r'filename=([-\w" .()]+)', original_cd) if filename: filename = filename.group(1) if not filename: diff --git a/mitmproxy/utils/strutils.py b/mitmproxy/utils/strutils.py index 71d1c54c..388c765f 100644 --- a/mitmproxy/utils/strutils.py +++ b/mitmproxy/utils/strutils.py @@ -169,7 +169,7 @@ def split_special_areas( >>> split_special_areas( >>> "test /* don't modify me */ foo", - >>> [r"/\*[\s\S]*?\*/"]) # (regex matching comments) + >>> [r"/\\*[\\s\\S]*?\\*/"]) # (regex matching comments) ["test ", "/* don't modify me */", " foo"] "".join(split_special_areas(x, ...)) == x always holds true. @@ -201,7 +201,7 @@ def escape_special_areas( >>> x = escape_special_areas(x, "{", ["'" + SINGLELINE_CONTENT + "'"]) >>> print(x) if (true) { console.log('�}'); } - >>> x = re.sub(r"\s*{\s*", " {\n ", x) + >>> x = re.sub(r"\\s*{\\s*", " {\n ", x) >>> x = unescape_special_areas(x) >>> print(x) if (true) { @@ -62,7 +62,7 @@ setup( # It is not considered best practice to use install_requires to pin dependencies to specific versions. install_requires=[ "blinker>=1.4, <1.5", - "brotlipy>=0.7.0,<0.8", + "Brotli>=1.0,<1.1", "certifi>=2015.11.20.1", # no semver here - this should always be on the last release! "click>=6.2, <7", "cryptography>=2.1.4,<2.5", @@ -80,7 +80,7 @@ setup( "sortedcontainers>=1.5.4,<2.1", "tornado>=4.3,<5.2", "urwid>=2.0.1,<2.1", - "wsproto>=0.12.0,<0.13.0", + "wsproto>=0.13.0,<0.14.0", ], extras_require={ ':sys_platform == "win32"': [ diff --git a/test/bench/benchmark.py b/test/bench/benchmark.py index 84ec6005..076ad6c9 100644 --- a/test/bench/benchmark.py +++ b/test/bench/benchmark.py @@ -31,7 +31,8 @@ class Benchmark: stdout=asyncio.subprocess.PIPE ) stdout, _ = await traf.communicate() - open(ctx.options.benchmark_save_path + ".bench", mode="wb").write(stdout) + with open(ctx.options.benchmark_save_path + ".bench", mode="wb") as f: + f.write(stdout) ctx.log.error("Proxy saw %s requests, %s responses" % (self.reqs, self.resps)) ctx.log.error(stdout.decode("ascii")) backend.kill() diff --git a/test/mitmproxy/addons/test_session.py b/test/mitmproxy/addons/test_session.py index 20feb69d..97351426 100644 --- a/test/mitmproxy/addons/test_session.py +++ b/test/mitmproxy/addons/test_session.py @@ -68,7 +68,8 @@ class TestSession: os.remove(path) con = sqlite3.connect(path) script_path = pkg_data.path("io/sql/session_create.sql") - qry = open(script_path, 'r').read() + with open(script_path) as f: + qry = f.read() with con: con.executescript(qry) blob = b'blob_of_data' diff --git a/test/mitmproxy/coretypes/test_basethread.py b/test/mitmproxy/coretypes/test_basethread.py index 4a383fea..6b0ae154 100644 --- a/test/mitmproxy/coretypes/test_basethread.py +++ b/test/mitmproxy/coretypes/test_basethread.py @@ -4,4 +4,4 @@ from mitmproxy.coretypes import basethread def test_basethread(): t = basethread.BaseThread('foobar') - assert re.match('foobar - age: \d+s', t._threadinfo()) + assert re.match(r'foobar - age: \d+s', t._threadinfo()) diff --git a/test/mitmproxy/net/http/test_cookies.py b/test/mitmproxy/net/http/test_cookies.py index 74233cca..06cfe1d3 100644 --- a/test/mitmproxy/net/http/test_cookies.py +++ b/test/mitmproxy/net/http/test_cookies.py @@ -27,7 +27,7 @@ cookie_pairs = [ [["one", "uno"], ["two", "due"]] ], [ - 'one="uno"; two="\due"', + 'one="uno"; two="\\due"', [["one", "uno"], ["two", "due"]] ], [ @@ -70,7 +70,7 @@ def test_read_key(): def test_read_quoted_string(): tokens = [ [('"foo" x', 0), ("foo", 5)], - [('"f\oo" x', 0), ("foo", 6)], + [('"f\\oo" x', 0), ("foo", 6)], [(r'"f\\o" x', 0), (r"f\o", 6)], [(r'"f\\" x', 0), (r"f" + '\\', 5)], [('"fo\\\"" x', 0), ("fo\"", 6)], diff --git a/test/mitmproxy/net/http/test_response.py b/test/mitmproxy/net/http/test_response.py index f3470384..27c16be6 100644 --- a/test/mitmproxy/net/http/test_response.py +++ b/test/mitmproxy/net/http/test_response.py @@ -148,7 +148,7 @@ class TestResponseUtils: def test_refresh(self): r = tresp() n = time.time() - r.headers["date"] = email.utils.formatdate(n) + r.headers["date"] = email.utils.formatdate(n, usegmt=True) pre = r.headers["date"] r.refresh(946681202) assert pre == r.headers["date"] diff --git a/test/mitmproxy/net/test_tcp.py b/test/mitmproxy/net/test_tcp.py index b6bb7cc1..22a306dc 100644 --- a/test/mitmproxy/net/test_tcp.py +++ b/test/mitmproxy/net/test_tcp.py @@ -102,7 +102,7 @@ class TestServerBind(tservers.ServerTestBase): # We may get an ipv4-mapped ipv6 address here, e.g. ::ffff:127.0.0.1. # Those still appear as "127.0.0.1" in the table, so we need to strip the prefix. peername = self.connection.getpeername() - address = re.sub("^::ffff:(?=\d+.\d+.\d+.\d+$)", "", peername[0]) + address = re.sub(r"^::ffff:(?=\d+.\d+.\d+.\d+$)", "", peername[0]) port = peername[1] self.wfile.write(str((address, port)).encode()) diff --git a/test/mitmproxy/test_proxy.py b/test/mitmproxy/test_proxy.py index 00086c4b..c8cf6c33 100644 --- a/test/mitmproxy/test_proxy.py +++ b/test/mitmproxy/test_proxy.py @@ -1,4 +1,5 @@ import argparse +import platform from unittest import mock import pytest @@ -52,8 +53,11 @@ class TestProcessProxyOptions: class TestProxyServer: @skip_windows + @pytest.mark.skipif(platform.mac_ver()[0].split('.')[:2] == ['10', '14'], + reason='Skipping due to macOS Mojave') def test_err(self): - # binding to 0.0.0.0:1 works without special permissions on Windows + # binding to 0.0.0.0:1 works without special permissions on Windows and + # macOS Mojave conf = ProxyConfig(options.Options(listen_port=1)) with pytest.raises(Exception, match="Error starting proxy server"): ProxyServer(conf) @@ -44,7 +44,7 @@ commands = passenv = TRAVIS_* APPVEYOR_* AWS_* TWINE_* DOCKER_* RTOOL_KEY WHEEL DOCKER PYINSTALLER WININSTALLER deps = -rrequirements.txt - pyinstaller==3.4 + pyinstaller==3.5 twine==1.12.1 awscli commands = diff --git a/web/src/js/ducks/ui/keyboard.js b/web/src/js/ducks/ui/keyboard.js index ed4dbba5..007d24db 100644 --- a/web/src/js/ducks/ui/keyboard.js +++ b/web/src/js/ducks/ui/keyboard.js @@ -6,7 +6,7 @@ import * as modalActions from "./modal" export function onKeyDown(e) { //console.debug("onKeyDown", e) - if (e.ctrlKey) { + if (e.ctrlKey || e.metaKey) { return () => { } } |