diff options
| -rw-r--r-- | .travis.yml | 1 | ||||
| -rw-r--r-- | examples/complex/har_dump.py | 5 | ||||
| -rw-r--r-- | mitmproxy/addons/core.py | 2 | ||||
| -rw-r--r-- | mitmproxy/contentviews/base.py | 2 | ||||
| -rw-r--r-- | mitmproxy/contentviews/image/image_parser.py | 2 | ||||
| -rw-r--r-- | mitmproxy/net/http/encoding.py | 24 | ||||
| -rw-r--r-- | mitmproxy/net/http/message.py | 2 | ||||
| -rw-r--r-- | mitmproxy/net/http/request.py | 2 | ||||
| -rw-r--r-- | setup.py | 3 | ||||
| -rw-r--r-- | test/mitmproxy/addons/test_view.py | 2 | ||||
| -rw-r--r-- | test/mitmproxy/net/http/test_encoding.py | 1 | ||||
| -rw-r--r-- | test/mitmproxy/script/test_concurrent.py | 26 |
12 files changed, 48 insertions, 24 deletions
diff --git a/.travis.yml b/.travis.yml index 20afc279..44d452c2 100644 --- a/.travis.yml +++ b/.travis.yml @@ -52,7 +52,6 @@ matrix: - wget https://github.com/gohugoio/hugo/releases/download/v0.41/hugo_0.41_Linux-64bit.deb - sudo dpkg -i hugo*.deb - pip install tox virtualenv setuptools - - pyenv global system 3.6 script: - tox after_success: diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index 33a2f79f..414b4f61 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -87,7 +87,10 @@ def response(flow): } # HAR timings are integers in ms, so we re-encode the raw timings to that format. - timings = dict([(k, int(1000 * v)) for k, v in timings_raw.items()]) + timings = { + k: int(1000 * v) if v != -1 else -1 + for k, v in timings_raw.items() + } # full_time is the sum of all timings. # Timings set to -1 will be ignored as per spec. diff --git a/mitmproxy/addons/core.py b/mitmproxy/addons/core.py index a908dbb3..5c9bbcd0 100644 --- a/mitmproxy/addons/core.py +++ b/mitmproxy/addons/core.py @@ -289,7 +289,7 @@ class Core: """ The possible values for an encoding specification. """ - return ["gzip", "deflate", "br"] + return ["gzip", "deflate", "br", "zstd"] @command.command("options.load") def options_load(self, path: mitmproxy.types.Path) -> None: diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index 6072dfb7..9b34f3d4 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -37,7 +37,7 @@ class View: def format_pairs( items: typing.Iterable[typing.Tuple[TTextType, TTextType]] -)-> typing.Iterator[TViewLine]: +) -> typing.Iterator[TViewLine]: """ Helper function that accepts a list of (k,v) pairs into a list of diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py index fcc50cb5..d5bb404f 100644 --- a/mitmproxy/contentviews/image/image_parser.py +++ b/mitmproxy/contentviews/image/image_parser.py @@ -54,7 +54,7 @@ def parse_gif(data: bytes) -> Metadata: entries = block.body.body.entries for entry in entries: comment = entry.bytes - if comment is not b'': + if comment != b'': parts.append(('comment', str(comment))) return parts diff --git a/mitmproxy/net/http/encoding.py b/mitmproxy/net/http/encoding.py index 8cb96e5c..16d399ca 100644 --- a/mitmproxy/net/http/encoding.py +++ b/mitmproxy/net/http/encoding.py @@ -9,6 +9,7 @@ from io import BytesIO import gzip import zlib import brotli +import zstandard as zstd from typing import Union, Optional, AnyStr # noqa @@ -52,7 +53,7 @@ def decode( decoded = custom_decode[encoding](encoded) except KeyError: decoded = codecs.decode(encoded, encoding, errors) - if encoding in ("gzip", "deflate", "br"): + if encoding in ("gzip", "deflate", "br", "zstd"): _cache = CachedDecode(encoded, encoding, errors, decoded) return decoded except TypeError: @@ -93,7 +94,7 @@ def encode(decoded: Optional[str], encoding: str, errors: str='strict') -> Optio encoded = custom_encode[encoding](decoded) except KeyError: encoded = codecs.encode(decoded, encoding, errors) - if encoding in ("gzip", "deflate", "br"): + if encoding in ("gzip", "deflate", "br", "zstd"): _cache = CachedDecode(encoded, encoding, errors, decoded) return encoded except TypeError: @@ -140,6 +141,23 @@ def encode_brotli(content: bytes) -> bytes: return brotli.compress(content) +def decode_zstd(content: bytes) -> bytes: + if not content: + return b"" + zstd_ctx = zstd.ZstdDecompressor() + try: + return zstd_ctx.decompress(content) + except zstd.ZstdError: + # If the zstd stream is streamed without a size header, + # try decoding with a 10MiB output buffer + return zstd_ctx.decompress(content, max_output_size=10 * 2**20) + + +def encode_zstd(content: bytes) -> bytes: + zstd_ctx = zstd.ZstdCompressor() + return zstd_ctx.compress(content) + + def decode_deflate(content: bytes) -> bytes: """ Returns decompressed data for DEFLATE. Some servers may respond with @@ -170,6 +188,7 @@ custom_decode = { "gzip": decode_gzip, "deflate": decode_deflate, "br": decode_brotli, + "zstd": decode_zstd, } custom_encode = { "none": identity, @@ -177,6 +196,7 @@ custom_encode = { "gzip": encode_gzip, "deflate": encode_deflate, "br": encode_brotli, + "zstd": encode_zstd, } __all__ = ["encode", "decode"] diff --git a/mitmproxy/net/http/message.py b/mitmproxy/net/http/message.py index 86782e8a..6830c6cd 100644 --- a/mitmproxy/net/http/message.py +++ b/mitmproxy/net/http/message.py @@ -236,7 +236,7 @@ class Message(serializable.Serializable): def encode(self, e): """ - Encodes body with the encoding e, where e is "gzip", "deflate", "identity", or "br". + Encodes body with the encoding e, where e is "gzip", "deflate", "identity", "br", or "zstd". Any existing content-encodings are overwritten, the content is not decoded beforehand. diff --git a/mitmproxy/net/http/request.py b/mitmproxy/net/http/request.py index 6ca3973f..ef33ca49 100644 --- a/mitmproxy/net/http/request.py +++ b/mitmproxy/net/http/request.py @@ -423,7 +423,7 @@ class Request(message.Message): self.headers["accept-encoding"] = ( ', '.join( e - for e in {"gzip", "identity", "deflate", "br"} + for e in {"gzip", "identity", "deflate", "br", "zstd"} if e in accept_encoding ) ) @@ -82,6 +82,7 @@ setup( "urwid>=2.0.1,<2.1", "wsproto>=0.13.0,<0.14.0", "publicsuffix2~=2.20" + "zstandard>=0.11.0,<0.13.0", ], extras_require={ ':sys_platform == "win32"': [ @@ -89,7 +90,7 @@ setup( ], 'dev': [ "asynctest>=0.12.0", - "flake8>=3.5,<3.7", + "flake8>=3.5,<=3.7.8", "Flask>=1.0,<1.1", "mypy>=0.590,<0.591", "parver>=0.1,<2.0", diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index 976c14b7..f5088a68 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -471,7 +471,7 @@ def test_focus(): v = view.View() v.add([tft()]) f = view.Focus(v) - assert f.index is 0 + assert f.index == 0 assert f.flow is v[0] # Start empty diff --git a/test/mitmproxy/net/http/test_encoding.py b/test/mitmproxy/net/http/test_encoding.py index 8dac12cb..7f768f39 100644 --- a/test/mitmproxy/net/http/test_encoding.py +++ b/test/mitmproxy/net/http/test_encoding.py @@ -19,6 +19,7 @@ def test_identity(encoder): 'gzip', 'br', 'deflate', + 'zstd', ]) def test_encoders(encoder): """ diff --git a/test/mitmproxy/script/test_concurrent.py b/test/mitmproxy/script/test_concurrent.py index 3ec58760..70d41511 100644 --- a/test/mitmproxy/script/test_concurrent.py +++ b/test/mitmproxy/script/test_concurrent.py @@ -43,17 +43,17 @@ class TestConcurrent(tservers.MasterTest): assert await tctx.master.await_log("decorator not supported") def test_concurrent_class(self, tdata): - with taddons.context() as tctx: - sc = tctx.script( - tdata.path( - "mitmproxy/data/addonscripts/concurrent_decorator_class.py" - ) + with taddons.context() as tctx: + sc = tctx.script( + tdata.path( + "mitmproxy/data/addonscripts/concurrent_decorator_class.py" ) - f1, f2 = tflow.tflow(), tflow.tflow() - tctx.cycle(sc, f1) - tctx.cycle(sc, f2) - start = time.time() - while time.time() - start < 5: - if f1.reply.state == f2.reply.state == "committed": - return - raise ValueError("Script never acked") + ) + f1, f2 = tflow.tflow(), tflow.tflow() + tctx.cycle(sc, f1) + tctx.cycle(sc, f2) + start = time.time() + while time.time() - start < 5: + if f1.reply.state == f2.reply.state == "committed": + return + raise ValueError("Script never acked") |