diff options
27 files changed, 368 insertions, 101 deletions
@@ -11,6 +11,7 @@ MANIFEST .cache/ .tox*/ build/ +mitmproxy/contrib/kaitaistruct/*.ksy # UI @@ -21,3 +22,4 @@ sslkeylogfile.log .tox/ .python-version coverage.xml +web/coverage/ @@ -62,7 +62,7 @@ Development Setup To get started hacking on mitmproxy, please follow the `advanced installation`_ steps to install mitmproxy from source, but stop right before running ``pip3 install mitmproxy``. Instead, do the following: -.. code-block:: text +.. code-block:: bash git clone https://github.com/mitmproxy/mitmproxy.git cd mitmproxy @@ -80,7 +80,7 @@ The main executables for the project - ``mitmdump``, ``mitmproxy``, virtualenv. After activating the virtualenv, they will be on your $PATH, and you can run them like any other command: -.. code-block:: text +.. code-block:: bash . venv/bin/activate # "venv\Scripts\activate" on Windows mitmdump --version @@ -91,13 +91,13 @@ Testing If you've followed the procedure above, you already have all the development requirements installed, and you can run the full test suite (including tests for code style and documentation) with tox_: -.. code-block:: text +.. code-block:: bash tox For speedier testing, we recommend you run `pytest`_ directly on individual test files or folders: -.. code-block:: text +.. code-block:: bash cd test/mitmproxy/addons pytest --cov mitmproxy.addons.anticache --looponfail test_anticache.py @@ -114,7 +114,7 @@ The mitmproxy documentation is build using Sphinx_, which is installed automatically if you set up a development environment as described above. After installation, you can render the documentation like this: -.. code-block:: text +.. code-block:: bash cd docs make clean @@ -136,7 +136,7 @@ This is automatically enforced on every PR. If we detect a linting error, the PR checks will fail and block merging. You can run our lint checks yourself with the following command: -.. code-block:: text +.. code-block:: bash tox -e lint diff --git a/docs/howmitmproxy.rst b/docs/howmitmproxy.rst index 133863e3..4f3c804e 100644 --- a/docs/howmitmproxy.rst +++ b/docs/howmitmproxy.rst @@ -43,7 +43,7 @@ client connects to the proxy and makes a request that looks like this: CONNECT example.com:443 HTTP/1.1 -A conventional proxy can neither view nor manipulate an TLS-encrypted data +A conventional proxy can neither view nor manipulate a TLS-encrypted data stream, so a CONNECT request simply asks the proxy to open a pipe between the client and server. The proxy here is just a facilitator - it blindly forwards data in both directions without knowing anything about the contents. The @@ -63,7 +63,7 @@ exactly this attack, by allowing a trusted third-party to cryptographically sign a server's certificates to verify that they are legit. If this signature doesn't match or is from a non-trusted party, a secure client will simply drop the connection and refuse to proceed. Despite the many shortcomings of the CA system -as it exists today, this is usually fatal to attempts to MITM an TLS connection +as it exists today, this is usually fatal to attempts to MITM a TLS connection for analysis. Our answer to this conundrum is to become a trusted Certificate Authority ourselves. Mitmproxy includes a full CA implementation that generates interception certificates on the fly. To get the client to trust these @@ -143,7 +143,7 @@ Lets put all of this together into the complete explicitly proxied HTTPS flow. 2. Mitmproxy responds with a ``200 Connection Established``, as if it has set up the CONNECT pipe. 3. The client believes it's talking to the remote server, and initiates the TLS connection. It uses SNI to indicate the hostname it is connecting to. -4. Mitmproxy connects to the server, and establishes an TLS connection using the SNI hostname +4. Mitmproxy connects to the server, and establishes a TLS connection using the SNI hostname indicated by the client. 5. The server responds with the matching certificate, which contains the CN and SAN values needed to generate the interception certificate. @@ -217,7 +217,7 @@ explicit HTTPS connections to establish the CN and SANs, and cope with SNI. destination was. 3. The client believes it's talking to the remote server, and initiates the TLS connection. It uses SNI to indicate the hostname it is connecting to. -4. Mitmproxy connects to the server, and establishes an TLS connection using the SNI hostname +4. Mitmproxy connects to the server, and establishes a TLS connection using the SNI hostname indicated by the client. 5. The server responds with the matching certificate, which contains the CN and SAN values needed to generate the interception certificate. diff --git a/mitmproxy/addons/dumper.py b/mitmproxy/addons/dumper.py index ca0d32d3..5fd8408f 100644 --- a/mitmproxy/addons/dumper.py +++ b/mitmproxy/addons/dumper.py @@ -102,7 +102,7 @@ class Dumper: if flow.client_conn: client = click.style( strutils.escape_control_characters( - repr(flow.client_conn.address) + human.format_address(flow.client_conn.address) ) ) elif flow.request.is_replay: diff --git a/mitmproxy/addons/termstatus.py b/mitmproxy/addons/termstatus.py index 7b05f409..951ddd3c 100644 --- a/mitmproxy/addons/termstatus.py +++ b/mitmproxy/addons/termstatus.py @@ -1,4 +1,5 @@ from mitmproxy import ctx +from mitmproxy.utils import human """ A tiny addon to print the proxy status to terminal. Eventually this could @@ -17,7 +18,7 @@ class TermStatus: def running(self): if self.server: ctx.log.info( - "Proxy server listening at http://{}:{}".format( - *ctx.master.server.address, + "Proxy server listening at http://{}".format( + human.format_address(ctx.master.server.address) ) ) diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py index 062fb38e..7c74669a 100644 --- a/mitmproxy/contentviews/image/image_parser.py +++ b/mitmproxy/contentviews/image/image_parser.py @@ -37,7 +37,7 @@ def parse_gif(data: bytes) -> Metadata: descriptor = img.logical_screen_descriptor parts = [ ('Format', 'Compuserve GIF'), - ('Version', "GIF{}".format(img.header.version.decode('ASCII'))), + ('Version', "GIF{}".format(img.hdr.version)), ('Size', "{} x {} px".format(descriptor.screen_width, descriptor.screen_height)), ('background', str(descriptor.bg_color_index)) ] diff --git a/mitmproxy/contrib/kaitaistruct/exif.py b/mitmproxy/contrib/kaitaistruct/exif.py index 6236a708..d99cceef 100644 --- a/mitmproxy/contrib/kaitaistruct/exif.py +++ b/mitmproxy/contrib/kaitaistruct/exif.py @@ -1,16 +1,20 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -# The source was exif.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/24e2d00048b8084ceec30a187a79cb87a79a48ba/image/exif.ksy import array import struct import zlib from enum import Enum +from pkg_resources import parse_version -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +if parse_version(ks_version) < parse_version('0.7'): + raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) + from .exif_le import ExifLe from .exif_be import ExifBe + class Exif(KaitaiStruct): def __init__(self, _io, _parent=None, _root=None): self._io = _io diff --git a/mitmproxy/contrib/kaitaistruct/exif_be.py b/mitmproxy/contrib/kaitaistruct/exif_be.py index 7980a9e8..8a6e7a2b 100644 --- a/mitmproxy/contrib/kaitaistruct/exif_be.py +++ b/mitmproxy/contrib/kaitaistruct/exif_be.py @@ -1,14 +1,17 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -# The source was exif_be.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/24e2d00048b8084ceec30a187a79cb87a79a48ba/image/exif_be.ksy import array import struct import zlib from enum import Enum +from pkg_resources import parse_version -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +if parse_version(ks_version) < parse_version('0.7'): + raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) + class ExifBe(KaitaiStruct): def __init__(self, _io, _parent=None, _root=None): self._io = _io @@ -569,3 +572,5 @@ class ExifBe(KaitaiStruct): self._m_ifd0 = self._root.Ifd(self._io, self, self._root) self._io.seek(_pos) return self._m_ifd0 if hasattr(self, '_m_ifd0') else None + + diff --git a/mitmproxy/contrib/kaitaistruct/exif_le.py b/mitmproxy/contrib/kaitaistruct/exif_le.py index 207b3beb..84e53a38 100644 --- a/mitmproxy/contrib/kaitaistruct/exif_le.py +++ b/mitmproxy/contrib/kaitaistruct/exif_le.py @@ -1,14 +1,17 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -# The source was exif_le.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/24e2d00048b8084ceec30a187a79cb87a79a48ba/image/exif_le.ksy import array import struct import zlib from enum import Enum +from pkg_resources import parse_version -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +if parse_version(ks_version) < parse_version('0.7'): + raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) + class ExifLe(KaitaiStruct): def __init__(self, _io, _parent=None, _root=None): self._io = _io @@ -569,3 +572,5 @@ class ExifLe(KaitaiStruct): self._m_ifd0 = self._root.Ifd(self._io, self, self._root) self._io.seek(_pos) return self._m_ifd0 if hasattr(self, '_m_ifd0') else None + + diff --git a/mitmproxy/contrib/kaitaistruct/gif.py b/mitmproxy/contrib/kaitaistruct/gif.py index 61499cc7..820df568 100644 --- a/mitmproxy/contrib/kaitaistruct/gif.py +++ b/mitmproxy/contrib/kaitaistruct/gif.py @@ -1,14 +1,17 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -# The source was png.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/562154250bea0081fed4e232751b934bc270a0c7/image/gif.ksy import array import struct import zlib from enum import Enum +from pkg_resources import parse_version -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +if parse_version(ks_version) < parse_version('0.7'): + raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) + class Gif(KaitaiStruct): class BlockType(Enum): @@ -24,8 +27,8 @@ class Gif(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self - self.header = self._root.Header(self._io, self, self._root) - self.logical_screen_descriptor = self._root.LogicalScreenDescriptor(self._io, self, self._root) + self.hdr = self._root.Header(self._io, self, self._root) + self.logical_screen_descriptor = self._root.LogicalScreenDescriptorStruct(self._io, self, self._root) if self.logical_screen_descriptor.has_color_table: self._raw_global_color_table = self._io.read_bytes((self.logical_screen_descriptor.color_table_size * 3)) io = KaitaiStream(BytesIO(self._raw_global_color_table)) @@ -55,7 +58,7 @@ class Gif(KaitaiStruct): self.blue = self._io.read_u1() - class LogicalScreenDescriptor(KaitaiStruct): + class LogicalScreenDescriptorStruct(KaitaiStruct): def __init__(self, _io, _parent=None, _root=None): self._io = _io self._parent = _parent @@ -163,7 +166,7 @@ class Gif(KaitaiStruct): self._parent = _parent self._root = _root if _root else self self.magic = self._io.ensure_fixed_contents(struct.pack('3b', 71, 73, 70)) - self.version = self._io.read_bytes(3) + self.version = (self._io.read_bytes(3)).decode(u"ASCII") class ExtGraphicControl(KaitaiStruct): @@ -245,3 +248,6 @@ class Gif(KaitaiStruct): self.body = self._root.ExtGraphicControl(self._io, self, self._root) else: self.body = self._root.Subblocks(self._io, self, self._root) + + + diff --git a/mitmproxy/contrib/kaitaistruct/jpeg.py b/mitmproxy/contrib/kaitaistruct/jpeg.py index 08e382a9..33fc012f 100644 --- a/mitmproxy/contrib/kaitaistruct/jpeg.py +++ b/mitmproxy/contrib/kaitaistruct/jpeg.py @@ -1,15 +1,19 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -# The source was jpeg.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/24e2d00048b8084ceec30a187a79cb87a79a48ba/image/jpeg.ksy import array import struct import zlib from enum import Enum +from pkg_resources import parse_version -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +if parse_version(ks_version) < parse_version('0.7'): + raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) + from .exif import Exif + class Jpeg(KaitaiStruct): class ComponentId(Enum): @@ -127,7 +131,7 @@ class Jpeg(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self - self.magic = self._io.read_strz("ASCII", 0, False, True, True) + self.magic = (self._io.read_bytes_term(0, False, True, True)).decode(u"ASCII") _on = self.magic if _on == u"Exif": self.body = self._root.ExifInJpeg(self._io, self, self._root) @@ -195,7 +199,7 @@ class Jpeg(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self - self.magic = self._io.read_str_byte_limit(5, "ASCII") + self.magic = (self._io.read_bytes(5)).decode(u"ASCII") self.version_major = self._io.read_u1() self.version_minor = self._io.read_u1() self.density_units = self._root.SegmentApp0.DensityUnit(self._io.read_u1()) diff --git a/mitmproxy/contrib/kaitaistruct/make.sh b/mitmproxy/contrib/kaitaistruct/make.sh new file mode 100755 index 00000000..218d5198 --- /dev/null +++ b/mitmproxy/contrib/kaitaistruct/make.sh @@ -0,0 +1,10 @@ +#!/usr/bin/env bash + +wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/exif_be.ksy +wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/exif_le.ksy +wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/exif.ksy +wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/gif.ksy +wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/jpeg.ksy +wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/png.ksy + +kaitai-struct-compiler --target python --opaque-types=true *.ksy diff --git a/mitmproxy/contrib/kaitaistruct/png.py b/mitmproxy/contrib/kaitaistruct/png.py index 2f3c1a5c..98a70693 100644 --- a/mitmproxy/contrib/kaitaistruct/png.py +++ b/mitmproxy/contrib/kaitaistruct/png.py @@ -1,14 +1,17 @@ # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild -# The source was png.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/9370c720b7d2ad329102d89bdc880ba6a706ef26/image/png.ksy import array import struct import zlib from enum import Enum +from pkg_resources import parse_version -from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO +from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO +if parse_version(ks_version) < parse_version('0.7'): + raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version)) + class Png(KaitaiStruct): class ColorType(Enum): @@ -51,7 +54,7 @@ class Png(KaitaiStruct): self._parent = _parent self._root = _root if _root else self self.len = self._io.read_u4be() - self.type = self._io.read_str_byte_limit(4, "UTF-8") + self.type = (self._io.read_bytes(4)).decode(u"UTF-8") _on = self.type if _on == u"iTXt": self._raw_body = self._io.read_bytes(self.len) @@ -194,7 +197,7 @@ class Png(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self - self.keyword = self._io.read_strz("UTF-8", 0, False, True, True) + self.keyword = (self._io.read_bytes_term(0, False, True, True)).decode(u"UTF-8") self.compression_method = self._io.read_u1() self._raw_text_datastream = self._io.read_bytes_full() self.text_datastream = zlib.decompress(self._raw_text_datastream) @@ -259,12 +262,12 @@ class Png(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self - self.keyword = self._io.read_strz("UTF-8", 0, False, True, True) + self.keyword = (self._io.read_bytes_term(0, False, True, True)).decode(u"UTF-8") self.compression_flag = self._io.read_u1() self.compression_method = self._io.read_u1() - self.language_tag = self._io.read_strz("ASCII", 0, False, True, True) - self.translated_keyword = self._io.read_strz("UTF-8", 0, False, True, True) - self.text = self._io.read_str_eos("UTF-8") + self.language_tag = (self._io.read_bytes_term(0, False, True, True)).decode(u"ASCII") + self.translated_keyword = (self._io.read_bytes_term(0, False, True, True)).decode(u"UTF-8") + self.text = (self._io.read_bytes_full()).decode(u"UTF-8") class TextChunk(KaitaiStruct): @@ -272,8 +275,8 @@ class Png(KaitaiStruct): self._io = _io self._parent = _parent self._root = _root if _root else self - self.keyword = self._io.read_strz("iso8859-1", 0, False, True, True) - self.text = self._io.read_str_eos("iso8859-1") + self.keyword = (self._io.read_bytes_term(0, False, True, True)).decode(u"iso8859-1") + self.text = (self._io.read_bytes_full()).decode(u"iso8859-1") class TimeChunk(KaitaiStruct): @@ -287,3 +290,6 @@ class Png(KaitaiStruct): self.hour = self._io.read_u1() self.minute = self._io.read_u1() self.second = self._io.read_u1() + + + diff --git a/mitmproxy/net/http/http1/read.py b/mitmproxy/net/http/http1/read.py index ef88fd6c..491135ac 100644 --- a/mitmproxy/net/http/http1/read.py +++ b/mitmproxy/net/http/http1/read.py @@ -271,7 +271,9 @@ def _parse_authority_form(hostport): ValueError, if the input is malformed """ try: - host, port = hostport.split(b":") + host, port = hostport.rsplit(b":", 1) + if host.startswith(b"[") and host.endswith(b"]"): + host = host[1:-1] port = int(port) if not check.is_valid_host(host) or not check.is_valid_port(port): raise ValueError() diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 9f783bc3..50a2b76b 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -12,6 +12,7 @@ from mitmproxy.proxy import modes from mitmproxy.proxy import root_context from mitmproxy.net import tcp from mitmproxy.net.http import http1 +from mitmproxy.utils import human class DummyServer: @@ -152,5 +153,5 @@ class ConnectionHandler: self.client_conn.finish() def log(self, msg, level): - msg = "{}: {}".format(repr(self.client_conn.address), msg) + msg = "{}: {}".format(human.format_address(self.client_conn.address), msg) self.channel.tell("log", log.LogEntry(msg, level)) diff --git a/mitmproxy/tools/console/flowdetailview.py b/mitmproxy/tools/console/flowdetailview.py index 691f19a5..9ed063bc 100644 --- a/mitmproxy/tools/console/flowdetailview.py +++ b/mitmproxy/tools/console/flowdetailview.py @@ -30,9 +30,10 @@ def flowdetails(state, flow: http.HTTPFlow): if sc is not None: text.append(urwid.Text([("head", "Server Connection:")])) parts = [ - ["Address", "{}:{}".format(sc.address[0], sc.address[1])], - ["Resolved Address", "{}:{}".format(sc.ip_address[0], sc.ip_address[1])], + ["Address", human.format_address(sc.address)], ] + if sc.ip_address: + parts.append(["Resolved Address", human.format_address(sc.ip_address)]) if resp: parts.append(["HTTP Version", resp.http_version]) if sc.alpn_proto_negotiated: diff --git a/mitmproxy/tools/console/flowlist.py b/mitmproxy/tools/console/flowlist.py index 31d48ee3..044f8f05 100644 --- a/mitmproxy/tools/console/flowlist.py +++ b/mitmproxy/tools/console/flowlist.py @@ -57,6 +57,10 @@ class LogBufferBox(urwid.ListBox): self.master = master urwid.ListBox.__init__(self, master.logbuffer) + def set_focus(self, index): + if 0 <= index < len(self.master.logbuffer): + super().set_focus(index) + def keypress(self, size, key): key = common.shortcuts(key) if key == "z": diff --git a/mitmproxy/tools/web/static/vendor.js b/mitmproxy/tools/web/static/vendor.js index e83b8ba4..89f0d20c 100644 --- a/mitmproxy/tools/web/static/vendor.js +++ b/mitmproxy/tools/web/static/vendor.js @@ -3208,7 +3208,7 @@ function updateWidgetHeight(line) { } // Compute the lines that are visible in a given viewport (defaults -// the the current scroll position). viewport may contain top, +// to the current scroll position). viewport may contain top, // height, and ensure (see op.scrollToPos) properties. function visibleLines(display, doc, viewport) { var top = viewport && viewport.top != null ? Math.max(0, viewport.top) : display.scroller.scrollTop diff --git a/mitmproxy/utils/human.py b/mitmproxy/utils/human.py index 72e96d30..b3934846 100644 --- a/mitmproxy/utils/human.py +++ b/mitmproxy/utils/human.py @@ -1,7 +1,7 @@ import datetime +import ipaddress import time - SIZE_TABLE = [ ("b", 1024 ** 0), ("k", 1024 ** 1), @@ -62,3 +62,20 @@ def format_timestamp(s): def format_timestamp_with_milli(s): d = datetime.datetime.fromtimestamp(s) return d.strftime("%Y-%m-%d %H:%M:%S.%f")[:-3] + + +def format_address(address: tuple) -> str: + """ + This function accepts IPv4/IPv6 tuples and + returns the formatted address string with port number + """ + try: + host = ipaddress.ip_address(address[0]) + if host.version == 4: + return "{}:{}".format(str(host), address[1]) + # If IPv6 is mapped to IPv4 + elif host.ipv4_mapped: + return "{}:{}".format(str(host.ipv4_mapped), address[1]) + return "[{}]:{}".format(str(host), address[1]) + except ValueError: + return "{}:{}".format(address[0], address[1]) diff --git a/mitmproxy/version.py b/mitmproxy/version.py index 006ec868..3cae2a04 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -4,7 +4,7 @@ PATHOD = "pathod " + VERSION MITMPROXY = "mitmproxy " + VERSION # Serialization format version. This is displayed nowhere, it just needs to be incremented by one -# for each change the the file format. +# for each change in the file format. FLOW_FORMAT_VERSION = 5 if __name__ == "__main__": @@ -70,15 +70,15 @@ setup( "html2text>=2016.1.8, <=2016.9.19", "hyperframe>=5.0, <6", "jsbeautifier>=1.6.3, <1.7", - "kaitaistruct>=0.6, <0.7", + "kaitaistruct>=0.7, <0.8", "passlib>=1.6.5, <1.8", "pyasn1>=0.1.9, <0.3", - "pyOpenSSL>=16.0, <17.0", + "pyOpenSSL>=16.0, <17.1", "pyparsing>=2.1.3, <2.3", "pyperclip>=1.5.22, <1.6", "requests>=2.9.1, <3", "ruamel.yaml>=0.13.2, <0.15", - "tornado>=4.3, <4.5", + "tornado>=4.3, <4.6", "urwid>=1.3.1, <1.4", "watchdog>=0.8.3, <0.9", "brotlipy>=0.5.1, <0.7", @@ -113,7 +113,7 @@ setup( ], 'examples': [ "beautifulsoup4>=4.4.1, <4.6", - "Pillow>=3.2, <4.1", + "Pillow>=3.2, <4.2", ] } ) diff --git a/test/mitmproxy/console/test_flowlist.py b/test/mitmproxy/console/test_flowlist.py index 7c442b63..d63dab1c 100644 --- a/test/mitmproxy/console/test_flowlist.py +++ b/test/mitmproxy/console/test_flowlist.py @@ -1,4 +1,5 @@ from unittest import mock +import urwid import mitmproxy.tools.console.flowlist as flowlist from mitmproxy.tools import console @@ -19,3 +20,18 @@ class TestFlowlist: with mock.patch('mitmproxy.tools.console.signals.status_message.send') as mock_thing: x.new_request("nonexistent url", "GET") mock_thing.assert_called_once_with(message="Invalid URL: No hostname given") + + def test_logbuffer_set_focus(self): + m = self.mkmaster() + b = flowlist.LogBufferBox(m) + e = urwid.Text("Log message") + m.logbuffer.append(e) + m.logbuffer.append(e) + + assert len(m.logbuffer) == 2 + b.set_focus(0) + assert m.logbuffer.focus == 0 + b.set_focus(1) + assert m.logbuffer.focus == 1 + b.set_focus(2) + assert m.logbuffer.focus == 1 diff --git a/test/mitmproxy/net/http/http1/test_read.py b/test/mitmproxy/net/http/http1/test_read.py index 642b91c0..b3589c92 100644 --- a/test/mitmproxy/net/http/http1/test_read.py +++ b/test/mitmproxy/net/http/http1/test_read.py @@ -243,6 +243,7 @@ def test_read_request_line(): def test_parse_authority_form(): assert _parse_authority_form(b"foo:42") == (b"foo", 42) + assert _parse_authority_form(b"[2001:db8:42::]:443") == (b"2001:db8:42::", 443) with pytest.raises(exceptions.HttpSyntaxException): _parse_authority_form(b"foo") with pytest.raises(exceptions.HttpSyntaxException): diff --git a/test/mitmproxy/utils/test_human.py b/test/mitmproxy/utils/test_human.py index 3d65dfd1..76dc2f88 100644 --- a/test/mitmproxy/utils/test_human.py +++ b/test/mitmproxy/utils/test_human.py @@ -46,3 +46,10 @@ def test_pretty_duration(): assert human.pretty_duration(10000) == "10000s" assert human.pretty_duration(1.123) == "1.12s" assert human.pretty_duration(0.123) == "123ms" + + +def test_format_address(): + assert human.format_address(("::1", "54010", "0", "0")) == "[::1]:54010" + assert human.format_address(("::ffff:127.0.0.1", "54010", "0", "0")) == "127.0.0.1:54010" + assert human.format_address(("127.0.0.1", "54010")) == "127.0.0.1:54010" + assert human.format_address(("example.com", "54010")) == "example.com:54010" diff --git a/web/src/js/__tests__/ducks/flowsSpec.js b/web/src/js/__tests__/ducks/flowsSpec.js index 185e12bb..868e0621 100644 --- a/web/src/js/__tests__/ducks/flowsSpec.js +++ b/web/src/js/__tests__/ducks/flowsSpec.js @@ -1,32 +1,209 @@ jest.unmock('../../ducks/flows'); jest.mock('../../utils') -import reduceFlows, * as flowActions from '../../ducks/flows' -import reduceStore from '../../ducks/utils/store' - -describe('select flow', () => { - - let state = reduceFlows(undefined, {}) +import reduceFlows from "../../ducks/flows" +import * as flowActions from "../../ducks/flows" +import reduceStore from "../../ducks/utils/store" +import {fetchApi} from "../../utils" +import {createStore} from "./tutils" + +describe('flow reducer', () => { + let state = undefined for (let i of [1, 2, 3, 4]) { - state = reduceFlows(state, {type: flowActions.ADD, data: {id: i}, cmd: 'add'}) + state = reduceFlows(state, { type: flowActions.ADD, data: { id: i }, cmd: 'add' }) } - it('should be possible to select a single flow', () => { - expect(reduceFlows(state, flowActions.select(2))).toEqual( - { - ...state, - selected: [2], - } - ) - }) - - it('should be possible to deselect a flow', () => { - expect(reduceFlows({ ...state, selected: [1] }, flowActions.select())).toEqual( - { - ...state, - selected: [], - } - ) + it('should return initial state', () => { + expect(reduceFlows(undefined, {})).toEqual({ + highlight: null, + filter: null, + sort: { column: null, desc: false }, + selected: [], + ...reduceStore(undefined, {}) + }) + }) + + describe('selections', () => { + it('should be possible to select a single flow', () => { + expect(reduceFlows(state, flowActions.select(2))).toEqual( + { + ...state, + selected: [2], + } + ) + }) + + it('should be possible to deselect a flow', () => { + expect(reduceFlows({ ...state, selected: [1] }, flowActions.select())).toEqual( + { + ...state, + selected: [], + } + ) + }) + + it('should be possible to select relative', () => { + // haven't selected any flow + expect( + flowActions.selectRelative(state, 1) + ).toEqual( + flowActions.select(4) + ) + + // already selected some flows + expect( + flowActions.selectRelative({ ...state, selected: [2] }, 1) + ).toEqual( + flowActions.select(3) + ) + }) + + it('should update state.selected on remove', () => { + let next + next = reduceFlows({ ...state, selected: [2] }, { + type: flowActions.REMOVE, + data: 2, + cmd: 'remove' + }) + expect(next.selected).toEqual([3]) + + //last row + next = reduceFlows({ ...state, selected: [4] }, { + type: flowActions.REMOVE, + data: 4, + cmd: 'remove' + }) + expect(next.selected).toEqual([3]) + + //multiple selection + next = reduceFlows({ ...state, selected: [2, 3, 4] }, { + type: flowActions.REMOVE, + data: 3, + cmd: 'remove' + }) + expect(next.selected).toEqual([2, 4]) + }) + }) + + it('should be possible to set filter', () => { + let filt = "~u 123" + expect(reduceFlows(undefined, flowActions.setFilter(filt)).filter).toEqual(filt) + }) + + it('should be possible to set highlight', () => { + let key = "foo" + expect(reduceFlows(undefined, flowActions.setHighlight(key)).highlight).toEqual(key) + }) + + it('should be possible to set sort', () => { + let sort = { column: "TLSColumn", desc: 1 } + expect(reduceFlows(undefined, flowActions.setSort(sort.column, sort.desc)).sort).toEqual(sort) + }) + +}) + +describe('flows actions', () => { + + let store = createStore({reduceFlows}) + + let tflow = { id: 1 } + it('should handle resume action', () => { + store.dispatch(flowActions.resume(tflow)) + expect(fetchApi).toBeCalledWith('/flows/1/resume', { method: 'POST' }) + }) + + it('should handle resumeAll action', () => { + flowActions.resumeAll()() + }) + + it('should handle kill action', () => { + flowActions.kill(tflow)() + }) + + it('should handle killAll action', () => { + flowActions.killAll()() + }) + + it('should handle remove action', () => { + flowActions.remove(tflow)() + }) + + it('should handle duplicate action', () => { + flowActions.duplicate(tflow)() + }) + + it('should handle replay action', () => { + flowActions.replay(tflow)() + }) + + it('should handle revert action', () => { + flowActions.revert(tflow)() + }) + + it('should handle update action', () => { + flowActions.update(tflow, "foo")() + }) + + it('should handle updateContent action', () => { + flowActions.uploadContent(tflow, "foo", "foo")() + }) + + it('should handle clear action', () => { + flowActions.clear()() + }) + + it('should handle download action', () => { + let state = reduceFlows(undefined, {}) + expect(reduceFlows(state, flowActions.download())).toEqual(state) + }) + + it('should handle upload action', () => { + flowActions.upload("foo")() + }) +}) + +describe('makeSort', () => { + it('should be possible to sort by TLSColumn', () => { + let sort = flowActions.makeSort({ column: 'TLSColumn', desc: true }), + a = { request: { scheme: 'http' } }, + b = { request: { scheme: 'https' } } + expect(sort(a, b)).toEqual(1) + }) + + it('should be possible to sort by PathColumn', () => { + let sort = flowActions.makeSort({ column: 'PathColumn', desc: true }), + a = { request: {} }, + b = { request: {} } + expect(sort(a, b)).toEqual(0) + + }) + + it('should be possible to sort by MethodColumn', () => { + let sort = flowActions.makeSort({ column: 'MethodColumn', desc: true }), + a = { request: { method: 'GET' } }, + b = { request: { method: 'POST' } } + expect(sort(b, a)).toEqual(-1) + }) + + it('should be possible to sort by StatusColumn', () => { + let sort = flowActions.makeSort({ column: 'StatusColumn', desc: false }), + a = { response: { status_code: 200 } }, + b = { response: { status_code: 404 } } + expect(sort(a, b)).toEqual(-1) + }) + + it('should be possible to sort by TimeColumn', () => { + let sort = flowActions.makeSort({ column: 'TimeColumn', desc: false }), + a = { response: { timestamp_end: 9 }, request: { timestamp_start: 8 } }, + b = { response: { timestamp_end: 10 }, request: { timestamp_start: 8 } } + expect(sort(b, a)).toEqual(1) + }) + + it('should be possible to sort by SizeColumn', () => { + let sort = flowActions.makeSort({ column: 'SizeColumn', desc: true }), + a = { request: { contentLength: 1 }, response: { contentLength: 1 } }, + b = { request: { contentLength: 1 } } + expect(sort(a, b)).toEqual(-1) }) it('should be possible to select relative',() => { diff --git a/web/src/js/ducks/flows.js b/web/src/js/ducks/flows.js index 92408891..d36bc247 100644 --- a/web/src/js/ducks/flows.js +++ b/web/src/js/ducks/flows.js @@ -1,5 +1,6 @@ import { fetchApi } from "../utils" -import reduceStore, * as storeActions from "./utils/store" +import reduceStore from "./utils/store" +import * as storeActions from "./utils/store" import Filt from "../filt/filt" import { RequestUtils } from "../flow/utils" @@ -29,8 +30,6 @@ export default function reduce(state = defaultState, action) { case UPDATE: case REMOVE: case RECEIVE: - // FIXME: Update state.selected on REMOVE: - // The selected flow may have been removed, we need to select the next one in the view. let storeAction = storeActions[action.cmd]( action.data, makeFilter(state.filter), @@ -152,22 +151,20 @@ export function setSort(column, desc) { return { type: SET_SORT, sort: { column, desc } } } -export function selectRelative(shift) { - return (dispatch, getState) => { - let currentSelectionIndex = getState().flows.viewIndex[getState().flows.selected[0]] - let minIndex = 0 - let maxIndex = getState().flows.view.length - 1 - let newIndex - if (currentSelectionIndex === undefined) { - newIndex = (shift < 0) ? minIndex : maxIndex - } else { - newIndex = currentSelectionIndex + shift - newIndex = window.Math.max(newIndex, minIndex) - newIndex = window.Math.min(newIndex, maxIndex) - } - let flow = getState().flows.view[newIndex] - dispatch(select(flow ? flow.id : undefined)) +export function selectRelative(flows, shift) { + let currentSelectionIndex = flows.viewIndex[flows.selected[0]] + let minIndex = 0 + let maxIndex = flows.view.length - 1 + let newIndex + if (currentSelectionIndex === undefined) { + newIndex = (shift < 0) ? minIndex : maxIndex + } else { + newIndex = currentSelectionIndex + shift + newIndex = window.Math.max(newIndex, minIndex) + newIndex = window.Math.min(newIndex, maxIndex) } + let flow = flows.view[newIndex] + return select(flow ? flow.id : undefined) } diff --git a/web/src/js/ducks/ui/keyboard.js b/web/src/js/ducks/ui/keyboard.js index 30fd76e1..0e3491fa 100644 --- a/web/src/js/ducks/ui/keyboard.js +++ b/web/src/js/ducks/ui/keyboard.js @@ -9,39 +9,40 @@ export function onKeyDown(e) { return () => { } } - var key = e.keyCode - var shiftKey = e.shiftKey + let key = e.keyCode, + shiftKey = e.shiftKey e.preventDefault() return (dispatch, getState) => { - const flow = getState().flows.byId[getState().flows.selected[0]] + const flows = getState().flows, + flow = flows.byId[getState().flows.selected[0]] switch (key) { case Key.K: case Key.UP: - dispatch(flowsActions.selectRelative(-1)) + dispatch(flowsActions.selectRelative(flows, -1)) break case Key.J: case Key.DOWN: - dispatch(flowsActions.selectRelative(+1)) + dispatch(flowsActions.selectRelative(flows, +1)) break case Key.SPACE: case Key.PAGE_DOWN: - dispatch(flowsActions.selectRelative(+10)) + dispatch(flowsActions.selectRelative(flows, +10)) break case Key.PAGE_UP: - dispatch(flowsActions.selectRelative(-10)) + dispatch(flowsActions.selectRelative(flows, -10)) break case Key.END: - dispatch(flowsActions.selectRelative(+1e10)) + dispatch(flowsActions.selectRelative(flows, +1e10)) break case Key.HOME: - dispatch(flowsActions.selectRelative(-1e10)) + dispatch(flowsActions.selectRelative(flows, -1e10)) break case Key.ESC: |