diff options
101 files changed, 1126 insertions, 2199 deletions
diff --git a/.appveyor.yml b/.appveyor.yml index 3ef985be..6891f1b3 100644 --- a/.appveyor.yml +++ b/.appveyor.yml @@ -79,6 +79,7 @@ deploy_script: ($Env:TOXENV -match "py35") -and (($Env:APPVEYOR_REPO_BRANCH -In ("master", "pyinstaller")) -or ($Env:APPVEYOR_REPO_TAG -match "true")) ) { + tox -e rtool -- decrypt release\known_hosts.enc release\known_hosts tox -e rtool -- upload-snapshot --bdist --wheel --installer } diff --git a/.travis.yml b/.travis.yml index a29d0c75..b7504097 100644 --- a/.travis.yml +++ b/.travis.yml @@ -73,6 +73,7 @@ after_success: - | if [[ $BDIST == "1" && $TRAVIS_PULL_REQUEST == "false" && ($TRAVIS_BRANCH == "pyinstaller" || $TRAVIS_BRANCH == "master" || -n $TRAVIS_TAG) ]] then + tox -e rtool -- decrypt release/known_hosts.enc release/known_hosts tox -e rtool -- upload-snapshot --bdist fi @@ -186,4 +186,4 @@ with the following command: .. _PEP8: https://www.python.org/dev/peps/pep-0008 .. _`Google Style Guide`: https://google.github.io/styleguide/pyguide.html .. _forums: https://discourse.mitmproxy.org/ -.. _`good first contributions`: https://github.com/mitmproxy/mitmproxy/issues?q=is%3Aissue+is%3Aopen+label%3Agood-first-contribution +.. _`good first contributions`: https://github.com/mitmproxy/mitmproxy/issues?q=is%3Aissue+is%3Aopen+label%3A%22help+wanted%22 diff --git a/docs/features/passthrough.rst b/docs/features/passthrough.rst index dbaf3506..91fcb9b6 100644 --- a/docs/features/passthrough.rst +++ b/docs/features/passthrough.rst @@ -38,7 +38,7 @@ There are two important quirks to consider: - **In transparent mode, the ignore pattern is matched against the IP and ClientHello SNI host.** While we usually infer the hostname from the Host header if the ``--host`` argument is passed to mitmproxy, we do not have access to this information before the SSL handshake. If the client uses SNI however, then we treat the SNI host as an ignore target. -- **In regular mode, explicit HTTP requests are never ignored.** [#explicithttp]_ The ignore pattern is +- **In regular and upstream proxy mode, explicit HTTP requests are never ignored.** [#explicithttp]_ The ignore pattern is applied on CONNECT requests, which initiate HTTPS or clear-text WebSocket connections. Tutorial diff --git a/examples/complex/dns_spoofing.py b/examples/complex/dns_spoofing.py index 632783a7..e28934ab 100644 --- a/examples/complex/dns_spoofing.py +++ b/examples/complex/dns_spoofing.py @@ -33,7 +33,7 @@ parse_host_header = re.compile(r"^(?P<host>[^:]+|\[.+\])(?::(?P<port>\d+))?$") class Rerouter: def request(self, flow): - if flow.client_conn.ssl_established: + if flow.client_conn.tls_established: flow.request.scheme = "https" sni = flow.client_conn.connection.get_servername() port = 443 diff --git a/examples/complex/har_dump.py b/examples/complex/har_dump.py index 21bcc341..66a81a7d 100644 --- a/examples/complex/har_dump.py +++ b/examples/complex/har_dump.py @@ -58,8 +58,8 @@ def response(flow): connect_time = (flow.server_conn.timestamp_tcp_setup - flow.server_conn.timestamp_start) - if flow.server_conn.timestamp_ssl_setup is not None: - ssl_time = (flow.server_conn.timestamp_ssl_setup - + if flow.server_conn.timestamp_tls_setup is not None: + ssl_time = (flow.server_conn.timestamp_tls_setup - flow.server_conn.timestamp_tcp_setup) SERVERS_SEEN.add(flow.server_conn) diff --git a/mitmproxy/addons/clientplayback.py b/mitmproxy/addons/clientplayback.py index bed06e82..2dd488b9 100644 --- a/mitmproxy/addons/clientplayback.py +++ b/mitmproxy/addons/clientplayback.py @@ -27,6 +27,7 @@ class ClientPlayback: Stop client replay. """ self.flows = [] + ctx.log.alert("Client replay stopped.") ctx.master.addons.trigger("update", []) @command.command("replay.client") @@ -34,7 +35,11 @@ class ClientPlayback: """ Replay requests from flows. """ + for f in flows: + if f.live: + raise exceptions.CommandError("Can't replay live flow.") self.flows = list(flows) + ctx.log.alert("Replaying %s flows." % len(self.flows)) ctx.master.addons.trigger("update", []) @command.command("replay.client.file") @@ -43,7 +48,9 @@ class ClientPlayback: flows = io.read_flows_from_paths([path]) except exceptions.FlowReadException as e: raise exceptions.CommandError(str(e)) + ctx.log.alert("Replaying %s flows." % len(self.flows)) self.flows = flows + ctx.master.addons.trigger("update", []) def configure(self, updated): if not self.configured and ctx.options.client_replay: diff --git a/mitmproxy/addons/cut.py b/mitmproxy/addons/cut.py index f4b560e8..d684b8c7 100644 --- a/mitmproxy/addons/cut.py +++ b/mitmproxy/addons/cut.py @@ -43,7 +43,7 @@ def extract(cut: str, f: flow.Flow) -> typing.Union[str, bytes]: return part elif isinstance(part, bool): return "true" if part else "false" - elif isinstance(part, certs.SSLCert): + elif isinstance(part, certs.Cert): return part.to_pem().decode("ascii") current = part return str(current or "") diff --git a/mitmproxy/addons/onboardingapp/static/images/favicon.ico b/mitmproxy/addons/onboardingapp/static/images/favicon.ico Binary files differnew file mode 100644 index 00000000..3c3b891c --- /dev/null +++ b/mitmproxy/addons/onboardingapp/static/images/favicon.ico diff --git a/mitmproxy/addons/onboardingapp/static/images/mitmproxy-long.png b/mitmproxy/addons/onboardingapp/static/images/mitmproxy-long.png Binary files differnew file mode 100644 index 00000000..f9397d1e --- /dev/null +++ b/mitmproxy/addons/onboardingapp/static/images/mitmproxy-long.png diff --git a/mitmproxy/addons/onboardingapp/static/mitmproxy.css b/mitmproxy/addons/onboardingapp/static/mitmproxy.css index b390976a..969bd62b 100644 --- a/mitmproxy/addons/onboardingapp/static/mitmproxy.css +++ b/mitmproxy/addons/onboardingapp/static/mitmproxy.css @@ -1,8 +1,6 @@ - #certbank div { text-align: center; - - + padding-top: 20px; } .fronttable { @@ -40,7 +38,6 @@ section { .masthead { padding: 50px 0 60px; text-align: center; - } .header { diff --git a/mitmproxy/addons/onboardingapp/templates/index.html b/mitmproxy/addons/onboardingapp/templates/index.html index fc6213ea..38aa27ed 100644 --- a/mitmproxy/addons/onboardingapp/templates/index.html +++ b/mitmproxy/addons/onboardingapp/templates/index.html @@ -4,59 +4,135 @@ <script> function changeTo(device) { if (device == "apple") { - var text = `<h3>Apple: How to install on macOS / OSX</h3> - <ul> - <li>Double-click the PEM file</li> - <li>The "Keychain Access" applications opens</li> - <li>Find the new certificate "mitmproxy" in the list</li> - <li>Double-click the "mitmproxy" entry</li> - <li>A dialog window openes up</li> - <li>Change "Secure Socket Layer (SSL)" to "Always Trust"</li> - <li>Close the dialog window (and enter your password if prompted)</li> - <li>For iOS version 10.3 or up, you need to make sure mitmproxy is enabled in<br> - Certificate Trust Settings, you can check it by going to<br> - Settings > General > About > Certificate Trust Settings</li> - <li>Done!</li> - </ul>`; + var text = `<div class = "container"> + <div> + <div class="col-md-4"> + <h3 class="text-center">How to install on macOS</h3> + <ul class="left"> + <li>Double-click the PEM file</li> + <li>The "Keychain Access" applications opens</li> + <li>Find the new certificate "mitmproxy" in the list</li> + <li>Double-click the "mitmproxy" entry</li> + <li>A dialog window openes up</li> + <li>Change "Secure Socket Layer (SSL)" to "Always Trust"</li> + <li>Close the dialog window (and enter your password if prompted)</li> + <li>Done!</li> + </ul> + </div> + <div class="col-md-4"> + <h3 class="text-center">How to install on browsers</h3> + <ul> + <li>Safari on macOS uses the macOS keychain. So installing our CA in the system is enough.</li> + <li>Chrome on macOS uses the macOS keychain. So installing our CA in the system is enough.</li> + <li>Firefox on macOS has its own CA store and needs to be installed with Firefox-specific instructions that can be found <a href="https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox">HERE</a> .</li> + </ul> + </div> + <div class="col-md-4"> + <h3 class="text-center">How to install on iOS v10.3</h3> + <ul> + <li>After certificate installation, open Settings</li> + <li>Navigate to General and then About</li> + <li>Select Certificate Trust Settings</li> + <li>Each root that has been installed via a profile will be listed below the heading Enable Full Trust For Root Certificates. Toggle mitmproxy on</li> + <li>Done!</li> + </div> + </div> + </div>`; } else if (device == "windows") { - var text = `<h3>Windows: How to install on Windows</h3> - <ul> - <li>Double-click the P12 file</li> - <li>Select Store Location for Current User and click Next</li> - <li>Click Next</li> - <li>Leave the Password column blank and click Next</li> - <li>Select Place all certificates in the following store</li> - <li>Click Browse and select Trusted Root Certification Authorities</li> - <li>Click Next and then click Finish</li> - <li>Click Yes if prompted for confirmation</li> - <li>Done!</li> - </ul>`; + var text = `<div class = "container"> + <div class="row"> + <div class="col-md-4"> + <h3 class="text-center">How to install on Windows</h3> + <ul> + <li>Double-click the P12 file</li> + <li>Select Store Location for Current User and click Next</li> + <li>Click Next</li> + <li>Leave the Password column blank and click Next</li> + <li>Select Place all certificates in the following store</li> + <li>Click Browse and select Trusted Root Certification Authorities</li> + <li>Click Next and then click Finish</li> + <li>Click Yes if prompted for confirmation</li> + <li>Done!</li> + </ul> + </div> + <div class="col-md-4"> + <h3 class="text-center">How to install on browsers</h3> + <ul> + <li>Edge and IE use the Windows CA store. So installing our CA in the system is enough.</li> + <li>Chrome on Windows uses the Windows CA store. So installing our CA in the system is enough.</li> + <li>Firefox on Windows has its own CA store and needs to be installed with Firefox-specific instructions that can be found <a href="https://wiki.mozilla.org/MozillaRootCertificate#Mozilla_Firefox">HERE</a> .</li> + </ul> + </div> + <div class="col-md-4"> + <h3 class="text-center">How to install on Windows (Automated)</h3> + <ul> + <li> >>> certutil.exe -importpfx Root mitmproxy-ca-cert.p12 </li> + <li> To know more click <a href="https://technet.microsoft.com/en-us/library/cc732443.aspx">HERE</a> </li> + </ul> + </div> + </div> + </div>`; } else if (device == "android") { - var text = `<h3>Android: How to install on Android</h3> - <ul> - <li>Open your device's Settings app</li> - <li>Under "Credential storage," tap Install from storage</li> - <li>Under "Open from," tap where you saved the certificate</li> - <li>Tap the file</li> - <li>If prompted, enter the key store password and tap OK</li> - <li>Type a name for the certificate</li> - <li>Pick VPN and apps</li> - <li>Tap OK</li> - <li>Done!</li> - </ul>`; + var text = `<div class = "container"> + <div class="row"> + <div class="col-md-4"> + <h3 class="text-center">How to install on Android</h3> + <ul> + <li>Open your device's Settings app</li> + <li>Under "Credential storage," tap Install from storage</li> + <li>Under "Open from," tap where you saved the certificate</li> + <li>Tap the file</li> + <li>If prompted, enter the key store password and tap OK</li> + <li>Type a name for the certificate</li> + <li>Pick VPN and apps</li> + <li>Tap OK</li> + <li>Done!</li> + </ul> + </div> + </div> + </div>`; } else if (device == "asterisk") { - var text = ""; + var text = `<div class = "container"> + <div class="row"> + <div class="col-md-4"> + <h3 class="text-center">How to install on Chrome on Debian/Ubuntu</h3> + <ul> + <li>Using Chrome, hit a page on your server via HTTPS and continue past the red warning page (assuming you haven't done this already)</li> + <li>Open up Chrome Settings > Show advanced settings > HTTPS/SSL > Manage Certificates</li> + <li>Click the Authorities tab and scroll down to find your certificate under the Organization Name that you gave to the certificate</li> + <li>Select it, click Edit (NOTE: in recent versions of Chrome, the button is now "Advanced" instead of "Edit"), check all the boxes and click OK. You may have to restart Chrome</li> + </ul> + </div> + <div class="col-md-4"> + <h3 class="text-center">How to install on Chrome on Linux</h3> + <ul> + <li>Open Developer Tools > Security, and select View certificate</li> + <li>Click the Details tab > Export. Choose PKCS #7, single certificate as the file format</li> + <li>Then follow my original instructions to get to the Manage Certificates page. Click the Authorities tab > Import and choose the file to which you exported the certificate, and make sure to choose PKCS #7, single certificate as the file type</li> + <li>If prompted certification store, choose Trusted Root Certificate Authorities</li> + <li>Check all boxes and click OK. Restart Chrome</li> + </ul> + </div> + <div class="col-md-4"> + <h3 class="text-center">How to install on Ubuntu (Manually)</h3> + <ul> + <li>Create a directory for extra CA certificates in /usr/share/ca-certificates: <div class="text-muted">$ sudo mkdir /usr/share/ca-certificates/extra<div></li> + <li>Copy the CA mitmproxy.crt file to this directory: <div class="text-muted">$ sudo cp mitmproxy.crt /usr/share/ca-certificates/extra/mitmproxy.crt<div></li> + <li>Let Ubuntu add the mitmproxy.crt file's path relative to /usr/share/ca-certificates to /etc/ca-certificates.conf: <div class="text-muted">$ sudo dpkg-reconfigure ca-certificates</div></li> + <li>In case of a .pem file on Ubuntu, it must first be converted to a .crt file: <div class="text-muted">$ openssl x509 -in foo.pem -inform PEM -out foo.crt</div></li> + </ul> + </div> + </div> + </div>`; } document.getElementById("dynamic").innerHTML = text; } </script> -<center> -<h2> Click to install your mitmproxy certificate: </h2> -</center> +<h2 class="text-center"> Click to install your mitmproxy certificate </h2> <div id="certbank" class="row"> <div class="col-md-3"> <a onclick="changeTo('apple')" href="/cert/pem"><i class="fa fa-apple fa-5x"></i></a> diff --git a/mitmproxy/addons/onboardingapp/templates/layout.html b/mitmproxy/addons/onboardingapp/templates/layout.html index 8726a788..f6e1b286 100644 --- a/mitmproxy/addons/onboardingapp/templates/layout.html +++ b/mitmproxy/addons/onboardingapp/templates/layout.html @@ -12,20 +12,23 @@ <link href="/static/bootstrap.min.css" rel="stylesheet"> <link href="/static/mitmproxy.css" rel="stylesheet"> <link href="/static/fontawesome/css/font-awesome.min.css" rel="stylesheet"> + <link rel="icon" href="/static/images/favicon.ico" type="image/x-icon"/> </head> <body> <div class="navbar navbar-default" role="navigation"> <div class="container"> <div class="navbar-header"> - <a class="navbar-brand" href="#">mitmproxy</a> + <a class="navbar-brand" href="#"> + <img height="20px" src="static/images/mitmproxy-long.png"/> + </a> </div> </div> </div> <div class="container"> - {% block content %} - {% end %} + {% block content %} + {% end %} </div> </body> diff --git a/mitmproxy/addons/proxyauth.py b/mitmproxy/addons/proxyauth.py index 64233e88..dc99d5cc 100644 --- a/mitmproxy/addons/proxyauth.py +++ b/mitmproxy/addons/proxyauth.py @@ -146,14 +146,14 @@ class ProxyAuth: ) elif ctx.options.proxyauth.startswith("ldap"): parts = ctx.options.proxyauth.split(':') - security = parts[0] - ldap_server = parts[1] - dn_baseauth = parts[2] - password_baseauth = parts[3] if len(parts) != 5: raise exceptions.OptionsError( "Invalid ldap specification" ) + security = parts[0] + ldap_server = parts[1] + dn_baseauth = parts[2] + password_baseauth = parts[3] if security == "ldaps": server = ldap3.Server(ldap_server, use_ssl=True) elif security == "ldap": diff --git a/mitmproxy/addons/termlog.py b/mitmproxy/addons/termlog.py index 3a9f2c19..2a7e2d09 100644 --- a/mitmproxy/addons/termlog.py +++ b/mitmproxy/addons/termlog.py @@ -24,7 +24,8 @@ class TermLog: click.secho( e.msg, file=outfile, - fg=dict(error="red", warn="yellow").get(e.level), + fg=dict(error="red", warn="yellow", + alert="magenta").get(e.level), dim=(e.level == "debug"), err=(e.level == "error") ) diff --git a/mitmproxy/addons/view.py b/mitmproxy/addons/view.py index 3a15fd3e..e87daf35 100644 --- a/mitmproxy/addons/view.py +++ b/mitmproxy/addons/view.py @@ -238,7 +238,7 @@ class View(collections.Sequence): @command.command("view.order.options") def order_options(self) -> typing.Sequence[str]: """ - Choices supported by the console_order option. + Choices supported by the view_order option. """ return list(sorted(self.orders.keys())) @@ -365,7 +365,8 @@ class View(collections.Sequence): self.add([i.copy()]) except IOError as e: ctx.log.error(e.strerror) - return + except exceptions.FlowReadException as e: + ctx.log.error(str(e)) @command.command("view.go") def go(self, dst: int) -> None: @@ -441,7 +442,10 @@ class View(collections.Sequence): @command.command("view.create") def create(self, method: str, url: str) -> None: - req = http.HTTPRequest.make(method.upper(), url) + try: + req = http.HTTPRequest.make(method.upper(), url) + except ValueError as e: + raise exceptions.CommandError("Invalid URL: %s" % e) c = connections.ClientConnection.make_dummy(("", 0)) s = connections.ServerConnection.make_dummy((req.host, req.port)) f = http.HTTPFlow(c, s) diff --git a/mitmproxy/certs.py b/mitmproxy/certs.py index c29d67f3..4e10529a 100644 --- a/mitmproxy/certs.py +++ b/mitmproxy/certs.py @@ -112,7 +112,7 @@ def dummy_cert(privkey, cacert, commonname, sans): [OpenSSL.crypto.X509Extension(b"subjectAltName", False, ss)]) cert.set_pubkey(cacert.get_pubkey()) cert.sign(privkey, "sha256") - return SSLCert(cert) + return Cert(cert) class CertStoreEntry: @@ -249,7 +249,7 @@ class CertStore: def add_cert_file(self, spec: str, path: str) -> None: with open(path, "rb") as f: raw = f.read() - cert = SSLCert( + cert = Cert( OpenSSL.crypto.load_certificate( OpenSSL.crypto.FILETYPE_PEM, raw)) @@ -345,7 +345,7 @@ class _GeneralNames(univ.SequenceOf): constraint.ValueSizeConstraint(1, 1024) -class SSLCert(serializable.Serializable): +class Cert(serializable.Serializable): def __init__(self, cert): """ @@ -436,7 +436,7 @@ class SSLCert(serializable.Serializable): Returns: All DNS altnames. """ - # tcp.TCPClient.convert_to_ssl assumes that this property only contains DNS altnames for hostname verification. + # tcp.TCPClient.convert_to_tls assumes that this property only contains DNS altnames for hostname verification. altnames = [] for i in range(self.x509.get_extension_count()): ext = self.x509.get_extension(i) diff --git a/mitmproxy/connections.py b/mitmproxy/connections.py index 01721a71..86565b7b 100644 --- a/mitmproxy/connections.py +++ b/mitmproxy/connections.py @@ -1,11 +1,13 @@ import time import os +import typing import uuid -from mitmproxy import stateobject +from mitmproxy import stateobject, exceptions from mitmproxy import certs from mitmproxy.net import tcp +from mitmproxy.net import tls from mitmproxy.utils import strutils @@ -16,16 +18,17 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): Attributes: address: Remote address - ssl_established: True if TLS is established, False otherwise + tls_established: True if TLS is established, False otherwise clientcert: The TLS client certificate mitmcert: The MITM'ed TLS server certificate presented to the client timestamp_start: Connection start timestamp - timestamp_ssl_setup: TLS established timestamp + timestamp_tls_setup: TLS established timestamp timestamp_end: Connection end timestamp sni: Server Name Indication sent by client during the TLS handshake cipher_name: The current used cipher alpn_proto_negotiated: The negotiated application protocol tls_version: TLS version + tls_extensions: TLS ClientHello extensions """ def __init__(self, client_connection, address, server): @@ -40,23 +43,24 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): self.rfile = None self.address = None self.clientcert = None - self.ssl_established = None + self.tls_established = None self.id = str(uuid.uuid4()) self.mitmcert = None self.timestamp_start = time.time() self.timestamp_end = None - self.timestamp_ssl_setup = None + self.timestamp_tls_setup = None self.sni = None self.cipher_name = None self.alpn_proto_negotiated = None self.tls_version = None + self.tls_extensions = None def connected(self): return bool(self.connection) and not self.finished def __repr__(self): - if self.ssl_established: + if self.tls_established: tls = "[{}] ".format(self.tls_version) else: tls = "" @@ -83,27 +87,20 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): def __hash__(self): return hash(self.id) - @property - def tls_established(self): - return self.ssl_established - - @tls_established.setter - def tls_established(self, value): - self.ssl_established = value - _stateobject_attributes = dict( id=str, address=tuple, - ssl_established=bool, - clientcert=certs.SSLCert, - mitmcert=certs.SSLCert, + tls_established=bool, + clientcert=certs.Cert, + mitmcert=certs.Cert, timestamp_start=float, - timestamp_ssl_setup=float, + timestamp_tls_setup=float, timestamp_end=float, sni=str, cipher_name=str, alpn_proto_negotiated=bytes, tls_version=str, + tls_extensions=typing.List[typing.Tuple[int, bytes]], ) def send(self, message): @@ -125,19 +122,29 @@ class ClientConnection(tcp.BaseHandler, stateobject.StateObject): address=address, clientcert=None, mitmcert=None, - ssl_established=False, + tls_established=False, timestamp_start=None, timestamp_end=None, - timestamp_ssl_setup=None, + timestamp_tls_setup=None, sni=None, cipher_name=None, alpn_proto_negotiated=None, tls_version=None, + tls_extensions=None, )) - def convert_to_ssl(self, cert, *args, **kwargs): - super().convert_to_ssl(cert, *args, **kwargs) - self.timestamp_ssl_setup = time.time() + def convert_to_tls(self, cert, *args, **kwargs): + # Unfortunately OpenSSL provides no way to expose all TLS extensions, so we do this dance + # here and use our Kaitai parser. + try: + client_hello = tls.ClientHello.from_file(self.rfile) + except exceptions.TlsProtocolException: # pragma: no cover + pass # if this fails, we don't want everything to go down. + else: + self.tls_extensions = client_hello.extensions + + super().convert_to_tls(cert, *args, **kwargs) + self.timestamp_tls_setup = time.time() self.mitmcert = cert sni = self.connection.get_servername() if sni: @@ -162,7 +169,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): address: Remote address. Can be both a domain or an IP address. ip_address: Resolved remote IP address. source_address: Local IP address or client's source IP address. - ssl_established: True if TLS is established, False otherwise + tls_established: True if TLS is established, False otherwise cert: The certificate presented by the remote during the TLS handshake sni: Server Name Indication sent by the proxy during the TLS handshake alpn_proto_negotiated: The negotiated application protocol @@ -170,7 +177,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): via: The underlying server connection (e.g. the connection to the upstream proxy in upstream proxy mode) timestamp_start: Connection start timestamp timestamp_tcp_setup: TCP ACK received timestamp - timestamp_ssl_setup: TLS established timestamp + timestamp_tls_setup: TLS established timestamp timestamp_end: Connection end timestamp """ @@ -184,15 +191,15 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.timestamp_start = None self.timestamp_end = None self.timestamp_tcp_setup = None - self.timestamp_ssl_setup = None + self.timestamp_tls_setup = None def connected(self): return bool(self.connection) and not self.finished def __repr__(self): - if self.ssl_established and self.sni: + if self.tls_established and self.sni: tls = "[{}: {}] ".format(self.tls_version or "TLS", self.sni) - elif self.ssl_established: + elif self.tls_established: tls = "[{}] ".format(self.tls_version or "TLS") else: tls = "" @@ -217,27 +224,19 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): def __hash__(self): return hash(self.id) - @property - def tls_established(self): - return self.ssl_established - - @tls_established.setter - def tls_established(self, value): - self.ssl_established = value - _stateobject_attributes = dict( id=str, address=tuple, ip_address=tuple, source_address=tuple, - ssl_established=bool, - cert=certs.SSLCert, + tls_established=bool, + cert=certs.Cert, sni=str, alpn_proto_negotiated=bytes, tls_version=str, timestamp_start=float, timestamp_tcp_setup=float, - timestamp_ssl_setup=float, + timestamp_tls_setup=float, timestamp_end=float, ) @@ -258,10 +257,10 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): alpn_proto_negotiated=None, tls_version=None, source_address=('', 0), - ssl_established=False, + tls_established=False, timestamp_start=None, timestamp_tcp_setup=None, - timestamp_ssl_setup=None, + timestamp_tls_setup=None, timestamp_end=None, via=None )) @@ -277,7 +276,7 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): self.wfile.write(message) self.wfile.flush() - def establish_ssl(self, clientcerts, sni, **kwargs): + def establish_tls(self, clientcerts, sni, **kwargs): if sni and not isinstance(sni, str): raise ValueError("sni must be str, not " + type(sni).__name__) clientcert = None @@ -291,11 +290,11 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): if os.path.exists(path): clientcert = path - self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) + self.convert_to_tls(cert=clientcert, sni=sni, **kwargs) self.sni = sni self.alpn_proto_negotiated = self.get_alpn_proto_negotiated() self.tls_version = self.connection.get_protocol_version_name() - self.timestamp_ssl_setup = time.time() + self.timestamp_tls_setup = time.time() def finish(self): tcp.TCPClient.finish(self) diff --git a/mitmproxy/contentviews/base.py b/mitmproxy/contentviews/base.py index 97740eea..bdab1e99 100644 --- a/mitmproxy/contentviews/base.py +++ b/mitmproxy/contentviews/base.py @@ -43,9 +43,11 @@ def format_dict( ) -> typing.Iterator[TViewLine]: """ Helper function that transforms the given dictionary into a list of + [ ("key", key ) ("value", value) - tuples, where key is padded to a uniform width. + ] + entries, where key is padded to a uniform width. """ max_key_len = max(len(k) for k in d.keys()) max_key_len = min(max_key_len, KEY_MAX) diff --git a/mitmproxy/contrib/kaitaistruct/gif.py b/mitmproxy/contrib/kaitaistruct/gif.py index 820df568..76d7fc16 100644 --- a/mitmproxy/contrib/kaitaistruct/gif.py +++ b/mitmproxy/contrib/kaitaistruct/gif.py @@ -35,9 +35,11 @@ class Gif(KaitaiStruct): self.global_color_table = self._root.ColorTable(io, self, self._root) self.blocks = [] - while not self._io.is_eof(): - self.blocks.append(self._root.Block(self._io, self, self._root)) - + while True: + _ = self._root.Block(self._io, self, self._root) + self.blocks.append(_) + if ((self._io.is_eof()) or (_.block_type == self._root.BlockType.end_of_file)) : + break class ImageData(KaitaiStruct): def __init__(self, _io, _parent=None, _root=None): diff --git a/mitmproxy/contrib/wsproto/__init__.py b/mitmproxy/contrib/wsproto/__init__.py deleted file mode 100644 index d0592bc5..00000000 --- a/mitmproxy/contrib/wsproto/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from . import compat -from . import connection -from . import events -from . import extensions -from . import frame_protocol - -__all__ = [ - 'compat', - 'connection', - 'events', - 'extensions', - 'frame_protocol', -] diff --git a/mitmproxy/contrib/wsproto/compat.py b/mitmproxy/contrib/wsproto/compat.py deleted file mode 100644 index 1911f83c..00000000 --- a/mitmproxy/contrib/wsproto/compat.py +++ /dev/null @@ -1,20 +0,0 @@ -# flake8: noqa - -import sys - - -PY2 = sys.version_info.major == 2 -PY3 = sys.version_info.major == 3 - - -if PY3: - unicode = str - - def Utf8Validator(): - return None -else: - unicode = unicode - try: - from wsaccel.utf8validator import Utf8Validator - except ImportError: - from .utf8validator import Utf8Validator diff --git a/mitmproxy/contrib/wsproto/connection.py b/mitmproxy/contrib/wsproto/connection.py deleted file mode 100644 index f994cd3a..00000000 --- a/mitmproxy/contrib/wsproto/connection.py +++ /dev/null @@ -1,477 +0,0 @@ -# -*- coding: utf-8 -*- -""" -wsproto/connection -~~~~~~~~~~~~~~ - -An implementation of a WebSocket connection. -""" - -import os -import base64 -import hashlib -from collections import deque - -from enum import Enum - -import h11 - -from .events import ( - ConnectionRequested, ConnectionEstablished, ConnectionClosed, - ConnectionFailed, TextReceived, BytesReceived, PingReceived, PongReceived -) -from .frame_protocol import FrameProtocol, ParseFailed, CloseReason, Opcode - - -# RFC6455, Section 1.3 - Opening Handshake -ACCEPT_GUID = b"258EAFA5-E914-47DA-95CA-C5AB0DC85B11" - - -class ConnectionState(Enum): - """ - RFC 6455, Section 4 - Opening Handshake - """ - CONNECTING = 0 - OPEN = 1 - CLOSING = 2 - CLOSED = 3 - - -class ConnectionType(Enum): - CLIENT = 1 - SERVER = 2 - - -CLIENT = ConnectionType.CLIENT -SERVER = ConnectionType.SERVER - - -# Some convenience utilities for working with HTTP headers -def _normed_header_dict(h11_headers): - # This mangles Set-Cookie headers. But it happens that we don't care about - # any of those, so it's OK. For every other HTTP header, if there are - # multiple instances then you're allowed to join them together with - # commas. - name_to_values = {} - for name, value in h11_headers: - name_to_values.setdefault(name, []).append(value) - name_to_normed_value = {} - for name, values in name_to_values.items(): - name_to_normed_value[name] = b", ".join(values) - return name_to_normed_value - - -# We use this for parsing the proposed protocol list, and for parsing the -# proposed and accepted extension lists. For the proposed protocol list it's -# fine, because the ABNF is just 1#token. But for the extension lists, it's -# wrong, because those can contain quoted strings, which can in turn contain -# commas. XX FIXME -def _split_comma_header(value): - return [piece.decode('ascii').strip() for piece in value.split(b',')] - - -class WSConnection(object): - """ - A low-level WebSocket connection object. - - This wraps two other protocol objects, an HTTP/1.1 protocol object used - to do the initial HTTP upgrade handshake and a WebSocket frame protocol - object used to exchange messages and other control frames. - - :param conn_type: Whether this object is on the client- or server-side of - a connection. To initialise as a client pass ``CLIENT`` otherwise - pass ``SERVER``. - :type conn_type: ``ConnectionType`` - - :param host: The hostname to pass to the server when acting as a client. - :type host: ``str`` - - :param resource: The resource (aka path) to pass to the server when acting - as a client. - :type resource: ``str`` - - :param extensions: A list of extensions to use on this connection. - Extensions should be instances of a subclass of - :class:`Extension <wsproto.extensions.Extension>`. - - :param subprotocols: A list of subprotocols to request when acting as a - client, ordered by preference. This has no impact on the connection - itself. - :type subprotocol: ``list`` of ``str`` - """ - - def __init__(self, conn_type, host=None, resource=None, extensions=None, - subprotocols=None): - self.client = conn_type is ConnectionType.CLIENT - - self.host = host - self.resource = resource - - self.subprotocols = subprotocols or [] - self.extensions = extensions or [] - - self.version = b'13' - - self._state = ConnectionState.CONNECTING - self._close_reason = None - - self._nonce = None - self._outgoing = b'' - self._events = deque() - self._proto = None - - if self.client: - self._upgrade_connection = h11.Connection(h11.CLIENT) - else: - self._upgrade_connection = h11.Connection(h11.SERVER) - - if self.client: - if self.host is None: - raise ValueError( - "Host must not be None for a client-side connection.") - if self.resource is None: - raise ValueError( - "Resource must not be None for a client-side connection.") - self.initiate_connection() - - def initiate_connection(self): - self._generate_nonce() - - headers = { - b"Host": self.host.encode('ascii'), - b"Upgrade": b'WebSocket', - b"Connection": b'Upgrade', - b"Sec-WebSocket-Key": self._nonce, - b"Sec-WebSocket-Version": self.version, - } - - if self.subprotocols: - headers[b"Sec-WebSocket-Protocol"] = ", ".join(self.subprotocols) - - if self.extensions: - offers = {e.name: e.offer(self) for e in self.extensions} - extensions = [] - for name, params in offers.items(): - if params is True: - extensions.append(name.encode('ascii')) - elif params: - # py34 annoyance: doesn't support bytestring formatting - extensions.append(('%s; %s' % (name, params)) - .encode("ascii")) - if extensions: - headers[b'Sec-WebSocket-Extensions'] = b', '.join(extensions) - - upgrade = h11.Request(method=b'GET', target=self.resource, - headers=headers.items()) - self._outgoing += self._upgrade_connection.send(upgrade) - - def send_data(self, payload, final=True): - """ - Send a message or part of a message to the remote peer. - - If ``final`` is ``False`` it indicates that this is part of a longer - message. If ``final`` is ``True`` it indicates that this is either a - self-contained message or the last part of a longer message. - - If ``payload`` is of type ``bytes`` then the message is flagged as - being binary If it is of type ``str`` encoded as UTF-8 and sent as - text. - - :param payload: The message body to send. - :type payload: ``bytes`` or ``str`` - - :param final: Whether there are more parts to this message to be sent. - :type final: ``bool`` - """ - - self._outgoing += self._proto.send_data(payload, final) - - def close(self, code=CloseReason.NORMAL_CLOSURE, reason=None): - self._outgoing += self._proto.close(code, reason) - self._state = ConnectionState.CLOSING - - @property - def closed(self): - return self._state is ConnectionState.CLOSED - - def bytes_to_send(self, amount=None): - """ - Return any data that is to be sent to the remote peer. - - :param amount: (optional) The maximum number of bytes to be provided. - If ``None`` or not provided it will return all available bytes. - :type amount: ``int`` - """ - - if amount is None: - data = self._outgoing - self._outgoing = b'' - else: - data = self._outgoing[:amount] - self._outgoing = self._outgoing[amount:] - - return data - - def receive_bytes(self, data): - """ - Pass some received bytes to the connection for processing. - - :param data: The data received from the remote peer. - :type data: ``bytes`` - """ - - if data is None and self._state is ConnectionState.OPEN: - # "If _The WebSocket Connection is Closed_ and no Close control - # frame was received by the endpoint (such as could occur if the - # underlying transport connection is lost), _The WebSocket - # Connection Close Code_ is considered to be 1006." - self._events.append(ConnectionClosed(CloseReason.ABNORMAL_CLOSURE)) - self._state = ConnectionState.CLOSED - return - elif data is None: - self._state = ConnectionState.CLOSED - return - - if self._state is ConnectionState.CONNECTING: - event, data = self._process_upgrade(data) - if event is not None: - self._events.append(event) - - if self._state is ConnectionState.OPEN: - self._proto.receive_bytes(data) - - def _process_upgrade(self, data): - self._upgrade_connection.receive_data(data) - while True: - try: - event = self._upgrade_connection.next_event() - except h11.RemoteProtocolError: - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "Bad HTTP message"), b'' - if event is h11.NEED_DATA: - break - elif self.client and isinstance(event, (h11.InformationalResponse, - h11.Response)): - data = self._upgrade_connection.trailing_data[0] - return self._establish_client_connection(event), data - elif not self.client and isinstance(event, h11.Request): - return self._process_connection_request(event), None - else: - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "Bad HTTP message"), b'' - - self._incoming = b'' - return None, None - - def events(self): - """ - Return a generator that provides any events that have been generated - by protocol activity. - - :returns: generator - """ - - while self._events: - yield self._events.popleft() - - if self._proto is None: - return - - try: - for frame in self._proto.received_frames(): - if frame.opcode is Opcode.PING: - assert frame.frame_finished and frame.message_finished - self._outgoing += self._proto.pong(frame.payload) - yield PingReceived(frame.payload) - - elif frame.opcode is Opcode.PONG: - assert frame.frame_finished and frame.message_finished - yield PongReceived(frame.payload) - - elif frame.opcode is Opcode.CLOSE: - code, reason = frame.payload - self.close(code, reason) - yield ConnectionClosed(code, reason) - - elif frame.opcode is Opcode.TEXT: - yield TextReceived(frame.payload, - frame.frame_finished, - frame.message_finished) - - elif frame.opcode is Opcode.BINARY: - yield BytesReceived(frame.payload, - frame.frame_finished, - frame.message_finished) - except ParseFailed as exc: - # XX FIXME: apparently autobahn intentionally deviates from the - # spec in that on protocol errors it just closes the connection - # rather than trying to send a CLOSE frame. Investigate whether we - # should do the same. - self.close(code=exc.code, reason=str(exc)) - yield ConnectionClosed(exc.code, reason=str(exc)) - - def _generate_nonce(self): - # os.urandom may be overkill for this use case, but I don't think this - # is a bottleneck, and better safe than sorry... - self._nonce = base64.b64encode(os.urandom(16)) - - def _generate_accept_token(self, token): - accept_token = token + ACCEPT_GUID - accept_token = hashlib.sha1(accept_token).digest() - return base64.b64encode(accept_token) - - def _establish_client_connection(self, event): - if event.status_code != 101: - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "Bad status code from server") - headers = _normed_header_dict(event.headers) - if headers[b'connection'].lower() != b'upgrade': - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "Missing Connection: Upgrade header") - if headers[b'upgrade'].lower() != b'websocket': - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "Missing Upgrade: WebSocket header") - - accept_token = self._generate_accept_token(self._nonce) - if headers[b'sec-websocket-accept'] != accept_token: - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "Bad accept token") - - subprotocol = headers.get(b'sec-websocket-protocol', None) - if subprotocol is not None: - subprotocol = subprotocol.decode('ascii') - if subprotocol not in self.subprotocols: - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "unrecognized subprotocol {!r}" - .format(subprotocol)) - - extensions = headers.get(b'sec-websocket-extensions', None) - if extensions: - accepts = _split_comma_header(extensions) - - for accept in accepts: - name = accept.split(';', 1)[0].strip() - for extension in self.extensions: - if extension.name == name: - extension.finalize(self, accept) - break - else: - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "unrecognized extension {!r}" - .format(name)) - - self._proto = FrameProtocol(self.client, self.extensions) - self._state = ConnectionState.OPEN - return ConnectionEstablished(subprotocol, extensions) - - def _process_connection_request(self, event): - if event.method != b'GET': - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "Request method must be GET") - headers = _normed_header_dict(event.headers) - if headers[b'connection'].lower() != b'upgrade': - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "Missing Connection: Upgrade header") - if headers[b'upgrade'].lower() != b'websocket': - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "Missing Upgrade: WebSocket header") - - if b'sec-websocket-version' not in headers: - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "Missing Sec-WebSocket-Version header") - # XX FIXME: need to check Sec-Websocket-Version, and respond with a - # 400 if it's not what we expect - - if b'sec-websocket-protocol' in headers: - proposed_subprotocols = _split_comma_header( - headers[b'sec-websocket-protocol']) - else: - proposed_subprotocols = [] - - if b'sec-websocket-key' not in headers: - return ConnectionFailed(CloseReason.PROTOCOL_ERROR, - "Missing Sec-WebSocket-Key header") - - return ConnectionRequested(proposed_subprotocols, event) - - def _extension_accept(self, extensions_header): - accepts = {} - offers = _split_comma_header(extensions_header) - - for offer in offers: - name = offer.split(';', 1)[0].strip() - for extension in self.extensions: - if extension.name == name: - accept = extension.accept(self, offer) - if accept is True: - accepts[extension.name] = True - elif accept is not False and accept is not None: - accepts[extension.name] = accept.encode('ascii') - - if accepts: - extensions = [] - for name, params in accepts.items(): - if params is True: - extensions.append(name.encode('ascii')) - else: - # py34 annoyance: doesn't support bytestring formatting - params = params.decode("ascii") - extensions.append(('%s; %s' % (name, params)) - .encode("ascii")) - return b', '.join(extensions) - - return None - - def accept(self, event, subprotocol=None): - request = event.h11request - request_headers = _normed_header_dict(request.headers) - - nonce = request_headers[b'sec-websocket-key'] - accept_token = self._generate_accept_token(nonce) - - headers = { - b"Upgrade": b'WebSocket', - b"Connection": b'Upgrade', - b"Sec-WebSocket-Accept": accept_token, - } - - if subprotocol is not None: - if subprotocol not in event.proposed_subprotocols: - raise ValueError( - "unexpected subprotocol {!r}".format(subprotocol)) - headers[b'Sec-WebSocket-Protocol'] = subprotocol - - extensions = request_headers.get(b'sec-websocket-extensions', None) - if extensions: - accepts = self._extension_accept(extensions) - if accepts: - headers[b"Sec-WebSocket-Extensions"] = accepts - - response = h11.InformationalResponse(status_code=101, - headers=headers.items()) - self._outgoing += self._upgrade_connection.send(response) - self._proto = FrameProtocol(self.client, self.extensions) - self._state = ConnectionState.OPEN - - def ping(self, payload=None): - """ - Send a PING message to the peer. - - :param payload: an optional payload to send with the message - """ - - payload = bytes(payload or b'') - self._outgoing += self._proto.ping(payload) - - def pong(self, payload=None): - """ - Send a PONG message to the peer. - - This method can be used to send an unsolicted PONG to the peer. - It is not needed otherwise since every received PING causes a - corresponding PONG to be sent automatically. - - :param payload: an optional payload to send with the message - """ - - payload = bytes(payload or b'') - self._outgoing += self._proto.pong(payload) diff --git a/mitmproxy/contrib/wsproto/events.py b/mitmproxy/contrib/wsproto/events.py deleted file mode 100644 index 73ce27aa..00000000 --- a/mitmproxy/contrib/wsproto/events.py +++ /dev/null @@ -1,81 +0,0 @@ -# -*- coding: utf-8 -*- -""" -wsproto/events -~~~~~~~~~~ - -Events that result from processing data on a WebSocket connection. -""" - - -class ConnectionRequested(object): - def __init__(self, proposed_subprotocols, h11request): - self.proposed_subprotocols = proposed_subprotocols - self.h11request = h11request - - def __repr__(self): - path = self.h11request.target - - headers = dict(self.h11request.headers) - host = headers[b'host'] - version = headers[b'sec-websocket-version'] - subprotocol = headers.get(b'sec-websocket-protocol', None) - extensions = [] - - fmt = '<%s host=%s path=%s version=%s subprotocol=%r extensions=%r>' - return fmt % (self.__class__.__name__, host, path, version, - subprotocol, extensions) - - -class ConnectionEstablished(object): - def __init__(self, subprotocol=None, extensions=None): - self.subprotocol = subprotocol - self.extensions = extensions - if self.extensions is None: - self.extensions = [] - - def __repr__(self): - return '<ConnectionEstablished subprotocol=%r extensions=%r>' % \ - (self.subprotocol, self.extensions) - - -class ConnectionClosed(object): - def __init__(self, code, reason=None): - self.code = code - self.reason = reason - - def __repr__(self): - return '<%s code=%r reason="%s">' % (self.__class__.__name__, - self.code, self.reason) - - -class ConnectionFailed(ConnectionClosed): - pass - - -class DataReceived(object): - def __init__(self, data, frame_finished, message_finished): - self.data = data - # This has no semantic content, but is provided just in case some - # weird edge case user wants to be able to reconstruct the - # fragmentation pattern of the original stream. You don't want it: - self.frame_finished = frame_finished - # This is the field that you almost certainly want: - self.message_finished = message_finished - - -class TextReceived(DataReceived): - pass - - -class BytesReceived(DataReceived): - pass - - -class PingReceived(object): - def __init__(self, payload): - self.payload = payload - - -class PongReceived(object): - def __init__(self, payload): - self.payload = payload diff --git a/mitmproxy/contrib/wsproto/extensions.py b/mitmproxy/contrib/wsproto/extensions.py deleted file mode 100644 index 0e0d2018..00000000 --- a/mitmproxy/contrib/wsproto/extensions.py +++ /dev/null @@ -1,259 +0,0 @@ -# type: ignore - -# -*- coding: utf-8 -*- -""" -wsproto/extensions -~~~~~~~~~~~~~~ - -WebSocket extensions. -""" - -import zlib - -from .frame_protocol import CloseReason, Opcode, RsvBits - - -class Extension(object): - name = None - - def enabled(self): - return False - - def offer(self, connection): - pass - - def accept(self, connection, offer): - pass - - def finalize(self, connection, offer): - pass - - def frame_inbound_header(self, proto, opcode, rsv, payload_length): - return RsvBits(False, False, False) - - def frame_inbound_payload_data(self, proto, data): - return data - - def frame_inbound_complete(self, proto, fin): - pass - - def frame_outbound(self, proto, opcode, rsv, data, fin): - return (rsv, data) - - -class PerMessageDeflate(Extension): - name = 'permessage-deflate' - - DEFAULT_CLIENT_MAX_WINDOW_BITS = 15 - DEFAULT_SERVER_MAX_WINDOW_BITS = 15 - - def __init__(self, client_no_context_takeover=False, - client_max_window_bits=None, server_no_context_takeover=False, - server_max_window_bits=None): - self.client_no_context_takeover = client_no_context_takeover - if client_max_window_bits is None: - client_max_window_bits = self.DEFAULT_CLIENT_MAX_WINDOW_BITS - self.client_max_window_bits = client_max_window_bits - self.server_no_context_takeover = server_no_context_takeover - if server_max_window_bits is None: - server_max_window_bits = self.DEFAULT_SERVER_MAX_WINDOW_BITS - self.server_max_window_bits = server_max_window_bits - - self._compressor = None - self._decompressor = None - # This refers to the current frame - self._inbound_is_compressible = None - # This refers to the ongoing message (which might span multiple - # frames). Only the first frame in a fragmented message is flagged for - # compression, so this carries that bit forward. - self._inbound_compressed = None - - self._enabled = False - - def _compressible_opcode(self, opcode): - return opcode in (Opcode.TEXT, Opcode.BINARY, Opcode.CONTINUATION) - - def enabled(self): - return self._enabled - - def offer(self, connection): - parameters = [ - 'client_max_window_bits=%d' % self.client_max_window_bits, - 'server_max_window_bits=%d' % self.server_max_window_bits, - ] - - if self.client_no_context_takeover: - parameters.append('client_no_context_takeover') - if self.server_no_context_takeover: - parameters.append('server_no_context_takeover') - - return '; '.join(parameters) - - def finalize(self, connection, offer): - bits = [b.strip() for b in offer.split(';')] - for bit in bits[1:]: - if bit.startswith('client_no_context_takeover'): - self.client_no_context_takeover = True - elif bit.startswith('server_no_context_takeover'): - self.server_no_context_takeover = True - elif bit.startswith('client_max_window_bits'): - self.client_max_window_bits = int(bit.split('=', 1)[1].strip()) - elif bit.startswith('server_max_window_bits'): - self.server_max_window_bits = int(bit.split('=', 1)[1].strip()) - - self._enabled = True - - def _parse_params(self, params): - client_max_window_bits = None - server_max_window_bits = None - - bits = [b.strip() for b in params.split(';')] - for bit in bits[1:]: - if bit.startswith('client_no_context_takeover'): - self.client_no_context_takeover = True - elif bit.startswith('server_no_context_takeover'): - self.server_no_context_takeover = True - elif bit.startswith('client_max_window_bits'): - if '=' in bit: - client_max_window_bits = int(bit.split('=', 1)[1].strip()) - else: - client_max_window_bits = self.client_max_window_bits - elif bit.startswith('server_max_window_bits'): - if '=' in bit: - server_max_window_bits = int(bit.split('=', 1)[1].strip()) - else: - server_max_window_bits = self.server_max_window_bits - - return client_max_window_bits, server_max_window_bits - - def accept(self, connection, offer): - client_max_window_bits, server_max_window_bits = \ - self._parse_params(offer) - - self._enabled = True - - parameters = [] - - if self.client_no_context_takeover: - parameters.append('client_no_context_takeover') - if client_max_window_bits is not None: - parameters.append('client_max_window_bits=%d' % - client_max_window_bits) - self.client_max_window_bits = client_max_window_bits - if self.server_no_context_takeover: - parameters.append('server_no_context_takeover') - if server_max_window_bits is not None: - parameters.append('server_max_window_bits=%d' % - server_max_window_bits) - self.server_max_window_bits = server_max_window_bits - - return '; '.join(parameters) - - def frame_inbound_header(self, proto, opcode, rsv, payload_length): - if rsv.rsv1 and opcode.iscontrol(): - return CloseReason.PROTOCOL_ERROR - elif rsv.rsv1 and opcode is Opcode.CONTINUATION: - return CloseReason.PROTOCOL_ERROR - - self._inbound_is_compressible = self._compressible_opcode(opcode) - - if self._inbound_compressed is None: - self._inbound_compressed = rsv.rsv1 - if self._inbound_compressed: - assert self._inbound_is_compressible - if proto.client: - bits = self.server_max_window_bits - else: - bits = self.client_max_window_bits - if self._decompressor is None: - self._decompressor = zlib.decompressobj(-int(bits)) - - return RsvBits(True, False, False) - - def frame_inbound_payload_data(self, proto, data): - if not self._inbound_compressed or not self._inbound_is_compressible: - return data - - try: - return self._decompressor.decompress(bytes(data)) - except zlib.error: - return CloseReason.INVALID_FRAME_PAYLOAD_DATA - - def frame_inbound_complete(self, proto, fin): - if not fin: - return - elif not self._inbound_is_compressible: - return - elif not self._inbound_compressed: - return - - try: - data = self._decompressor.decompress(b'\x00\x00\xff\xff') - data += self._decompressor.flush() - except zlib.error: - return CloseReason.INVALID_FRAME_PAYLOAD_DATA - - if proto.client: - no_context_takeover = self.server_no_context_takeover - else: - no_context_takeover = self.client_no_context_takeover - - if no_context_takeover: - self._decompressor = None - - self._inbound_compressed = None - - return data - - def frame_outbound(self, proto, opcode, rsv, data, fin): - if not self._compressible_opcode(opcode): - return (rsv, data) - - if opcode is not Opcode.CONTINUATION: - rsv = RsvBits(True, *rsv[1:]) - - if self._compressor is None: - assert opcode is not Opcode.CONTINUATION - if proto.client: - bits = self.client_max_window_bits - else: - bits = self.server_max_window_bits - self._compressor = zlib.compressobj(zlib.Z_DEFAULT_COMPRESSION, - zlib.DEFLATED, -int(bits)) - - data = self._compressor.compress(bytes(data)) - - if fin: - data += self._compressor.flush(zlib.Z_SYNC_FLUSH) - data = data[:-4] - - if proto.client: - no_context_takeover = self.client_no_context_takeover - else: - no_context_takeover = self.server_no_context_takeover - - if no_context_takeover: - self._compressor = None - - return (rsv, data) - - def __repr__(self): - descr = ['client_max_window_bits=%d' % self.client_max_window_bits] - if self.client_no_context_takeover: - descr.append('client_no_context_takeover') - descr.append('server_max_window_bits=%d' % self.server_max_window_bits) - if self.server_no_context_takeover: - descr.append('server_no_context_takeover') - - descr = '; '.join(descr) - - return '<%s %s>' % (self.__class__.__name__, descr) - - -#: SUPPORTED_EXTENSIONS maps all supported extension names to their class. -#: This can be used to iterate all supported extensions of wsproto, instantiate -#: new extensions based on their name, or check if a given extension is -#: supported or not. -SUPPORTED_EXTENSIONS = { - PerMessageDeflate.name: PerMessageDeflate -} diff --git a/mitmproxy/contrib/wsproto/frame_protocol.py b/mitmproxy/contrib/wsproto/frame_protocol.py deleted file mode 100644 index 30f146c6..00000000 --- a/mitmproxy/contrib/wsproto/frame_protocol.py +++ /dev/null @@ -1,581 +0,0 @@ -# type: ignore - -# -*- coding: utf-8 -*- -""" -wsproto/frame_protocol -~~~~~~~~~~~~~~ - -WebSocket frame protocol implementation. -""" - -import os -import itertools -import struct -from codecs import getincrementaldecoder -from collections import namedtuple - -from enum import Enum, IntEnum - -from .compat import unicode, Utf8Validator - -try: - from wsaccel.xormask import XorMaskerSimple -except ImportError: - class XorMaskerSimple: - def __init__(self, masking_key): - self._maskbytes = itertools.cycle(bytearray(masking_key)) - - def process(self, data): - maskbytes = self._maskbytes - return bytearray(b ^ next(maskbytes) for b in bytearray(data)) - - -class XorMaskerNull: - def process(self, data): - return data - - -# RFC6455, Section 5.2 - Base Framing Protocol - -# Payload length constants -PAYLOAD_LENGTH_TWO_BYTE = 126 -PAYLOAD_LENGTH_EIGHT_BYTE = 127 -MAX_PAYLOAD_NORMAL = 125 -MAX_PAYLOAD_TWO_BYTE = 2 ** 16 - 1 -MAX_PAYLOAD_EIGHT_BYTE = 2 ** 64 - 1 -MAX_FRAME_PAYLOAD = MAX_PAYLOAD_EIGHT_BYTE - -# MASK and PAYLOAD LEN are packed into a byte -MASK_MASK = 0x80 -PAYLOAD_LEN_MASK = 0x7f - -# FIN, RSV[123] and OPCODE are packed into a single byte -FIN_MASK = 0x80 -RSV1_MASK = 0x40 -RSV2_MASK = 0x20 -RSV3_MASK = 0x10 -OPCODE_MASK = 0x0f - - -class Opcode(IntEnum): - """ - RFC 6455, Section 5.2 - Base Framing Protocol - """ - CONTINUATION = 0x0 - TEXT = 0x1 - BINARY = 0x2 - CLOSE = 0x8 - PING = 0x9 - PONG = 0xA - - def iscontrol(self): - return bool(self & 0x08) - - -class CloseReason(IntEnum): - """ - RFC 6455, Section 7.4.1 - Defined Status Codes - """ - NORMAL_CLOSURE = 1000 - GOING_AWAY = 1001 - PROTOCOL_ERROR = 1002 - UNSUPPORTED_DATA = 1003 - NO_STATUS_RCVD = 1005 - ABNORMAL_CLOSURE = 1006 - INVALID_FRAME_PAYLOAD_DATA = 1007 - POLICY_VIOLATION = 1008 - MESSAGE_TOO_BIG = 1009 - MANDATORY_EXT = 1010 - INTERNAL_ERROR = 1011 - SERVICE_RESTART = 1012 - TRY_AGAIN_LATER = 1013 - TLS_HANDSHAKE_FAILED = 1015 - - -# RFC 6455, Section 7.4.1 - Defined Status Codes -LOCAL_ONLY_CLOSE_REASONS = ( - CloseReason.NO_STATUS_RCVD, - CloseReason.ABNORMAL_CLOSURE, - CloseReason.TLS_HANDSHAKE_FAILED, -) - - -# RFC 6455, Section 7.4.2 - Status Code Ranges -MIN_CLOSE_REASON = 1000 -MIN_PROTOCOL_CLOSE_REASON = 1000 -MAX_PROTOCOL_CLOSE_REASON = 2999 -MIN_LIBRARY_CLOSE_REASON = 3000 -MAX_LIBRARY_CLOSE_REASON = 3999 -MIN_PRIVATE_CLOSE_REASON = 4000 -MAX_PRIVATE_CLOSE_REASON = 4999 -MAX_CLOSE_REASON = 4999 - - -NULL_MASK = struct.pack("!I", 0) - - -class ParseFailed(Exception): - def __init__(self, msg, code=CloseReason.PROTOCOL_ERROR): - super(ParseFailed, self).__init__(msg) - self.code = code - - -Header = namedtuple("Header", "fin rsv opcode payload_len masking_key".split()) - - -Frame = namedtuple("Frame", - "opcode payload frame_finished message_finished".split()) - - -RsvBits = namedtuple("RsvBits", "rsv1 rsv2 rsv3".split()) - - -def _truncate_utf8(data, nbytes): - if len(data) <= nbytes: - return data - - # Truncate - data = data[:nbytes] - # But we might have cut a codepoint in half, in which case we want to - # discard the partial character so the data is at least - # well-formed. This is a little inefficient since it processes the - # whole message twice when in theory we could just peek at the last - # few characters, but since this is only used for close messages (max - # length = 125 bytes) it really doesn't matter. - data = data.decode("utf-8", errors="ignore").encode("utf-8") - return data - - -class Buffer(object): - def __init__(self, initial_bytes=None): - self.buffer = bytearray() - self.bytes_used = 0 - if initial_bytes: - self.feed(initial_bytes) - - def feed(self, new_bytes): - self.buffer += new_bytes - - def consume_at_most(self, nbytes): - if not nbytes: - return bytearray() - - data = self.buffer[self.bytes_used:self.bytes_used + nbytes] - self.bytes_used += len(data) - return data - - def consume_exactly(self, nbytes): - if len(self.buffer) - self.bytes_used < nbytes: - return None - - return self.consume_at_most(nbytes) - - def commit(self): - # In CPython 3.4+, del[:n] is amortized O(n), *not* quadratic - del self.buffer[:self.bytes_used] - self.bytes_used = 0 - - def rollback(self): - self.bytes_used = 0 - - def __len__(self): - return len(self.buffer) - - -class MessageDecoder(object): - def __init__(self): - self.opcode = None - self.validator = None - self.decoder = None - - def process_frame(self, frame): - assert not frame.opcode.iscontrol() - - if self.opcode is None: - if frame.opcode is Opcode.CONTINUATION: - raise ParseFailed("unexpected CONTINUATION") - self.opcode = frame.opcode - elif frame.opcode is not Opcode.CONTINUATION: - raise ParseFailed("expected CONTINUATION, got %r" % frame.opcode) - - if frame.opcode is Opcode.TEXT: - self.validator = Utf8Validator() - self.decoder = getincrementaldecoder("utf-8")() - - finished = frame.frame_finished and frame.message_finished - - if self.decoder is not None: - data = self.decode_payload(frame.payload, finished) - else: - data = frame.payload - - frame = Frame(self.opcode, data, frame.frame_finished, finished) - - if finished: - self.opcode = None - self.decoder = None - - return frame - - def decode_payload(self, data, finished): - if self.validator is not None: - results = self.validator.validate(bytes(data)) - if not results[0] or (finished and not results[1]): - raise ParseFailed(u'encountered invalid UTF-8 while processing' - ' text message at payload octet index %d' % - results[3], - CloseReason.INVALID_FRAME_PAYLOAD_DATA) - - try: - return self.decoder.decode(data, finished) - except UnicodeDecodeError as exc: - raise ParseFailed(str(exc), CloseReason.INVALID_FRAME_PAYLOAD_DATA) - - -class FrameDecoder(object): - def __init__(self, client, extensions=None): - self.client = client - self.extensions = extensions or [] - - self.buffer = Buffer() - - self.header = None - self.effective_opcode = None - self.masker = None - self.payload_required = 0 - self.payload_consumed = 0 - - def receive_bytes(self, data): - self.buffer.feed(data) - - def process_buffer(self): - if not self.header: - if not self.parse_header(): - return None - - if len(self.buffer) < self.payload_required: - return None - - payload_remaining = self.header.payload_len - self.payload_consumed - payload = self.buffer.consume_at_most(payload_remaining) - if not payload and self.header.payload_len > 0: - return None - self.buffer.commit() - - self.payload_consumed += len(payload) - finished = self.payload_consumed == self.header.payload_len - - payload = self.masker.process(payload) - - for extension in self.extensions: - payload = extension.frame_inbound_payload_data(self, payload) - if isinstance(payload, CloseReason): - raise ParseFailed("error in extension", payload) - - if finished: - final = bytearray() - for extension in self.extensions: - result = extension.frame_inbound_complete(self, - self.header.fin) - if isinstance(result, CloseReason): - raise ParseFailed("error in extension", result) - if result is not None: - final += result - payload += final - - frame = Frame(self.effective_opcode, payload, finished, - self.header.fin) - - if finished: - self.header = None - self.effective_opcode = None - self.masker = None - else: - self.effective_opcode = Opcode.CONTINUATION - - return frame - - def parse_header(self): - data = self.buffer.consume_exactly(2) - if data is None: - self.buffer.rollback() - return False - - fin = bool(data[0] & FIN_MASK) - rsv = RsvBits(bool(data[0] & RSV1_MASK), - bool(data[0] & RSV2_MASK), - bool(data[0] & RSV3_MASK)) - opcode = data[0] & OPCODE_MASK - try: - opcode = Opcode(opcode) - except ValueError: - raise ParseFailed("Invalid opcode {:#x}".format(opcode)) - - if opcode.iscontrol() and not fin: - raise ParseFailed("Invalid attempt to fragment control frame") - - has_mask = bool(data[1] & MASK_MASK) - payload_len = data[1] & PAYLOAD_LEN_MASK - payload_len = self.parse_extended_payload_length(opcode, payload_len) - if payload_len is None: - self.buffer.rollback() - return False - - self.extension_processing(opcode, rsv, payload_len) - - if has_mask and self.client: - raise ParseFailed("client received unexpected masked frame") - if not has_mask and not self.client: - raise ParseFailed("server received unexpected unmasked frame") - if has_mask: - masking_key = self.buffer.consume_exactly(4) - if masking_key is None: - self.buffer.rollback() - return False - self.masker = XorMaskerSimple(masking_key) - else: - self.masker = XorMaskerNull() - - self.buffer.commit() - self.header = Header(fin, rsv, opcode, payload_len, None) - self.effective_opcode = self.header.opcode - if self.header.opcode.iscontrol(): - self.payload_required = payload_len - else: - self.payload_required = 0 - self.payload_consumed = 0 - return True - - def parse_extended_payload_length(self, opcode, payload_len): - if opcode.iscontrol() and payload_len > MAX_PAYLOAD_NORMAL: - raise ParseFailed("Control frame with payload len > 125") - if payload_len == PAYLOAD_LENGTH_TWO_BYTE: - data = self.buffer.consume_exactly(2) - if data is None: - return None - (payload_len,) = struct.unpack("!H", data) - if payload_len <= MAX_PAYLOAD_NORMAL: - raise ParseFailed( - "Payload length used 2 bytes when 1 would have sufficed") - elif payload_len == PAYLOAD_LENGTH_EIGHT_BYTE: - data = self.buffer.consume_exactly(8) - if data is None: - return None - (payload_len,) = struct.unpack("!Q", data) - if payload_len <= MAX_PAYLOAD_TWO_BYTE: - raise ParseFailed( - "Payload length used 8 bytes when 2 would have sufficed") - if payload_len >> 63: - # I'm not sure why this is illegal, but that's what the RFC - # says, so... - raise ParseFailed("8-byte payload length with non-zero MSB") - - return payload_len - - def extension_processing(self, opcode, rsv, payload_len): - rsv_used = [False, False, False] - for extension in self.extensions: - result = extension.frame_inbound_header(self, opcode, rsv, - payload_len) - if isinstance(result, CloseReason): - raise ParseFailed("error in extension", result) - for bit, used in enumerate(result): - if used: - rsv_used[bit] = True - for expected, found in zip(rsv_used, rsv): - if found and not expected: - raise ParseFailed("Reserved bit set unexpectedly") - - -class FrameProtocol(object): - class State(Enum): - HEADER = 1 - PAYLOAD = 2 - FRAME_COMPLETE = 3 - FAILED = 4 - - def __init__(self, client, extensions): - self.client = client - self.extensions = [ext for ext in extensions if ext.enabled()] - - # Global state - self._frame_decoder = FrameDecoder(self.client, self.extensions) - self._message_decoder = MessageDecoder() - self._parse_more = self.parse_more_gen() - - self._outbound_opcode = None - - def _process_close(self, frame): - data = frame.payload - - if not data: - # "If this Close control frame contains no status code, _The - # WebSocket Connection Close Code_ is considered to be 1005" - data = (CloseReason.NO_STATUS_RCVD, "") - elif len(data) == 1: - raise ParseFailed("CLOSE with 1 byte payload") - else: - (code,) = struct.unpack("!H", data[:2]) - if code < MIN_CLOSE_REASON or code > MAX_CLOSE_REASON: - raise ParseFailed("CLOSE with invalid code") - try: - code = CloseReason(code) - except ValueError: - pass - if code in LOCAL_ONLY_CLOSE_REASONS: - raise ParseFailed( - "remote CLOSE with local-only reason") - if not isinstance(code, CloseReason) and \ - code <= MAX_PROTOCOL_CLOSE_REASON: - raise ParseFailed( - "CLOSE with unknown reserved code") - validator = Utf8Validator() - if validator is not None: - results = validator.validate(bytes(data[2:])) - if not (results[0] and results[1]): - raise ParseFailed(u'encountered invalid UTF-8 while' - ' processing close message at payload' - ' octet index %d' % - results[3], - CloseReason.INVALID_FRAME_PAYLOAD_DATA) - try: - reason = data[2:].decode("utf-8") - except UnicodeDecodeError as exc: - raise ParseFailed( - "Error decoding CLOSE reason: " + str(exc), - CloseReason.INVALID_FRAME_PAYLOAD_DATA) - data = (code, reason) - - return Frame(frame.opcode, data, frame.frame_finished, - frame.message_finished) - - def parse_more_gen(self): - # Consume as much as we can from self._buffer, yielding events, and - # then yield None when we need more data. Or raise ParseFailed. - - # XX FIXME this should probably be refactored so that we never see - # disabled extensions in the first place... - self.extensions = [ext for ext in self.extensions if ext.enabled()] - closed = False - - while not closed: - frame = self._frame_decoder.process_buffer() - - if frame is not None: - if not frame.opcode.iscontrol(): - frame = self._message_decoder.process_frame(frame) - elif frame.opcode == Opcode.CLOSE: - frame = self._process_close(frame) - closed = True - - yield frame - - def receive_bytes(self, data): - self._frame_decoder.receive_bytes(data) - - def received_frames(self): - for event in self._parse_more: - if event is None: - break - else: - yield event - - def close(self, code=None, reason=None): - payload = bytearray() - if code is None and reason is not None: - raise TypeError("cannot specify a reason without a code") - if code in LOCAL_ONLY_CLOSE_REASONS: - code = CloseReason.NORMAL_CLOSURE - if code is not None: - payload += bytearray(struct.pack('!H', code)) - if reason is not None: - payload += _truncate_utf8(reason.encode('utf-8'), - MAX_PAYLOAD_NORMAL - 2) - - return self._serialize_frame(Opcode.CLOSE, payload) - - def ping(self, payload=b''): - return self._serialize_frame(Opcode.PING, payload) - - def pong(self, payload=b''): - return self._serialize_frame(Opcode.PONG, payload) - - def send_data(self, payload=b'', fin=True): - if isinstance(payload, (bytes, bytearray, memoryview)): - opcode = Opcode.BINARY - elif isinstance(payload, unicode): - opcode = Opcode.TEXT - payload = payload.encode('utf-8') - else: - raise ValueError('Must provide bytes or text') - - if self._outbound_opcode is None: - self._outbound_opcode = opcode - elif self._outbound_opcode is not opcode: - raise TypeError('Data type mismatch inside message') - else: - opcode = Opcode.CONTINUATION - - if fin: - self._outbound_opcode = None - - return self._serialize_frame(opcode, payload, fin) - - def _make_fin_rsv_opcode(self, fin, rsv, opcode): - fin = int(fin) << 7 - rsv = (int(rsv.rsv1) << 6) + (int(rsv.rsv2) << 5) + \ - (int(rsv.rsv3) << 4) - opcode = int(opcode) - - return fin | rsv | opcode - - def _serialize_frame(self, opcode, payload=b'', fin=True): - rsv = RsvBits(False, False, False) - for extension in reversed(self.extensions): - rsv, payload = extension.frame_outbound(self, opcode, rsv, payload, - fin) - - fin_rsv_opcode = self._make_fin_rsv_opcode(fin, rsv, opcode) - - payload_length = len(payload) - quad_payload = False - if payload_length <= MAX_PAYLOAD_NORMAL: - first_payload = payload_length - second_payload = None - elif payload_length <= MAX_PAYLOAD_TWO_BYTE: - first_payload = PAYLOAD_LENGTH_TWO_BYTE - second_payload = payload_length - else: - first_payload = PAYLOAD_LENGTH_EIGHT_BYTE - second_payload = payload_length - quad_payload = True - - if self.client: - first_payload |= 1 << 7 - - header = bytearray([fin_rsv_opcode, first_payload]) - if second_payload is not None: - if opcode.iscontrol(): - raise ValueError("payload too long for control frame") - if quad_payload: - header += bytearray(struct.pack('!Q', second_payload)) - else: - header += bytearray(struct.pack('!H', second_payload)) - - if self.client: - # "The masking key is a 32-bit value chosen at random by the - # client. When preparing a masked frame, the client MUST pick a - # fresh masking key from the set of allowed 32-bit values. The - # masking key needs to be unpredictable; thus, the masking key - # MUST be derived from a strong source of entropy, and the masking - # key for a given frame MUST NOT make it simple for a server/proxy - # to predict the masking key for a subsequent frame. The - # unpredictability of the masking key is essential to prevent - # authors of malicious applications from selecting the bytes that - # appear on the wire." - # -- https://tools.ietf.org/html/rfc6455#section-5.3 - masking_key = os.urandom(4) - masker = XorMaskerSimple(masking_key) - return header + masking_key + masker.process(payload) - - return header + payload diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py index 944c032d..6a27a4a8 100644 --- a/mitmproxy/flow.py +++ b/mitmproxy/flow.py @@ -87,7 +87,7 @@ class Flow(stateobject.StateObject): type=str, intercepted=bool, marked=bool, - metadata=dict, + metadata=typing.Dict[str, typing.Any], ) def get_state(self): diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index da9d2a44..51bd116b 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -1,5 +1,9 @@ """ This module handles the import of mitmproxy flows generated by old versions. + +The flow file version is decoupled from the mitmproxy release cycle (since +v3.0.0dev) and versioning. Every change or migration gets a new flow file +version number, this prevents issues with developer builds and snapshots. """ import uuid from typing import Any, Dict, Mapping, Union # noqa @@ -119,6 +123,7 @@ def convert_200_300(data): def convert_300_4(data): data["version"] = 4 + # Ths is an empty migration to transition to the new versioning scheme. return data @@ -149,6 +154,24 @@ def convert_4_5(data): return data +def convert_5_6(data): + data["version"] = 6 + data["client_conn"]["tls_established"] = data["client_conn"].pop("ssl_established") + data["client_conn"]["timestamp_tls_setup"] = data["client_conn"].pop("timestamp_ssl_setup") + data["server_conn"]["tls_established"] = data["server_conn"].pop("ssl_established") + data["server_conn"]["timestamp_tls_setup"] = data["server_conn"].pop("timestamp_ssl_setup") + if data["server_conn"]["via"]: + data["server_conn"]["via"]["tls_established"] = data["server_conn"]["via"].pop("ssl_established") + data["server_conn"]["via"]["timestamp_tls_setup"] = data["server_conn"]["via"].pop("timestamp_ssl_setup") + return data + + +def convert_6_7(data): + data["version"] = 7 + data["client_conn"]["tls_extensions"] = None + return data + + def _convert_dict_keys(o: Any) -> Any: if isinstance(o, dict): return {strutils.always_str(k): _convert_dict_keys(v) for k, v in o.items()} @@ -201,6 +224,8 @@ converters = { (2, 0): convert_200_300, (3, 0): convert_300_4, 4: convert_4_5, + 5: convert_5_6, + 6: convert_6_7, } diff --git a/mitmproxy/master.py b/mitmproxy/master.py index de3b24e1..a5e948f6 100644 --- a/mitmproxy/master.py +++ b/mitmproxy/master.py @@ -77,7 +77,7 @@ class Master: def add_log(self, e, level): """ - level: debug, info, warn, error + level: debug, alert, info, warn, error """ self.addons.trigger("log", log.LogEntry(e, level)) diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index 86f65cfd..f938cb12 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -76,7 +76,7 @@ def encode(s: Sequence[Tuple[str, str]], similar_to: str=None) -> str: encoded = urllib.parse.urlencode(s, False, errors="surrogateescape") - if remove_trailing_equal: + if encoded and remove_trailing_equal: encoded = encoded.replace("=&", "&") if encoded[-1] == '=': encoded = encoded[:-1] diff --git a/mitmproxy/net/tcp.py b/mitmproxy/net/tcp.py index d08938c9..85217794 100644 --- a/mitmproxy/net/tcp.py +++ b/mitmproxy/net/tcp.py @@ -301,11 +301,11 @@ class _Connection: self.rfile = None self.wfile = None - self.ssl_established = False + self.tls_established = False self.finished = False def get_current_cipher(self): - if not self.ssl_established: + if not self.tls_established: return None name = self.connection.get_cipher_name() @@ -381,7 +381,7 @@ class TCPClient(_Connection): else: close_socket(self.connection) - def convert_to_ssl(self, sni=None, alpn_protos=None, **sslctx_kwargs): + def convert_to_tls(self, sni=None, alpn_protos=None, **sslctx_kwargs): context = tls.create_client_context( alpn_protos=alpn_protos, sni=sni, @@ -400,13 +400,13 @@ class TCPClient(_Connection): else: raise exceptions.TlsException("SSL handshake error: %s" % repr(v)) - self.cert = certs.SSLCert(self.connection.get_peer_certificate()) + self.cert = certs.Cert(self.connection.get_peer_certificate()) # Keep all server certificates in a list for i in self.connection.get_peer_cert_chain(): - self.server_certs.append(certs.SSLCert(i)) + self.server_certs.append(certs.Cert(i)) - self.ssl_established = True + self.tls_established = True self.rfile.set_descriptor(self.connection) self.wfile.set_descriptor(self.connection) @@ -473,7 +473,7 @@ class TCPClient(_Connection): return self.connection.gettimeout() def get_alpn_proto_negotiated(self): - if self.ssl_established: + if self.tls_established: return self.connection.get_alpn_proto_negotiated() else: return b"" @@ -491,7 +491,7 @@ class BaseHandler(_Connection): self.server = server self.clientcert = None - def convert_to_ssl(self, cert, key, **sslctx_kwargs): + def convert_to_tls(self, cert, key, **sslctx_kwargs): """ Convert connection to SSL. For a list of parameters, see tls.create_server_context(...) @@ -507,10 +507,10 @@ class BaseHandler(_Connection): self.connection.do_handshake() except SSL.Error as v: raise exceptions.TlsException("SSL handshake error: %s" % repr(v)) - self.ssl_established = True + self.tls_established = True cert = self.connection.get_peer_certificate() if cert: - self.clientcert = certs.SSLCert(cert) + self.clientcert = certs.Cert(cert) self.rfile.set_descriptor(self.connection) self.wfile.set_descriptor(self.connection) @@ -521,7 +521,7 @@ class BaseHandler(_Connection): self.connection.settimeout(n) def get_alpn_proto_negotiated(self): - if self.ssl_established: + if self.tls_established: return self.connection.get_alpn_proto_negotiated() else: return b"" diff --git a/mitmproxy/net/tls.py b/mitmproxy/net/tls.py index 74911f1e..0e43a2ac 100644 --- a/mitmproxy/net/tls.py +++ b/mitmproxy/net/tls.py @@ -2,15 +2,20 @@ # then add options to disable certain methods # https://bugs.launchpad.net/pyopenssl/+bug/1020632/comments/3 import binascii +import io import os +import struct import threading import typing from ssl import match_hostname, CertificateError import certifi from OpenSSL import SSL +from kaitaistruct import KaitaiStream from mitmproxy import exceptions, certs +from mitmproxy.contrib.kaitaistruct import tls_client_hello +from mitmproxy.net import check BASIC_OPTIONS = ( SSL.OP_CIPHER_SERVER_PREFERENCE @@ -189,7 +194,7 @@ def _create_ssl_context( def create_client_context( cert: str = None, sni: str = None, - address: str=None, + address: str = None, verify: int = SSL.VERIFY_NONE, **sslctx_kwargs ) -> SSL.Context: @@ -213,7 +218,7 @@ def create_client_context( ) -> bool: if is_cert_verified and depth == 0: # Verify hostname of leaf certificate. - cert = certs.SSLCert(x509) + cert = certs.Cert(x509) try: crt = dict( subjectAltName=[("DNS", x.decode("ascii", "strict")) for x in cert.altnames] @@ -270,17 +275,17 @@ def create_client_context( def create_server_context( - cert: typing.Union[certs.SSLCert, str], + cert: typing.Union[certs.Cert, str], key: SSL.PKey, handle_sni: typing.Optional[typing.Callable[[SSL.Connection], None]] = None, request_client_cert: bool = False, chain_file=None, dhparams=None, - extra_chain_certs: typing.Iterable[certs.SSLCert] = None, + extra_chain_certs: typing.Iterable[certs.Cert] = None, **sslctx_kwargs ) -> SSL.Context: """ - cert: A certs.SSLCert object or the path to a certificate + cert: A certs.Cert object or the path to a certificate chain file. handle_sni: SNI handler, should take a connection object. Server @@ -321,7 +326,7 @@ def create_server_context( ) context.use_privatekey(key) - if isinstance(cert, certs.SSLCert): + if isinstance(cert, certs.Cert): context.use_certificate(cert.x509) else: context.use_certificate_chain_file(cert) @@ -338,3 +343,119 @@ def create_server_context( SSL._lib.SSL_CTX_set_tmp_dh(context._context, dhparams) return context + + +def is_tls_record_magic(d): + """ + Returns: + True, if the passed bytes start with the TLS record magic bytes. + False, otherwise. + """ + d = d[:3] + + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + return ( + len(d) == 3 and + d[0] == 0x16 and + d[1] == 0x03 and + 0x0 <= d[2] <= 0x03 + ) + + +def get_client_hello(rfile): + """ + Peek into the socket and read all records that contain the initial client hello message. + + client_conn: + The :py:class:`client connection <mitmproxy.connections.ClientConnection>`. + + Returns: + The raw handshake packet bytes, without TLS record header(s). + """ + client_hello = b"" + client_hello_size = 1 + offset = 0 + while len(client_hello) < client_hello_size: + record_header = rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) < 5: + raise exceptions.TlsProtocolException( + 'Expected TLS record, got "%s" instead.' % record_header) + record_size = struct.unpack_from("!H", record_header, 3)[0] + 5 + record_body = rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise exceptions.TlsProtocolException( + "Unexpected EOF in TLS handshake: %s" % record_body) + client_hello += record_body + offset += record_size + client_hello_size = struct.unpack("!I", b'\x00' + client_hello[1:4])[0] + 4 + return client_hello + + +class ClientHello: + + def __init__(self, raw_client_hello): + self._client_hello = tls_client_hello.TlsClientHello( + KaitaiStream(io.BytesIO(raw_client_hello)) + ) + + @property + def cipher_suites(self): + return self._client_hello.cipher_suites.cipher_suites + + @property + def sni(self): + if self._client_hello.extensions: + for extension in self._client_hello.extensions.extensions: + is_valid_sni_extension = ( + extension.type == 0x00 and + len(extension.body.server_names) == 1 and + extension.body.server_names[0].name_type == 0 and + check.is_valid_host(extension.body.server_names[0].host_name) + ) + if is_valid_sni_extension: + return extension.body.server_names[0].host_name.decode("idna") + return None + + @property + def alpn_protocols(self): + if self._client_hello.extensions: + for extension in self._client_hello.extensions.extensions: + if extension.type == 0x10: + return list(x.name for x in extension.body.alpn_protocols) + return [] + + @property + def extensions(self) -> typing.List[typing.Tuple[int, bytes]]: + ret = [] + if self._client_hello.extensions: + for extension in self._client_hello.extensions.extensions: + body = getattr(extension, "_raw_body", extension.body) + ret.append((extension.type, body)) + return ret + + @classmethod + def from_file(cls, client_conn) -> "ClientHello": + """ + Peek into the connection, read the initial client hello and parse it to obtain ALPN values. + client_conn: + The :py:class:`client connection <mitmproxy.connections.ClientConnection>`. + Returns: + :py:class:`client hello <mitmproxy.net.tls.ClientHello>`. + """ + try: + raw_client_hello = get_client_hello(client_conn)[4:] # exclude handshake header. + except exceptions.ProtocolException as e: + raise exceptions.TlsProtocolException('Cannot read raw Client Hello: %s' % repr(e)) + + try: + return cls(raw_client_hello) + except EOFError as e: + raise exceptions.TlsProtocolException( + 'Cannot parse Client Hello: %s, Raw Client Hello: %s' % + (repr(e), binascii.hexlify(raw_client_hello)) + ) + + def __repr__(self): + return "ClientHello(sni: %s, alpn_protocols: %s, cipher_suites: %s)" % \ + (self.sni, self.alpn_protocols, self.cipher_suites) diff --git a/mitmproxy/options.py b/mitmproxy/options.py index ff7edf39..862380c5 100644 --- a/mitmproxy/options.py +++ b/mitmproxy/options.py @@ -44,8 +44,6 @@ class Options(optmanager.OptManager): console_layout = None # type: str console_layout_headers = None # type: bool console_mouse = None # type: bool - console_order = None # type: str - console_order_reversed = None # type: bool console_palette = None # type: str console_palette_transparent = None # type: bool default_contentview = None # type: str @@ -98,6 +96,8 @@ class Options(optmanager.OptManager): upstream_cert = None # type: bool verbosity = None # type: str view_filter = None # type: Optional[str] + view_order = None # type: str + view_order_reversed = None # type: bool web_debug = None # type: bool web_iface = None # type: str web_open_browser = None # type: bool diff --git a/mitmproxy/proxy/protocol/__init__.py b/mitmproxy/proxy/protocol/__init__.py index 6dbdd13c..5860542a 100644 --- a/mitmproxy/proxy/protocol/__init__.py +++ b/mitmproxy/proxy/protocol/__init__.py @@ -36,13 +36,11 @@ from .http1 import Http1Layer from .http2 import Http2Layer from .websocket import WebSocketLayer from .rawtcp import RawTCPLayer -from .tls import TlsClientHello from .tls import TlsLayer -from .tls import is_tls_record_magic __all__ = [ "Layer", "ServerConnectionMixin", - "TlsLayer", "is_tls_record_magic", "TlsClientHello", + "TlsLayer", "UpstreamConnectLayer", "HttpLayer", "Http1Layer", diff --git a/mitmproxy/proxy/protocol/http_replay.py b/mitmproxy/proxy/protocol/http_replay.py index cc22c0b7..022e8133 100644 --- a/mitmproxy/proxy/protocol/http_replay.py +++ b/mitmproxy/proxy/protocol/http_replay.py @@ -75,7 +75,7 @@ class RequestReplayThread(basethread.BaseThread): ) if resp.status_code != 200: raise exceptions.ReplayException("Upstream server refuses CONNECT request") - server.establish_ssl( + server.establish_tls( self.options.client_certs, sni=self.f.server_conn.sni ) @@ -90,7 +90,7 @@ class RequestReplayThread(basethread.BaseThread): ) server.connect() if r.scheme == "https": - server.establish_ssl( + server.establish_tls( self.options.client_certs, sni=self.f.server_conn.sni ) diff --git a/mitmproxy/proxy/protocol/tls.py b/mitmproxy/proxy/protocol/tls.py index 21bf1417..d04c9801 100644 --- a/mitmproxy/proxy/protocol/tls.py +++ b/mitmproxy/proxy/protocol/tls.py @@ -1,14 +1,9 @@ -import struct from typing import Optional # noqa from typing import Union -import io -from kaitaistruct import KaitaiStream from mitmproxy import exceptions -from mitmproxy.contrib.kaitaistruct import tls_client_hello +from mitmproxy.net import tls as net_tls from mitmproxy.proxy.protocol import base -from mitmproxy.net import check - # taken from https://testssl.sh/openssl-rfc.mappping.html CIPHER_ID_NAME_MAP = { @@ -200,7 +195,6 @@ CIPHER_ID_NAME_MAP = { 0x080080: 'RC4-64-MD5', } - # We manually need to specify this, otherwise OpenSSL may select a non-HTTP2 cipher by default. # https://mozilla.github.io/server-side-tls/ssl-config-generator/?server=apache-2.2.15&openssl=1.0.2&hsts=yes&profile=old DEFAULT_CLIENT_CIPHERS = ( @@ -216,114 +210,7 @@ DEFAULT_CLIENT_CIPHERS = ( ) -def is_tls_record_magic(d): - """ - Returns: - True, if the passed bytes start with the TLS record magic bytes. - False, otherwise. - """ - d = d[:3] - - # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 - # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello - return ( - len(d) == 3 and - d[0] == 0x16 and - d[1] == 0x03 and - 0x0 <= d[2] <= 0x03 - ) - - -def get_client_hello(client_conn): - """ - Peek into the socket and read all records that contain the initial client hello message. - - client_conn: - The :py:class:`client connection <mitmproxy.connections.ClientConnection>`. - - Returns: - The raw handshake packet bytes, without TLS record header(s). - """ - client_hello = b"" - client_hello_size = 1 - offset = 0 - while len(client_hello) < client_hello_size: - record_header = client_conn.rfile.peek(offset + 5)[offset:] - if not is_tls_record_magic(record_header) or len(record_header) != 5: - raise exceptions.TlsProtocolException('Expected TLS record, got "%s" instead.' % record_header) - record_size = struct.unpack("!H", record_header[3:])[0] + 5 - record_body = client_conn.rfile.peek(offset + record_size)[offset + 5:] - if len(record_body) != record_size - 5: - raise exceptions.TlsProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) - client_hello += record_body - offset += record_size - client_hello_size = struct.unpack("!I", b'\x00' + client_hello[1:4])[0] + 4 - return client_hello - - -class TlsClientHello: - - def __init__(self, raw_client_hello): - self._client_hello = tls_client_hello.TlsClientHello(KaitaiStream(io.BytesIO(raw_client_hello))) - - def raw(self): - return self._client_hello - - @property - def cipher_suites(self): - return self._client_hello.cipher_suites.cipher_suites - - @property - def sni(self): - if self._client_hello.extensions: - for extension in self._client_hello.extensions.extensions: - is_valid_sni_extension = ( - extension.type == 0x00 and - len(extension.body.server_names) == 1 and - extension.body.server_names[0].name_type == 0 and - check.is_valid_host(extension.body.server_names[0].host_name) - ) - if is_valid_sni_extension: - return extension.body.server_names[0].host_name.decode("idna") - return None - - @property - def alpn_protocols(self): - if self._client_hello.extensions: - for extension in self._client_hello.extensions.extensions: - if extension.type == 0x10: - return list(x.name for x in extension.body.alpn_protocols) - return [] - - @classmethod - def from_client_conn(cls, client_conn): - """ - Peek into the connection, read the initial client hello and parse it to obtain ALPN values. - client_conn: - The :py:class:`client connection <mitmproxy.connections.ClientConnection>`. - Returns: - :py:class:`client hello <mitmproxy.proxy.protocol.tls.TlsClientHello>`. - """ - try: - raw_client_hello = get_client_hello(client_conn)[4:] # exclude handshake header. - except exceptions.ProtocolException as e: - raise exceptions.TlsProtocolException('Cannot read raw Client Hello: %s' % repr(e)) - - try: - return cls(raw_client_hello) - except EOFError as e: - raise exceptions.TlsProtocolException( - 'Cannot parse Client Hello: %s, Raw Client Hello: %s' % - (repr(e), raw_client_hello.encode("hex")) - ) - - def __repr__(self): - return "TlsClientHello( sni: %s alpn_protocols: %s, cipher_suites: %s)" % \ - (self.sni, self.alpn_protocols, self.cipher_suites) - - class TlsLayer(base.Layer): - """ The TLS layer implements transparent TLS connections. @@ -334,13 +221,13 @@ class TlsLayer(base.Layer): the server connection. """ - def __init__(self, ctx, client_tls, server_tls, custom_server_sni = None): + def __init__(self, ctx, client_tls, server_tls, custom_server_sni=None): super().__init__(ctx) self._client_tls = client_tls self._server_tls = server_tls self._custom_server_sni = custom_server_sni - self._client_hello = None # type: Optional[TlsClientHello] + self._client_hello = None # type: Optional[net_tls.ClientHello] def __call__(self): """ @@ -355,7 +242,7 @@ class TlsLayer(base.Layer): if self._client_tls: # Peek into the connection, read the initial client hello and parse it to obtain SNI and ALPN values. try: - self._client_hello = TlsClientHello.from_client_conn(self.client_conn) + self._client_hello = net_tls.ClientHello.from_file(self.client_conn.rfile) except exceptions.TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") @@ -414,7 +301,7 @@ class TlsLayer(base.Layer): if self._server_tls and not self.server_conn.tls_established: self._establish_tls_with_server() - def set_server_tls(self, server_tls: bool, sni: Union[str, None, bool]=None) -> None: + def set_server_tls(self, server_tls: bool, sni: Union[str, None, bool] = None) -> None: """ Set the TLS settings for the next server connection that will be established. This function will not alter an existing connection. @@ -487,7 +374,7 @@ class TlsLayer(base.Layer): extra_certs = None try: - self.client_conn.convert_to_ssl( + self.client_conn.convert_to_tls( cert, key, method=self.config.openssl_method_client, options=self.config.openssl_options_client, @@ -519,12 +406,14 @@ class TlsLayer(base.Layer): # We only support http/1.1 and h2. # If the server only supports spdy (next to http/1.1), it may select that # and mitmproxy would enter TCP passthrough mode, which we want to avoid. - alpn = [x for x in self._client_hello.alpn_protocols if - not (x.startswith(b"h2-") or x.startswith(b"spdy"))] + alpn = [ + x for x in self._client_hello.alpn_protocols if + not (x.startswith(b"h2-") or x.startswith(b"spdy")) + ] if alpn and b"h2" in alpn and not self.config.options.http2: alpn.remove(b"h2") - if self.client_conn.ssl_established and self.client_conn.get_alpn_proto_negotiated(): + if self.client_conn.tls_established and self.client_conn.get_alpn_proto_negotiated(): # If the client has already negotiated an ALP, then force the # server to use the same. This can only happen if the host gets # changed after the initial connection was established. E.g.: @@ -543,7 +432,7 @@ class TlsLayer(base.Layer): ciphers_server.append(CIPHER_ID_NAME_MAP[id]) ciphers_server = ':'.join(ciphers_server) - self.server_conn.establish_ssl( + self.server_conn.establish_tls( self.config.client_certs, self.server_sni, method=self.config.openssl_method_server, diff --git a/mitmproxy/proxy/protocol/websocket.py b/mitmproxy/proxy/protocol/websocket.py index 92f99518..2d8458a5 100644 --- a/mitmproxy/proxy/protocol/websocket.py +++ b/mitmproxy/proxy/protocol/websocket.py @@ -2,10 +2,10 @@ import socket from OpenSSL import SSL -from mitmproxy.contrib import wsproto -from mitmproxy.contrib.wsproto import events -from mitmproxy.contrib.wsproto.connection import ConnectionType, WSConnection -from mitmproxy.contrib.wsproto.extensions import PerMessageDeflate +import wsproto +from wsproto import events +from wsproto.connection import ConnectionType, WSConnection +from wsproto.extensions import PerMessageDeflate from mitmproxy import exceptions from mitmproxy import flow diff --git a/mitmproxy/proxy/root_context.py b/mitmproxy/proxy/root_context.py index c0ec64c9..eb0008cf 100644 --- a/mitmproxy/proxy/root_context.py +++ b/mitmproxy/proxy/root_context.py @@ -1,5 +1,6 @@ from mitmproxy import log from mitmproxy import exceptions +from mitmproxy.net import tls from mitmproxy.proxy import protocol from mitmproxy.proxy import modes from mitmproxy.proxy.protocol import http @@ -45,14 +46,14 @@ class RootContext: d = top_layer.client_conn.rfile.peek(3) except exceptions.TcpException as e: raise exceptions.ProtocolException(str(e)) - client_tls = protocol.is_tls_record_magic(d) + client_tls = tls.is_tls_record_magic(d) # 1. check for --ignore if self.config.check_ignore: ignore = self.config.check_ignore(top_layer.server_conn.address) if not ignore and client_tls: try: - client_hello = protocol.TlsClientHello.from_client_conn(self.client_conn) + client_hello = tls.ClientHello.from_file(self.client_conn.rfile) except exceptions.TlsProtocolException as e: self.log("Cannot parse Client Hello: %s" % repr(e), "error") else: @@ -76,10 +77,10 @@ class RootContext: # if the user manually sets a scheme for connect requests, we use this to decide if we # want TLS or not. if top_layer.connect_request.scheme: - tls = top_layer.connect_request.scheme == "https" + server_tls = top_layer.connect_request.scheme == "https" else: - tls = client_tls - return protocol.TlsLayer(top_layer, client_tls, tls) + server_tls = client_tls + return protocol.TlsLayer(top_layer, client_tls, server_tls) # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. if isinstance(top_layer, protocol.TlsLayer): diff --git a/mitmproxy/proxy/server.py b/mitmproxy/proxy/server.py index 5171fbee..5df5383a 100644 --- a/mitmproxy/proxy/server.py +++ b/mitmproxy/proxy/server.py @@ -114,9 +114,9 @@ class ConnectionHandler: def handle(self): self.log("clientconnect", "info") - root_layer = self._create_root_layer() - + root_layer = None try: + root_layer = self._create_root_layer() root_layer = self.channel.ask("clientconnect", root_layer) root_layer() except exceptions.Kill: @@ -151,7 +151,8 @@ class ConnectionHandler: print("Please lodge a bug report at: https://github.com/mitmproxy/mitmproxy", file=sys.stderr) self.log("clientdisconnect", "info") - self.channel.tell("clientdisconnect", root_layer) + if root_layer is not None: + self.channel.tell("clientdisconnect", root_layer) self.client_conn.finish() def log(self, msg, level): diff --git a/mitmproxy/stateobject.py b/mitmproxy/stateobject.py index 007339e8..ffaf285f 100644 --- a/mitmproxy/stateobject.py +++ b/mitmproxy/stateobject.py @@ -1,18 +1,12 @@ -from typing import Any -from typing import List +import typing +from typing import Any # noqa from typing import MutableMapping # noqa from mitmproxy.coretypes import serializable - - -def _is_list(cls): - # The typing module is broken on Python 3.5.0, fixed on 3.5.1. - is_list_bugfix = getattr(cls, "__origin__", False) == getattr(List[Any], "__origin__", True) - return issubclass(cls, List) or is_list_bugfix +from mitmproxy.utils import typecheck class StateObject(serializable.Serializable): - """ An object with serializable state. @@ -34,22 +28,7 @@ class StateObject(serializable.Serializable): state = {} for attr, cls in self._stateobject_attributes.items(): val = getattr(self, attr) - if val is None: - state[attr] = None - elif hasattr(val, "get_state"): - state[attr] = val.get_state() - elif _is_list(cls): - state[attr] = [x.get_state() for x in val] - elif isinstance(val, dict): - s = {} - for k, v in val.items(): - if hasattr(v, "get_state"): - s[k] = v.get_state() - else: - s[k] = v - state[attr] = s - else: - state[attr] = val + state[attr] = get_state(cls, val) return state def set_state(self, state): @@ -65,13 +44,51 @@ class StateObject(serializable.Serializable): curr = getattr(self, attr) if hasattr(curr, "set_state"): curr.set_state(val) - elif hasattr(cls, "from_state"): - obj = cls.from_state(val) - setattr(self, attr, obj) - elif _is_list(cls): - cls = cls.__parameters__[0] if cls.__parameters__ else cls.__args__[0] - setattr(self, attr, [cls.from_state(x) for x in val]) - else: # primitive types such as int, str, ... - setattr(self, attr, cls(val)) + else: + setattr(self, attr, make_object(cls, val)) if state: raise RuntimeWarning("Unexpected State in __setstate__: {}".format(state)) + + +def _process(typeinfo: typecheck.Type, val: typing.Any, make: bool) -> typing.Any: + if val is None: + return None + elif make and hasattr(typeinfo, "from_state"): + return typeinfo.from_state(val) + elif not make and hasattr(val, "get_state"): + return val.get_state() + + typename = str(typeinfo) + + if typename.startswith("typing.List"): + T = typecheck.sequence_type(typeinfo) + return [_process(T, x, make) for x in val] + elif typename.startswith("typing.Tuple"): + Ts = typecheck.tuple_types(typeinfo) + if len(Ts) != len(val): + raise ValueError("Invalid data. Expected {}, got {}.".format(Ts, val)) + return tuple( + _process(T, x, make) for T, x in zip(Ts, val) + ) + elif typename.startswith("typing.Dict"): + k_cls, v_cls = typecheck.mapping_types(typeinfo) + return { + _process(k_cls, k, make): _process(v_cls, v, make) + for k, v in val.items() + } + elif typename.startswith("typing.Any"): + # FIXME: Remove this when we remove flow.metadata + assert isinstance(val, (int, str, bool, bytes)) + return val + else: + return typeinfo(val) + + +def make_object(typeinfo: typecheck.Type, val: typing.Any) -> typing.Any: + """Create an object based on the state given in val.""" + return _process(typeinfo, val, True) + + +def get_state(typeinfo: typecheck.Type, val: typing.Any) -> typing.Any: + """Get the state of the object given as val.""" + return _process(typeinfo, val, False) diff --git a/mitmproxy/test/tflow.py b/mitmproxy/test/tflow.py index 91747866..204c7526 100644 --- a/mitmproxy/test/tflow.py +++ b/mitmproxy/test/tflow.py @@ -53,8 +53,8 @@ def twebsocketflow(client_conn=True, server_conn=True, messages=True, err=None, sec_websocket_version="13", sec_websocket_key="1234", ), - timestamp_start=1, - timestamp_end=2, + timestamp_start=946681200, + timestamp_end=946681201, content=b'' ) resp = http.HTTPResponse( @@ -66,8 +66,8 @@ def twebsocketflow(client_conn=True, server_conn=True, messages=True, err=None, upgrade='websocket', sec_websocket_accept=b'', ), - timestamp_start=1, - timestamp_end=2, + timestamp_start=946681202, + timestamp_end=946681203, content=b'', ) handshake_flow = http.HTTPFlow(client_conn, server_conn) @@ -157,14 +157,15 @@ def tclient_conn(): address=("127.0.0.1", 22), clientcert=None, mitmcert=None, - ssl_established=False, - timestamp_start=1, - timestamp_ssl_setup=2, - timestamp_end=3, + tls_established=False, + timestamp_start=946681200, + timestamp_tls_setup=946681201, + timestamp_end=946681206, sni="address", cipher_name="cipher", alpn_proto_negotiated=b"http/1.1", tls_version="TLSv1.2", + tls_extensions=[(0x00, bytes.fromhex("000e00000b6578616d"))], )) c.reply = controller.DummyReply() c.rfile = io.BytesIO() @@ -182,11 +183,11 @@ def tserver_conn(): source_address=("address", 22), ip_address=("192.168.0.1", 22), cert=None, - timestamp_start=1, - timestamp_tcp_setup=2, - timestamp_ssl_setup=3, - timestamp_end=4, - ssl_established=False, + timestamp_start=946681202, + timestamp_tcp_setup=946681203, + timestamp_tls_setup=946681204, + timestamp_end=946681205, + tls_established=False, sni="address", alpn_proto_negotiated=None, tls_version="TLSv1.2", diff --git a/mitmproxy/test/tutils.py b/mitmproxy/test/tutils.py index cd9f3b3f..d5b52bbe 100644 --- a/mitmproxy/test/tutils.py +++ b/mitmproxy/test/tutils.py @@ -31,8 +31,8 @@ def treq(**kwargs): http_version=b"HTTP/1.1", headers=http.Headers(((b"header", b"qvalue"), (b"content-length", b"7"))), content=b"content", - timestamp_start=1, - timestamp_end=2, + timestamp_start=946681200, + timestamp_end=946681201, ) default.update(kwargs) return http.Request(**default) @@ -49,8 +49,8 @@ def tresp(**kwargs): reason=b"OK", headers=http.Headers(((b"header-response", b"svalue"), (b"content-length", b"7"))), content=b"message", - timestamp_start=1, - timestamp_end=2, + timestamp_start=946681202, + timestamp_end=946681203, ) default.update(kwargs) return http.Response(**default) diff --git a/mitmproxy/tools/console/commander/commander.py b/mitmproxy/tools/console/commander/commander.py index 30e8b13b..e2088e71 100644 --- a/mitmproxy/tools/console/commander/commander.py +++ b/mitmproxy/tools/console/commander/commander.py @@ -178,5 +178,5 @@ class CommandEdit(urwid.WidgetWrap): x, y = calc_coords(self._w.get_text()[0], trans, p) return x, y - def get_value(self): + def get_edit_text(self): return self.cbuf.text diff --git a/mitmproxy/tools/console/common.py b/mitmproxy/tools/console/common.py index 47a30272..8a842799 100644 --- a/mitmproxy/tools/console/common.py +++ b/mitmproxy/tools/console/common.py @@ -1,9 +1,10 @@ import platform +import typing +from functools import lru_cache import urwid import urwid.util -from functools import lru_cache from mitmproxy.utils import human # Detect Windows Subsystem for Linux @@ -43,41 +44,48 @@ def highlight_key(str, key, textattr="text", keyattr="key"): KEY_MAX = 30 -def format_keyvals(lst, key="key", val="text", indent=0): +def format_keyvals( + entries: typing.List[typing.Tuple[str, typing.Union[None, str, urwid.Widget]]], + key_format: str = "key", + value_format: str = "text", + indent: int = 0 +) -> typing.List[urwid.Columns]: """ - Format a list of (key, value) tuples. - - If key is None, it's treated specially: - - We assume a sub-value, and add an extra indent. - - The value is treated as a pre-formatted list of directives. + Format a list of (key, value) tuples. + + Args: + entries: The list to format. keys must be strings, values can also be None or urwid widgets. + The latter makes it possible to use the result of format_keyvals() as a value. + key_format: The display attribute for the key. + value_format: The display attribute for the value. + indent: Additional indent to apply. """ + max_key_len = max((len(k) for k, v in entries if k is not None), default=0) + max_key_len = min(max_key_len, KEY_MAX) + + if indent > 2: + indent -= 2 # We use dividechars=2 below, which already adds two empty spaces + ret = [] - if lst: - maxk = min(max(len(i[0]) for i in lst if i and i[0]), KEY_MAX) - for i, kv in enumerate(lst): - if kv is None: - ret.append(urwid.Text("")) - else: - if isinstance(kv[1], urwid.Widget): - v = kv[1] - elif kv[1] is None: - v = urwid.Text("") - else: - v = urwid.Text([(val, kv[1])]) - ret.append( - urwid.Columns( - [ - ("fixed", indent, urwid.Text("")), - ( - "fixed", - maxk, - urwid.Text([(key, kv[0] or "")]) - ), - v - ], - dividechars = 2 - ) - ) + for k, v in entries: + if v is None: + v = urwid.Text("") + elif not isinstance(v, urwid.Widget): + v = urwid.Text([(value_format, v)]) + ret.append( + urwid.Columns( + [ + ("fixed", indent, urwid.Text("")), + ( + "fixed", + max_key_len, + urwid.Text([(key_format, k)]) + ), + v + ], + dividechars=2 + ) + ) return ret @@ -205,19 +213,15 @@ def format_flow(f, focus, extended=False, hostheader=False, max_url_len=False): focus=focus, extended=extended, max_url_len=max_url_len, - - intercepted = f.intercepted, - acked = acked, - - req_timestamp = f.request.timestamp_start, - req_is_replay = f.request.is_replay, - req_method = f.request.method, - req_url = f.request.pretty_url if hostheader else f.request.url, - req_http_version = f.request.http_version, - - err_msg = f.error.msg if f.error else None, - - marked = f.marked, + intercepted=f.intercepted, + acked=acked, + req_timestamp=f.request.timestamp_start, + req_is_replay=f.request.is_replay, + req_method=f.request.method, + req_url=f.request.pretty_url if hostheader else f.request.url, + req_http_version=f.request.http_version, + err_msg=f.error.msg if f.error else None, + marked=f.marked, ) if f.response: if f.response.raw_content: @@ -232,11 +236,11 @@ def format_flow(f, focus, extended=False, hostheader=False, max_url_len=False): roundtrip = human.pretty_duration(duration) d.update(dict( - resp_code = f.response.status_code, - resp_reason = f.response.reason, - resp_is_replay = f.response.is_replay, - resp_clen = contentdesc, - roundtrip = roundtrip, + resp_code=f.response.status_code, + resp_reason=f.response.reason, + resp_is_replay=f.response.is_replay, + resp_clen=contentdesc, + roundtrip=roundtrip, )) t = f.response.headers.get("content-type") diff --git a/mitmproxy/tools/console/consoleaddons.py b/mitmproxy/tools/console/consoleaddons.py index 20d54bc6..298770c1 100644 --- a/mitmproxy/tools/console/consoleaddons.py +++ b/mitmproxy/tools/console/consoleaddons.py @@ -272,7 +272,7 @@ class ConsoleAddon: @command.command("console.command") def console_command(self, *partial: str) -> None: """ - Prompt the user to edit a command with a (possilby empty) starting value. + Prompt the user to edit a command with a (possibly empty) starting value. """ signals.status_prompt_command.send(partial=" ".join(partial)) # type: ignore diff --git a/mitmproxy/tools/console/defaultkeys.py b/mitmproxy/tools/console/defaultkeys.py index f8a3df2d..d01d9b7e 100644 --- a/mitmproxy/tools/console/defaultkeys.py +++ b/mitmproxy/tools/console/defaultkeys.py @@ -59,7 +59,7 @@ def map(km): km.add("M", "view.marked.toggle", ["flowlist"], "Toggle viewing marked flows") km.add( "n", - "console.command view.create get https://google.com", + "console.command view.create get https://example.com/", ["flowlist"], "Create a new flow" ) @@ -67,14 +67,14 @@ def map(km): "o", """ console.choose.cmd Order view.order.options - set console_order={choice} + set view_order={choice} """, ["flowlist"], "Set flow list order" ) km.add("r", "replay.client @focus", ["flowlist", "flowview"], "Replay this flow") km.add("S", "console.command replay.server ", ["flowlist"], "Start server replay") - km.add("v", "set console_order_reversed=toggle", ["flowlist"], "Reverse flow list order") + km.add("v", "set view_order_reversed=toggle", ["flowlist"], "Reverse flow list order") km.add("U", "flow.mark @all false", ["flowlist"], "Un-set all marks") km.add("w", "console.command save.file @shown ", ["flowlist"], "Save listed flows to file") km.add("V", "flow.revert @focus", ["flowlist", "flowview"], "Revert changes to this flow") diff --git a/mitmproxy/tools/console/eventlog.py b/mitmproxy/tools/console/eventlog.py index 911aeb91..8083180d 100644 --- a/mitmproxy/tools/console/eventlog.py +++ b/mitmproxy/tools/console/eventlog.py @@ -47,7 +47,7 @@ class EventLog(urwid.ListBox, layoutwidget.LayoutWidget): if log.log_tier(self.master.options.verbosity) < log.log_tier(entry.level): return txt = "%s: %s" % (entry.level, str(entry.msg)) - if entry.level in ("error", "warn"): + if entry.level in ("error", "warn", "alert"): e = urwid.Text((entry.level, txt)) else: e = urwid.Text(txt) diff --git a/mitmproxy/tools/console/flowdetailview.py b/mitmproxy/tools/console/flowdetailview.py index 28fe1fbc..443ca526 100644 --- a/mitmproxy/tools/console/flowdetailview.py +++ b/mitmproxy/tools/console/flowdetailview.py @@ -23,157 +23,157 @@ def flowdetails(state, flow: http.HTTPFlow): metadata = flow.metadata if metadata is not None and len(metadata) > 0: - parts = [[str(k), repr(v)] for k, v in metadata.items()] + parts = [(str(k), repr(v)) for k, v in metadata.items()] text.append(urwid.Text([("head", "Metadata:")])) - text.extend(common.format_keyvals(parts, key="key", val="text", indent=4)) + text.extend(common.format_keyvals(parts, indent=4)) if sc is not None and sc.ip_address: text.append(urwid.Text([("head", "Server Connection:")])) parts = [ - ["Address", human.format_address(sc.address)], + ("Address", human.format_address(sc.address)), ] if sc.ip_address: - parts.append(["Resolved Address", human.format_address(sc.ip_address)]) + parts.append(("Resolved Address", human.format_address(sc.ip_address))) if resp: - parts.append(["HTTP Version", resp.http_version]) + parts.append(("HTTP Version", resp.http_version)) if sc.alpn_proto_negotiated: - parts.append(["ALPN", sc.alpn_proto_negotiated]) + parts.append(("ALPN", sc.alpn_proto_negotiated)) text.extend( - common.format_keyvals(parts, key="key", val="text", indent=4) + common.format_keyvals(parts, indent=4) ) c = sc.cert if c: text.append(urwid.Text([("head", "Server Certificate:")])) parts = [ - ["Type", "%s, %s bits" % c.keyinfo], - ["SHA1 digest", c.digest("sha1")], - ["Valid to", str(c.notafter)], - ["Valid from", str(c.notbefore)], - ["Serial", str(c.serial)], - [ + ("Type", "%s, %s bits" % c.keyinfo), + ("SHA1 digest", c.digest("sha1")), + ("Valid to", str(c.notafter)), + ("Valid from", str(c.notbefore)), + ("Serial", str(c.serial)), + ( "Subject", urwid.BoxAdapter( urwid.ListBox( common.format_keyvals( c.subject, - key="highlight", - val="text" + key_format="highlight" ) ), len(c.subject) ) - ], - [ + ), + ( "Issuer", urwid.BoxAdapter( urwid.ListBox( common.format_keyvals( - c.issuer, key="highlight", val="text" + c.issuer, + key_format="highlight" ) ), len(c.issuer) ) - ] + ) ] if c.altnames: parts.append( - [ + ( "Alt names", ", ".join(strutils.bytes_to_escaped_str(x) for x in c.altnames) - ] + ) ) text.extend( - common.format_keyvals(parts, key="key", val="text", indent=4) + common.format_keyvals(parts, indent=4) ) if cc is not None: text.append(urwid.Text([("head", "Client Connection:")])) parts = [ - ["Address", "{}:{}".format(cc.address[0], cc.address[1])], + ("Address", "{}:{}".format(cc.address[0], cc.address[1])), ] if req: - parts.append(["HTTP Version", req.http_version]) + parts.append(("HTTP Version", req.http_version)) if cc.tls_version: - parts.append(["TLS Version", cc.tls_version]) + parts.append(("TLS Version", cc.tls_version)) if cc.sni: - parts.append(["Server Name Indication", cc.sni]) + parts.append(("Server Name Indication", cc.sni)) if cc.cipher_name: - parts.append(["Cipher Name", cc.cipher_name]) + parts.append(("Cipher Name", cc.cipher_name)) if cc.alpn_proto_negotiated: - parts.append(["ALPN", cc.alpn_proto_negotiated]) + parts.append(("ALPN", cc.alpn_proto_negotiated)) text.extend( - common.format_keyvals(parts, key="key", val="text", indent=4) + common.format_keyvals(parts, indent=4) ) parts = [] if cc is not None and cc.timestamp_start: parts.append( - [ + ( "Client conn. established", maybe_timestamp(cc, "timestamp_start") - ] + ) ) - if cc.ssl_established: + if cc.tls_established: parts.append( - [ + ( "Client conn. TLS handshake", - maybe_timestamp(cc, "timestamp_ssl_setup") - ] + maybe_timestamp(cc, "timestamp_tls_setup") + ) ) if sc is not None and sc.timestamp_start: parts.append( - [ + ( "Server conn. initiated", maybe_timestamp(sc, "timestamp_start") - ] + ) ) parts.append( - [ + ( "Server conn. TCP handshake", maybe_timestamp(sc, "timestamp_tcp_setup") - ] + ) ) - if sc.ssl_established: + if sc.tls_established: parts.append( - [ + ( "Server conn. TLS handshake", - maybe_timestamp(sc, "timestamp_ssl_setup") - ] + maybe_timestamp(sc, "timestamp_tls_setup") + ) ) if req is not None and req.timestamp_start: parts.append( - [ + ( "First request byte", maybe_timestamp(req, "timestamp_start") - ] + ) ) parts.append( - [ + ( "Request complete", maybe_timestamp(req, "timestamp_end") - ] + ) ) if resp is not None and resp.timestamp_start: parts.append( - [ + ( "First response byte", maybe_timestamp(resp, "timestamp_start") - ] + ) ) parts.append( - [ + ( "Response complete", maybe_timestamp(resp, "timestamp_end") - ] + ) ) if parts: @@ -181,6 +181,6 @@ def flowdetails(state, flow: http.HTTPFlow): parts = sorted(parts, key=lambda p: p[1]) text.append(urwid.Text([("head", "Timing:")])) - text.extend(common.format_keyvals(parts, key="key", val="text", indent=4)) + text.extend(common.format_keyvals(parts, indent=4)) return searchable.Searchable(text) diff --git a/mitmproxy/tools/console/flowview.py b/mitmproxy/tools/console/flowview.py index 05d2573f..a4b629d4 100644 --- a/mitmproxy/tools/console/flowview.py +++ b/mitmproxy/tools/console/flowview.py @@ -13,6 +13,7 @@ from mitmproxy.tools.console import flowdetailview from mitmproxy.tools.console import searchable from mitmproxy.tools.console import tabs import mitmproxy.tools.console.master # noqa +from mitmproxy.utils import strutils class SearchError(Exception): @@ -152,10 +153,31 @@ class FlowDetails(tabs.Tabs): def conn_text(self, conn): if conn: + hdrs = [] + for k, v in conn.headers.fields: + # This will always force an ascii representation of headers. For example, if the server sends a + # + # X-Authors: Made with ❤ in Hamburg + # + # header, mitmproxy will display the following: + # + # X-Authors: Made with \xe2\x9d\xa4 in Hamburg. + # + # The alternative would be to just use the header's UTF-8 representation and maybe + # do `str.replace("\t", "\\t")` to exempt tabs from urwid's special characters escaping [1]. + # That would in some terminals allow rendering UTF-8 characters, but the mapping + # wouldn't be bijective, i.e. a user couldn't distinguish "\\t" and "\t". + # Also, from a security perspective, a mitmproxy user couldn't be fooled by homoglyphs. + # + # 1) https://github.com/mitmproxy/mitmproxy/issues/1833 + # https://github.com/urwid/urwid/blob/6608ee2c9932d264abd1171468d833b7a4082e13/urwid/display_common.py#L35-L36, + + k = strutils.bytes_to_escaped_str(k) + ":" + v = strutils.bytes_to_escaped_str(v) + hdrs.append((k, v)) txt = common.format_keyvals( - [(h + ":", v) for (h, v) in conn.headers.items(multi=True)], - key = "header", - val = "text" + hdrs, + key_format="header" ) viewmode = self.master.commands.call("console.flowview.mode") msg, body = self.content_view(viewmode, conn) diff --git a/mitmproxy/tools/console/grideditor/col.py b/mitmproxy/tools/console/grideditor/col.py deleted file mode 100644 index 3331f3e7..00000000 --- a/mitmproxy/tools/console/grideditor/col.py +++ /dev/null @@ -1,67 +0,0 @@ -import typing - -import urwid - -from mitmproxy.tools.console import signals -from mitmproxy.tools.console.grideditor import base -from mitmproxy.utils import strutils - -strbytes = typing.Union[str, bytes] - - -class Column(base.Column): - def Display(self, data): - return Display(data) - - def Edit(self, data): - return Edit(data) - - def blank(self): - return "" - - def keypress(self, key, editor): - if key in ["m_select"]: - editor.walker.start_edit() - else: - return key - - -class Display(base.Cell): - def __init__(self, data: strbytes) -> None: - self.data = data - if isinstance(data, bytes): - escaped = strutils.bytes_to_escaped_str(data) - else: - escaped = data.encode() - w = urwid.Text(escaped, wrap="any") - super().__init__(w) - - def get_data(self) -> strbytes: - return self.data - - -class Edit(base.Cell): - def __init__(self, data: strbytes) -> None: - if isinstance(data, bytes): - escaped = strutils.bytes_to_escaped_str(data) - else: - escaped = data.encode() - self.type = type(data) # type: typing.Type - w = urwid.Edit(edit_text=escaped, wrap="any", multiline=True) - w = urwid.AttrWrap(w, "editfield") - super().__init__(w) - - def get_data(self) -> strbytes: - txt = self._w.get_text()[0].strip() - try: - if self.type == bytes: - return strutils.escaped_str_to_bytes(txt) - else: - return txt.decode() - except ValueError: - signals.status_message.send( - self, - message="Invalid Python-style string encoding.", - expire=1000 - ) - raise diff --git a/mitmproxy/tools/console/grideditor/col_text.py b/mitmproxy/tools/console/grideditor/col_text.py index f0ac06f8..32518670 100644 --- a/mitmproxy/tools/console/grideditor/col_text.py +++ b/mitmproxy/tools/console/grideditor/col_text.py @@ -21,7 +21,7 @@ class Column(col_bytes.Column): return TEdit(data, self.encoding_args) def blank(self): - return u"" + return "" # This is the same for both edit and display. diff --git a/mitmproxy/tools/console/grideditor/col_viewany.py b/mitmproxy/tools/console/grideditor/col_viewany.py new file mode 100644 index 00000000..f5d35eee --- /dev/null +++ b/mitmproxy/tools/console/grideditor/col_viewany.py @@ -0,0 +1,33 @@ +""" +A display-only column that displays any data type. +""" + +import typing + +import urwid +from mitmproxy.tools.console.grideditor import base +from mitmproxy.utils import strutils + + +class Column(base.Column): + def Display(self, data): + return Display(data) + + Edit = Display + + def blank(self): + return "" + + +class Display(base.Cell): + def __init__(self, data: typing.Any) -> None: + self.data = data + if isinstance(data, bytes): + data = strutils.bytes_to_escaped_str(data) + if not isinstance(data, str): + data = repr(data) + w = urwid.Text(data, wrap="any") + super().__init__(w) + + def get_data(self) -> typing.Any: + return self.data diff --git a/mitmproxy/tools/console/grideditor/editors.py b/mitmproxy/tools/console/grideditor/editors.py index b5d16737..fbe48a1a 100644 --- a/mitmproxy/tools/console/grideditor/editors.py +++ b/mitmproxy/tools/console/grideditor/editors.py @@ -1,13 +1,14 @@ +import typing from mitmproxy import exceptions +from mitmproxy.net.http import Headers from mitmproxy.tools.console import layoutwidget +from mitmproxy.tools.console import signals from mitmproxy.tools.console.grideditor import base -from mitmproxy.tools.console.grideditor import col -from mitmproxy.tools.console.grideditor import col_text from mitmproxy.tools.console.grideditor import col_bytes from mitmproxy.tools.console.grideditor import col_subgrid -from mitmproxy.tools.console import signals -from mitmproxy.net.http import Headers +from mitmproxy.tools.console.grideditor import col_text +from mitmproxy.tools.console.grideditor import col_viewany class QueryEditor(base.FocusEditor): @@ -67,7 +68,6 @@ class RequestFormEditor(base.FocusEditor): class PathEditor(base.FocusEditor): # TODO: Next row on enter? - title = "Edit Path Components" columns = [ col_text.Column("Component"), @@ -175,11 +175,22 @@ class OptionsEditor(base.GridEditor, layoutwidget.LayoutWidget): class DataViewer(base.GridEditor, layoutwidget.LayoutWidget): title = None # type: str - def __init__(self, master, vals): + def __init__( + self, + master, + vals: typing.Union[ + typing.List[typing.List[typing.Any]], + typing.List[typing.Any], + str, + ]) -> None: if vals: + # Whatever vals is, make it a list of rows containing lists of column values. + if isinstance(vals, str): + vals = [vals] if not isinstance(vals[0], list): vals = [[i] for i in vals] - self.columns = [col.Column("")] * len(vals[0]) + + self.columns = [col_viewany.Column("")] * len(vals[0]) super().__init__(master, vals, self.callback) def callback(self, vals): diff --git a/mitmproxy/tools/console/help.py b/mitmproxy/tools/console/help.py index 439289f6..1b4b9ac6 100644 --- a/mitmproxy/tools/console/help.py +++ b/mitmproxy/tools/console/help.py @@ -76,7 +76,7 @@ class HelpView(tabs.Tabs, layoutwidget.LayoutWidget): def filtexp(self): text = [] - text.extend(common.format_keyvals(flowfilter.help, key="key", val="text", indent=4)) + text.extend(common.format_keyvals(flowfilter.help, indent=4)) text.append( urwid.Text( [ @@ -96,7 +96,7 @@ class HelpView(tabs.Tabs, layoutwidget.LayoutWidget): ("!(~q & ~t \"text/html\")", "Anything but requests with a text/html content type."), ] text.extend( - common.format_keyvals(examples, key="key", val="text", indent=4) + common.format_keyvals(examples, indent=4) ) return CListBox(text) diff --git a/mitmproxy/tools/console/master.py b/mitmproxy/tools/console/master.py index 04c7cc0c..da35047e 100644 --- a/mitmproxy/tools/console/master.py +++ b/mitmproxy/tools/console/master.py @@ -88,7 +88,7 @@ class ConsoleMaster(master.Master): def sig_add_log(self, event_store, entry: log.LogEntry): if log.log_tier(self.options.verbosity) < log.log_tier(entry.level): return - if entry.level in ("error", "warn"): + if entry.level in ("error", "warn", "alert"): if self.first_tick: self.start_err = entry else: diff --git a/mitmproxy/tools/console/palettes.py b/mitmproxy/tools/console/palettes.py index 465fd574..df69ff2f 100644 --- a/mitmproxy/tools/console/palettes.py +++ b/mitmproxy/tools/console/palettes.py @@ -24,7 +24,7 @@ class Palette: # List and Connections 'method', 'focus', 'code_200', 'code_300', 'code_400', 'code_500', 'code_other', - 'error', "warn", + 'error', "warn", "alert", 'header', 'highlight', 'intercept', 'replay', 'mark', # Hex view @@ -103,6 +103,7 @@ class LowDark(Palette): code_500 = ('light red', 'default'), code_other = ('dark red', 'default'), + alert = ('light magenta', 'default'), warn = ('brown', 'default'), error = ('light red', 'default'), @@ -176,6 +177,7 @@ class LowLight(Palette): error = ('light red', 'default'), warn = ('brown', 'default'), + alert = ('light magenta', 'default'), header = ('dark blue', 'default'), highlight = ('black,bold', 'default'), @@ -265,6 +267,7 @@ class SolarizedLight(LowLight): error = (sol_red, 'default'), warn = (sol_orange, 'default'), + alert = (sol_magenta, 'default'), header = (sol_blue, 'default'), highlight = (sol_base01, 'default'), @@ -319,6 +322,7 @@ class SolarizedDark(LowDark): error = (sol_red, 'default'), warn = (sol_orange, 'default'), + alert = (sol_magenta, 'default'), header = (sol_blue, 'default'), highlight = (sol_base01, 'default'), diff --git a/mitmproxy/tools/console/statusbar.py b/mitmproxy/tools/console/statusbar.py index 572b70fc..09cfd58a 100644 --- a/mitmproxy/tools/console/statusbar.py +++ b/mitmproxy/tools/console/statusbar.py @@ -101,7 +101,7 @@ class ActionBar(urwid.WidgetWrap): elif k in self.onekey: self.prompt_execute(k) elif k == "enter": - self.prompt_execute(self._w.get_value()) + self.prompt_execute(self._w.get_edit_text()) else: if common.is_keypress(k): self._w.keypress(size, k) diff --git a/mitmproxy/tools/console/window.py b/mitmproxy/tools/console/window.py index 87680f6e..c7bce7d3 100644 --- a/mitmproxy/tools/console/window.py +++ b/mitmproxy/tools/console/window.py @@ -16,7 +16,10 @@ from mitmproxy.tools.console import eventlog class StackWidget(urwid.Frame): - def __init__(self, widget, title, focus): + def __init__(self, window, widget, title, focus): + self.is_focused = focus + self.window = window + if title: header = urwid.AttrWrap( urwid.Text(title), @@ -29,6 +32,11 @@ class StackWidget(urwid.Frame): header=header ) + def mouse_event(self, size, event, button, col, row, focus): + if event == "mouse press" and button == 1 and not self.is_focused: + self.window.switch() + return super().mouse_event(size, event, button, col, row, focus) + def keypress(self, size, key): # Make sure that we don't propagate cursor events outside of the widget. # Otherwise, in a horizontal layout, urwid's Pile would change the focused widget @@ -162,6 +170,7 @@ class Window(urwid.Frame): else: title = None return StackWidget( + self, widget, title, self.pane == idx @@ -234,28 +243,34 @@ class Window(urwid.Frame): self.view_changed() self.focus_changed() - def current(self, keyctx): + def stacks_sorted_by_focus(self): """ - Returns the active widget, but only the current focus or overlay has - a matching key context. + Returns: + self.stacks, with the focused stack first. """ - t = self.focus_stack().top_widget() - if t.keyctx == keyctx: - return t + stacks = self.stacks.copy() + stacks.insert(0, stacks.pop(self.pane)) + return stacks - def current_window(self, keyctx): + def current(self, keyctx): """ - Returns the active window, ignoring overlays. + Returns the active widget with a matching key context, including overlays. + If multiple stacks have an active widget with a matching key context, + the currently focused stack is preferred. """ - t = self.focus_stack().top_window() - if t.keyctx == keyctx: - return t + for s in self.stacks_sorted_by_focus(): + t = s.top_widget() + if t.keyctx == keyctx: + return t - def any(self, keyctx): + def current_window(self, keyctx): """ - Returns the top window of either stack if they match the context. + Returns the active window with a matching key context, ignoring overlays. + If multiple stacks have an active widget with a matching key context, + the currently focused stack is preferred. """ - for t in [x.top_window() for x in self.stacks]: + for s in self.stacks_sorted_by_focus(): + t = s.top_window() if t.keyctx == keyctx: return t diff --git a/mitmproxy/tools/web/app.py b/mitmproxy/tools/web/app.py index 77695515..36c9d917 100644 --- a/mitmproxy/tools/web/app.py +++ b/mitmproxy/tools/web/app.py @@ -43,6 +43,8 @@ def flow_to_json(flow: mitmproxy.flow.Flow) -> dict: continue f[conn]["alpn_proto_negotiated"] = \ f[conn]["alpn_proto_negotiated"].decode(errors="backslashreplace") + # There are some bytes in here as well, let's skip it until we have them in the UI. + f["client_conn"].pop("tls_extensions", None) if flow.error: f["error"] = flow.error.get_state() diff --git a/mitmproxy/types.py b/mitmproxy/types.py index 8ae8b309..3875128d 100644 --- a/mitmproxy/types.py +++ b/mitmproxy/types.py @@ -267,14 +267,14 @@ class _CutSpecType(_BaseType): "client_conn.address.host", "client_conn.tls_version", "client_conn.sni", - "client_conn.ssl_established", + "client_conn.tls_established", "server_conn.address.port", "server_conn.address.host", "server_conn.ip_address.host", "server_conn.tls_version", "server_conn.sni", - "server_conn.ssl_established", + "server_conn.tls_established", ] def completion(self, manager: _CommandBase, t: type, s: str) -> typing.Sequence[str]: diff --git a/mitmproxy/utils/arg_check.py b/mitmproxy/utils/arg_check.py index 73f7047c..873bef06 100644 --- a/mitmproxy/utils/arg_check.py +++ b/mitmproxy/utils/arg_check.py @@ -66,9 +66,9 @@ REPLACEMENTS = { "--palette": "console_palette", "--palette-transparent": "console_palette_transparent:", "--follow": "console_focus_follow", - "--order": "console_order", + "--order": "view_order", "--no-mouse": "console_mouse", - "--reverse": "console_order_reversed", + "--reverse": "view_order_reversed", "--no-http2-priority": "http2_priority", "--no-websocket": "websocket", "--no-upstream-cert": "upstream_cert", diff --git a/mitmproxy/utils/typecheck.py b/mitmproxy/utils/typecheck.py index 1070fad0..22db68f5 100644 --- a/mitmproxy/utils/typecheck.py +++ b/mitmproxy/utils/typecheck.py @@ -1,7 +1,40 @@ import typing +Type = typing.Union[ + typing.Any # anything more elaborate really fails with mypy at the moment. +] -def check_option_type(name: str, value: typing.Any, typeinfo: typing.Any) -> None: + +def sequence_type(typeinfo: typing.Type[typing.List]) -> Type: + """Return the type of a sequence, e.g. typing.List""" + try: + return typeinfo.__args__[0] # type: ignore + except AttributeError: # Python 3.5.0 + return typeinfo.__parameters__[0] # type: ignore + + +def tuple_types(typeinfo: typing.Type[typing.Tuple]) -> typing.Sequence[Type]: + """Return the types of a typing.Tuple""" + try: + return typeinfo.__args__ # type: ignore + except AttributeError: # Python 3.5.x + return typeinfo.__tuple_params__ # type: ignore + + +def union_types(typeinfo: typing.Type[typing.Tuple]) -> typing.Sequence[Type]: + """return the types of a typing.Union""" + try: + return typeinfo.__args__ # type: ignore + except AttributeError: # Python 3.5.x + return typeinfo.__union_params__ # type: ignore + + +def mapping_types(typeinfo: typing.Type[typing.Mapping]) -> typing.Tuple[Type, Type]: + """return the types of a mapping, e.g. typing.Dict""" + return typeinfo.__args__ # type: ignore + + +def check_option_type(name: str, value: typing.Any, typeinfo: Type) -> None: """ Check if the provided value is an instance of typeinfo and raises a TypeError otherwise. This function supports only those types required for @@ -16,13 +49,7 @@ def check_option_type(name: str, value: typing.Any, typeinfo: typing.Any) -> Non typename = str(typeinfo) if typename.startswith("typing.Union"): - try: - types = typeinfo.__args__ # type: ignore - except AttributeError: - # Python 3.5.x - types = typeinfo.__union_params__ # type: ignore - - for T in types: + for T in union_types(typeinfo): try: check_option_type(name, value, T) except TypeError: @@ -31,12 +58,7 @@ def check_option_type(name: str, value: typing.Any, typeinfo: typing.Any) -> Non return raise e elif typename.startswith("typing.Tuple"): - try: - types = typeinfo.__args__ # type: ignore - except AttributeError: - # Python 3.5.x - types = typeinfo.__tuple_params__ # type: ignore - + types = tuple_types(typeinfo) if not isinstance(value, (tuple, list)): raise e if len(types) != len(value): @@ -45,11 +67,7 @@ def check_option_type(name: str, value: typing.Any, typeinfo: typing.Any) -> Non check_option_type("{}[{}]".format(name, i), x, T) return elif typename.startswith("typing.Sequence"): - try: - T = typeinfo.__args__[0] # type: ignore - except AttributeError: - # Python 3.5.0 - T = typeinfo.__parameters__[0] # type: ignore + T = sequence_type(typeinfo) if not isinstance(value, (tuple, list)): raise e for v in value: diff --git a/mitmproxy/version.py b/mitmproxy/version.py index 3073c3d3..c2cb3822 100644 --- a/mitmproxy/version.py +++ b/mitmproxy/version.py @@ -9,7 +9,7 @@ MITMPROXY = "mitmproxy " + VERSION # Serialization format version. This is displayed nowhere, it just needs to be incremented by one # for each change in the file format. -FLOW_FORMAT_VERSION = 5 +FLOW_FORMAT_VERSION = 7 def get_version(dev: bool = False, build: bool = False, refresh: bool = False) -> str: @@ -33,7 +33,7 @@ def get_version(dev: bool = False, build: bool = False, refresh: bool = False) - here = os.path.abspath(os.path.join(os.path.dirname(__file__), "..")) try: git_describe = subprocess.check_output( - ['git', 'describe', '--tags', '--long'], + ['git', 'describe', '--long'], stderr=subprocess.STDOUT, cwd=here, ) @@ -48,7 +48,7 @@ def get_version(dev: bool = False, build: bool = False, refresh: bool = False) - # Add suffix for non-tagged releases if tag_dist > 0: - mitmproxy_version += ".dev{tag_dist:04}".format(tag_dist=tag_dist) + mitmproxy_version += ".dev{tag_dist}".format(tag_dist=tag_dist) # The wheel build tag (we use the commit) must start with a digit, so we include "0x" mitmproxy_version += "-0x{commit}".format(commit=commit) @@ -60,5 +60,5 @@ def get_version(dev: bool = False, build: bool = False, refresh: bool = False) - return mitmproxy_version -if __name__ == "__main__": +if __name__ == "__main__": # pragma: no cover print(VERSION) diff --git a/mitmproxy/websocket.py b/mitmproxy/websocket.py index a37edb54..66257852 100644 --- a/mitmproxy/websocket.py +++ b/mitmproxy/websocket.py @@ -1,7 +1,8 @@ import time from typing import List, Optional -from mitmproxy.contrib import wsproto +from wsproto.frame_protocol import CloseReason +from wsproto.frame_protocol import Opcode from mitmproxy import flow from mitmproxy.net import websockets @@ -17,7 +18,7 @@ class WebSocketMessage(serializable.Serializable): def __init__( self, type: int, from_client: bool, content: bytes, timestamp: Optional[int]=None, killed: bool=False ) -> None: - self.type = wsproto.frame_protocol.Opcode(type) # type: ignore + self.type = Opcode(type) # type: ignore """indicates either TEXT or BINARY (from wsproto.frame_protocol.Opcode).""" self.from_client = from_client """True if this messages was sent by the client.""" @@ -37,10 +38,10 @@ class WebSocketMessage(serializable.Serializable): def set_state(self, state): self.type, self.from_client, self.content, self.timestamp, self.killed = state - self.type = wsproto.frame_protocol.Opcode(self.type) # replace enum with bare int + self.type = Opcode(self.type) # replace enum with bare int def __repr__(self): - if self.type == wsproto.frame_protocol.Opcode.TEXT: + if self.type == Opcode.TEXT: return "text message: {}".format(repr(self.content)) else: return "binary message: {}".format(strutils.bytes_to_escaped_str(self.content)) @@ -66,7 +67,7 @@ class WebSocketFlow(flow.Flow): """A list containing all WebSocketMessage's.""" self.close_sender = 'client' """'client' if the client initiated connection closing.""" - self.close_code = wsproto.frame_protocol.CloseReason.NORMAL_CLOSURE + self.close_code = CloseReason.NORMAL_CLOSURE """WebSocket close code.""" self.close_message = '(message missing)' """WebSocket close message.""" diff --git a/pathod/pathoc.py b/pathod/pathoc.py index e5fe4c2d..b177d556 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -79,7 +79,7 @@ class SSLInfo: } t = types.get(pk.type(), "Uknown") parts.append("\tPubkey: %s bit %s" % (pk.bits(), t)) - s = certs.SSLCert(i) + s = certs.Cert(i) if s.altnames: parts.append("\tSANs: %s" % " ".join(strutils.always_str(n, "utf8") for n in s.altnames)) return "\n".join(parts) @@ -313,7 +313,7 @@ class Pathoc(tcp.TCPClient): if self.use_http2: alpn_protos.append(b'h2') - self.convert_to_ssl( + self.convert_to_tls( sni=self.sni, cert=self.clientcert, method=self.ssl_version, diff --git a/pathod/pathod.py b/pathod/pathod.py index f8e64f9e..17db57ee 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -170,7 +170,7 @@ class PathodHandler(tcp.BaseHandler): ), cipher=None, ) - if self.ssl_established: + if self.tls_established: retlog["cipher"] = self.get_current_cipher() m = utils.MemBool() @@ -244,7 +244,7 @@ class PathodHandler(tcp.BaseHandler): if self.server.ssl: try: cert, key, _ = self.server.ssloptions.get_cert(None) - self.convert_to_ssl( + self.convert_to_tls( cert, key, handle_sni=self.handle_sni, diff --git a/pathod/protocols/http.py b/pathod/protocols/http.py index 4387b4fb..5fcb6618 100644 --- a/pathod/protocols/http.py +++ b/pathod/protocols/http.py @@ -27,7 +27,7 @@ class HTTPProtocol: cert, key, chain_file_ = self.pathod_handler.server.ssloptions.get_cert( connect[0].encode() ) - self.pathod_handler.convert_to_ssl( + self.pathod_handler.convert_to_tls( cert, key, handle_sni=self.pathod_handler.handle_sni, diff --git a/pathod/protocols/websockets.py b/pathod/protocols/websockets.py index 2d1f1bf6..63e6ee0b 100644 --- a/pathod/protocols/websockets.py +++ b/pathod/protocols/websockets.py @@ -30,7 +30,7 @@ class WebsocketsProtocol: ), cipher=None, ) - if self.pathod_handler.ssl_established: + if self.pathod_handler.tls_established: retlog["cipher"] = self.pathod_handler.get_current_cipher() self.pathod_handler.addlog(retlog) ld = language.websockets.NESTED_LEADER diff --git a/release/.gitignore b/release/.gitignore index 2247d5f9..905eec6e 100644 --- a/release/.gitignore +++ b/release/.gitignore @@ -1,2 +1,3 @@ /build /dist +known_hosts diff --git a/release/README.md b/release/README.md index a60b7f98..7bb89638 100644 --- a/release/README.md +++ b/release/README.md @@ -5,6 +5,10 @@ Make sure run all these steps on the correct branch you want to create a new rel - Update CHANGELOG - Verify that all CI tests pass - Tag the release and push to Github + - For alphas, betas, and release candidates, use lightweight tags. + This is necessary so that the .devXXXX counter does not reset. + - For final releases, use annotated tags. + This makes the .devXXXX counter reset. - Wait for tag CI to complete ## GitHub Release diff --git a/release/known_hosts.enc b/release/known_hosts.enc new file mode 100644 index 00000000..585ee678 --- /dev/null +++ b/release/known_hosts.enc @@ -0,0 +1 @@ +gAAAAABaTif138dCP2-G3sAJxqh5icnwM0Zy7qh4HFCxeKQBMiVDr4nJyf9T82U677M_QKWRJmp_PsbnrshHXPylq0FuHwak7Yx7kdiLue6d85VQ7_kkMs-MlPM7_Xn54_zyuj1c0b3TVAuix2xHfFLdSd_mCxygFukLzf47OyYbno7lMY_-q0HZfVPz3PBZdk95wDcbYprmgEkVJZd64Tu_LG1JDDiz56LlqADMA4znMcSAoRmbVtHu-II09HMcX3TkmcqJsNv-IVHMs4fxW_DFsq9w5ARggL6ANMfhnFQPyMtgVHjGLkSjOMRshLkQUBVYx8yWEGaQOkP0doVtDS3fZ-MKc6OJC_NSs6gkm1rswjVsQsmgZGPIqjcVf9oCbFYcw0m-JrfB1irdsLoGzpfJaSGxveC7XqOd9ArBpCHFPVO-6ilu-E1qZelvL0HiplrFvJCMEev1U2YvznC1BWKpy81vJfH--64QKZ35yQBHMV_VoH-wi80EfWtz4ISvCMQWdjRAvhLHKHSYYhUSIgBZvCCQcPySdFpbDtwsQnzIqC8MQKG787w1FiYAwzdIHTWZuanENaPMALo0t0GgMSqPV4UUyw7dto8XSMqoUXOCuZNYjunVh7AzAKS7oMUYjDs38o92sWh5sZUpPfv2WYIiecTiQw4uPae7PdSwMhkI3WIOsSb8LURnG484vvgFc2jMpQThw-BHJx7tGYC0yFLouRH2O7m9x6xgiCiVA_u_BdOj_2PFufvOCaB9wno5Vo7C1hUERGWqoBZH0htBqxYci27hh8GFwkvj6OjFUyV_kk920cBYBDG4jS4bTrTzn_znJ9TNw2XkP98nA8cwlRYhDQG9FypJG0WwYkft3TVLSQ3Hq7t0nhvhSZvXts-3LR4S0_Hm0QgFUpUc-VHViinwK8_vQH3ZjvVlEWiXnzPdpAujjX_tQXsi13UE1Zp90wGeLrmdxGXq2K76Shytu8IwTcLNZ7m0jh8KmmfNwn6oZv-czqNmC4hh0OqRDFBrv3nnjDg2Vw74uKSZmXgtZlF_Zj9hPqxVWzj7lJUcyRqABBFbBH6lTSWPHLrzQ4eTex5dnOkXC8c3hRYDUt06xUkmDqaLK0rGFcfNXawZj1YqpUJW0qaNgbtBZRsSs92kblkETxCzcwxOfupmAhWdSkmCoxt019crodz3heREcyN2xcD9qHvdY49_FD3l3U6UhrWvmkDkzyLMd7VmRPWqlW0lkzrwav8e92leIq-xKFcvbnWgSdSCWWbXvIVJKcQ6hML3jX4oY7SoBs33U1Q0HfC7SuS5lqTASuRIOVCfIGeFfRwlIfEszbWg_WDoUjR6StaVq9tbtIC3mimWND82Z9r1NfUNxr8kFYIpH_6hbxhcW26HNBKr4wLxWFFE9l1QZORPM3s6z-lT4LzUPCkFExd_eYFx3X6yUJ3cHZhkQQzCLQqG7jQqvcMwDIfM-MXkJnttLfpBq0yiq0-mc-SEas5uy27iSJgbXnsV7G3YiKEelKW_uWP2bw-rQGG_AXMGNGF2A_aREsvGrEqPnyeHAxfS1bBcnqslpIzEwr9vyyJ5v_bxfHFQC4bwYMUvPGkjHVFc0Wrk7ss9P5Kd1bzh46H7OfroUbocmYBmHMMWEg-LvsG0RZil3KWh_CSyIIPETkDjuC3W7teT-wZK0zbTEaKCuz99Dg-tjzT6fP25ipoI70cX5R3KPwrLP3XNODRTsg_Jh7IpaXo9O3o8yLV9R6_rST_1KKJwzR2MMIXIvKaJQD9w2DZIaYx3tcVsXGCDnU4Tw2hhdB5wMCl3vHx83UHfjLxnc1tJ6ObpQUjwHM1SgHK8wLW409SVHphBbSjSilX5mIaR1S1SOTK53iFj5z6asZHY9JgDj11rng1uLKeirbrNZDnUme3NNYU-HX8Ret6oOesn3374uIHux1giqgR8VsPdkcMhvunx2oTP9R2fRBTSQ8sKNqDznRC8_qlQaRC94RnWO6VRNXVBT24cXq7HTepNp4f02UvUqQRyaIUmyn2S02mjLFECDm1iMxRhuacCKbI-WSKwJcm-7p39_Uh7m_nTl2VTseeQ-3NS6i-BiGmCHt3iDxR1Fkm31b50kWW3jCe6fcwMDeu3I_8mkQs_7mCFUjSDbvFUr2Y45a5guRlw63_KUW_mNN9td9hk8POWfxWEGhcZ9eRXh_eEdEaYZmviZdHi0I8pV52CqiEO-ZrnMw-w4rSpUQeRn9oKwp3GgB9j51RNlLqK9LTp-jfSGGi5GM-ab9sPgFCJLQ-HvHdGu0tQsF2wTD3qbJwNqapx28yNVfY6e8F2jOWjmP-zzFez8VNXcfoS--Ji_zI-VqsDx-cfz3DccWEjL6vjQOvaQTRwzhI7
\ No newline at end of file diff --git a/release/rtool.py b/release/rtool.py index 4a07885c..9050107e 100755 --- a/release/rtool.py +++ b/release/rtool.py @@ -299,11 +299,15 @@ def upload_snapshot(host, port, user, private_key, private_key_password, wheel, """ Upload snapshot to snapshot server """ + cnopts = pysftp.CnOpts( + knownhosts=join(RELEASE_DIR, 'known_hosts') + ) with pysftp.Connection(host=host, port=port, username=user, private_key=private_key, - private_key_pass=private_key_password) as sftp: + private_key_pass=private_key_password, + cnopts=cnopts) as sftp: dir_name = "snapshots/v{}".format(get_version()) sftp.makedirs(dir_name) with sftp.cd(dir_name): @@ -75,7 +75,6 @@ exclude = mitmproxy/proxy/protocol/tls.py mitmproxy/proxy/root_context.py mitmproxy/proxy/server.py - mitmproxy/stateobject.py mitmproxy/utils/bits.py pathod/language/actions.py pathod/language/base.py @@ -81,6 +81,7 @@ setup( "sortedcontainers>=1.5.4, <1.6", "tornado>=4.3, <4.6", "urwid>=1.3.1, <1.4", + "wsproto>=0.11.0,<0.12.0", ], extras_require={ ':sys_platform == "win32"': [ @@ -104,7 +105,7 @@ setup( ], 'examples': [ "beautifulsoup4>=4.4.1, <4.7", - "Pillow>=4.3,<4.4", + "Pillow>=4.3,<5.1", ] } ) diff --git a/test/mitmproxy/addons/test_clientplayback.py b/test/mitmproxy/addons/test_clientplayback.py index 2dc7eb92..3f990668 100644 --- a/test/mitmproxy/addons/test_clientplayback.py +++ b/test/mitmproxy/addons/test_clientplayback.py @@ -52,6 +52,10 @@ class TestClientPlayback: cp.stop_replay() assert not cp.flows + df = tflow.DummyFlow(tflow.tclient_conn(), tflow.tserver_conn(), True) + with pytest.raises(exceptions.CommandError, match="Can't replay live flow."): + cp.start_replay([df]) + def test_load_file(self, tmpdir): cp = clientplayback.ClientPlayback() with taddons.context(): diff --git a/test/mitmproxy/addons/test_cut.py b/test/mitmproxy/addons/test_cut.py index 71e699db..c444b8ee 100644 --- a/test/mitmproxy/addons/test_cut.py +++ b/test/mitmproxy/addons/test_cut.py @@ -23,8 +23,8 @@ def test_extract(): ["request.text", "content"], ["request.content", b"content"], ["request.raw_content", b"content"], - ["request.timestamp_start", "1"], - ["request.timestamp_end", "2"], + ["request.timestamp_start", "946681200"], + ["request.timestamp_end", "946681201"], ["request.header[header]", "qvalue"], ["response.status_code", "200"], @@ -33,30 +33,29 @@ def test_extract(): ["response.content", b"message"], ["response.raw_content", b"message"], ["response.header[header-response]", "svalue"], - ["response.timestamp_start", "1"], - ["response.timestamp_end", "2"], + ["response.timestamp_start", "946681202"], + ["response.timestamp_end", "946681203"], ["client_conn.address.port", "22"], ["client_conn.address.host", "127.0.0.1"], ["client_conn.tls_version", "TLSv1.2"], ["client_conn.sni", "address"], - ["client_conn.ssl_established", "false"], + ["client_conn.tls_established", "false"], ["server_conn.address.port", "22"], ["server_conn.address.host", "address"], ["server_conn.ip_address.host", "192.168.0.1"], ["server_conn.tls_version", "TLSv1.2"], ["server_conn.sni", "address"], - ["server_conn.ssl_established", "false"], + ["server_conn.tls_established", "false"], ] - for t in tests: - ret = cut.extract(t[0], tf) - if ret != t[1]: - raise AssertionError("%s: Expected %s, got %s" % (t[0], t[1], ret)) + for spec, expected in tests: + ret = cut.extract(spec, tf) + assert spec and ret == expected with open(tutils.test_data.path("mitmproxy/net/data/text_cert"), "rb") as f: d = f.read() - c1 = certs.SSLCert.from_pem(d) + c1 = certs.Cert.from_pem(d) tf.server_conn.cert = c1 assert "CERTIFICATE" in cut.extract("server_conn.cert", tf) diff --git a/test/mitmproxy/addons/test_proxyauth.py b/test/mitmproxy/addons/test_proxyauth.py index 1d05e137..97259d1c 100644 --- a/test/mitmproxy/addons/test_proxyauth.py +++ b/test/mitmproxy/addons/test_proxyauth.py @@ -190,7 +190,7 @@ class TestProxyAuth: with pytest.raises(exceptions.OptionsError): ctx.configure(up, proxyauth="ldap:test:test:test") - with pytest.raises(IndexError): + with pytest.raises(exceptions.OptionsError): ctx.configure(up, proxyauth="ldap:fake_serveruid=?dc=example,dc=com:person") with pytest.raises(exceptions.OptionsError): diff --git a/test/mitmproxy/addons/test_view.py b/test/mitmproxy/addons/test_view.py index 1c76eb21..6f2a9ca5 100644 --- a/test/mitmproxy/addons/test_view.py +++ b/test/mitmproxy/addons/test_view.py @@ -41,7 +41,7 @@ def test_order_generators(): tf = tflow.tflow(resp=True) rs = view.OrderRequestStart(v) - assert rs.generate(tf) == 1 + assert rs.generate(tf) == 946681200 rm = view.OrderRequestMethod(v) assert rm.generate(tf) == tf.request.method @@ -147,6 +147,10 @@ def test_create(): assert v[0].request.url == "http://foo.com/" v.create("get", "http://foo.com") assert len(v) == 2 + with pytest.raises(exceptions.CommandError, match="Invalid URL"): + v.create("get", "http://foo.com\\") + with pytest.raises(exceptions.CommandError, match="Invalid URL"): + v.create("get", "http://") def test_orders(): @@ -175,6 +179,10 @@ def test_load(tmpdir): v.load_file("nonexistent_file_path") except IOError: assert False + with open(path, "wb") as f: + f.write(b"invalidflows") + v.load_file(path) + assert tctx.master.has_log("Invalid data format.") def test_resolve(): diff --git a/test/mitmproxy/net/http/test_response.py b/test/mitmproxy/net/http/test_response.py index a77435c9..af35bab3 100644 --- a/test/mitmproxy/net/http/test_response.py +++ b/test/mitmproxy/net/http/test_response.py @@ -150,10 +150,10 @@ class TestResponseUtils: n = time.time() r.headers["date"] = email.utils.formatdate(n) pre = r.headers["date"] - r.refresh(1) + r.refresh(946681202) assert pre == r.headers["date"] - r.refresh(61) + r.refresh(946681262) d = email.utils.parsedate_tz(r.headers["date"]) d = email.utils.mktime_tz(d) # Weird that this is not exact... diff --git a/test/mitmproxy/net/http/test_url.py b/test/mitmproxy/net/http/test_url.py index 2064aab8..c9f61faf 100644 --- a/test/mitmproxy/net/http/test_url.py +++ b/test/mitmproxy/net/http/test_url.py @@ -108,6 +108,7 @@ def test_empty_key_trailing_equal_sign(): def test_encode(): assert url.encode([('foo', 'bar')]) assert url.encode([('foo', surrogates)]) + assert not url.encode([], similar_to="justatext") def test_decode(): diff --git a/test/mitmproxy/net/test_tcp.py b/test/mitmproxy/net/test_tcp.py index e9084be4..8c012e42 100644 --- a/test/mitmproxy/net/test_tcp.py +++ b/test/mitmproxy/net/test_tcp.py @@ -178,7 +178,7 @@ class TestServerSSL(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni="foo.com", options=SSL.OP_ALL) + c.convert_to_tls(sni="foo.com", options=SSL.OP_ALL) testval = b"echo!\n" c.wfile.write(testval) c.wfile.flush() @@ -188,7 +188,7 @@ class TestServerSSL(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): assert not c.get_current_cipher() - c.convert_to_ssl(sni="foo.com") + c.convert_to_tls(sni="foo.com") ret = c.get_current_cipher() assert ret assert "AES" in ret[0] @@ -205,7 +205,7 @@ class TestSSLv3Only(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.TlsException): - c.convert_to_ssl(sni="foo.com") + c.convert_to_tls(sni="foo.com") class TestInvalidTrustFile(tservers.ServerTestBase): @@ -213,7 +213,7 @@ class TestInvalidTrustFile(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.TlsException): - c.convert_to_ssl( + c.convert_to_tls( sni="example.mitmproxy.org", verify=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/generate.py") @@ -231,7 +231,7 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): def test_mode_default_should_pass(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() # Verification errors should be saved even if connection isn't aborted # aborted @@ -245,7 +245,7 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): def test_mode_none_should_pass(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(verify=SSL.VERIFY_NONE) + c.convert_to_tls(verify=SSL.VERIFY_NONE) # Verification errors should be saved even if connection isn't aborted assert c.ssl_verification_error @@ -259,7 +259,7 @@ class TestSSLUpstreamCertVerificationWBadServerCert(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.InvalidCertificateException): - c.convert_to_ssl( + c.convert_to_tls( sni="example.mitmproxy.org", verify=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") @@ -284,7 +284,7 @@ class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.TlsException): - c.convert_to_ssl( + c.convert_to_tls( verify=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") ) @@ -292,7 +292,7 @@ class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): def test_mode_none_should_pass_without_sni(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl( + c.convert_to_tls( verify=SSL.VERIFY_NONE, ca_path=tutils.test_data.path("mitmproxy/net/data/verificationcerts/") ) @@ -303,7 +303,7 @@ class TestSSLUpstreamCertVerificationWBadHostname(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.InvalidCertificateException): - c.convert_to_ssl( + c.convert_to_tls( sni="mitmproxy.org", verify=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") @@ -322,7 +322,7 @@ class TestSSLUpstreamCertVerificationWValidCertChain(tservers.ServerTestBase): def test_mode_strict_w_pemfile_should_pass(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl( + c.convert_to_tls( sni="example.mitmproxy.org", verify=SSL.VERIFY_PEER, ca_pemfile=tutils.test_data.path("mitmproxy/net/data/verificationcerts/trusted-root.crt") @@ -338,7 +338,7 @@ class TestSSLUpstreamCertVerificationWValidCertChain(tservers.ServerTestBase): def test_mode_strict_w_cadir_should_pass(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl( + c.convert_to_tls( sni="example.mitmproxy.org", verify=SSL.VERIFY_PEER, ca_path=tutils.test_data.path("mitmproxy/net/data/verificationcerts/") @@ -372,7 +372,7 @@ class TestSSLClientCert(tservers.ServerTestBase): def test_clientcert(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl( + c.convert_to_tls( cert=tutils.test_data.path("mitmproxy/net/data/clientcert/client.pem")) assert c.rfile.readline().strip() == b"1" @@ -380,7 +380,7 @@ class TestSSLClientCert(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(exceptions.TlsException): - c.convert_to_ssl(cert=tutils.test_data.path("mitmproxy/net/data/clientcert/make")) + c.convert_to_tls(cert=tutils.test_data.path("mitmproxy/net/data/clientcert/make")) class TestSNI(tservers.ServerTestBase): @@ -400,15 +400,15 @@ class TestSNI(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni="foo.com") + c.convert_to_tls(sni="foo.com") assert c.sni == "foo.com" assert c.rfile.readline() == b"foo.com" def test_idn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni="mitmproxyäöüß.example.com") - assert c.ssl_established + c.convert_to_tls(sni="mitmproxyäöüß.example.com") + assert c.tls_established assert "doesn't match" not in str(c.ssl_verification_error) @@ -421,7 +421,7 @@ class TestServerCipherList(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni="foo.com") + c.convert_to_tls(sni="foo.com") expected = b"['AES256-GCM-SHA384']" assert c.rfile.read(len(expected) + 2) == expected @@ -442,7 +442,7 @@ class TestServerCurrentCipher(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(sni="foo.com") + c.convert_to_tls(sni="foo.com") assert b'AES256-GCM-SHA384' in c.rfile.readline() @@ -456,7 +456,7 @@ class TestServerCipherListError(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(Exception, match="handshake error"): - c.convert_to_ssl(sni="foo.com") + c.convert_to_tls(sni="foo.com") class TestClientCipherListError(tservers.ServerTestBase): @@ -469,7 +469,7 @@ class TestClientCipherListError(tservers.ServerTestBase): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): with pytest.raises(Exception, match="cipher specification"): - c.convert_to_ssl(sni="foo.com", cipher_list="bogus") + c.convert_to_tls(sni="foo.com", cipher_list="bogus") class TestSSLDisconnect(tservers.ServerTestBase): @@ -484,7 +484,7 @@ class TestSSLDisconnect(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() # Excercise SSL.ZeroReturnError c.rfile.read(10) c.close() @@ -501,7 +501,7 @@ class TestSSLHardDisconnect(tservers.ServerTestBase): def test_echo(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() # Exercise SSL.SysCallError c.rfile.read(10) c.close() @@ -565,7 +565,7 @@ class TestALPNClient(tservers.ServerTestBase): def test_alpn(self, monkeypatch, alpn_protos, expected_negotiated, expected_response): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(alpn_protos=alpn_protos) + c.convert_to_tls(alpn_protos=alpn_protos) assert c.get_alpn_proto_negotiated() == expected_negotiated assert c.rfile.readline().strip() == expected_response @@ -587,7 +587,7 @@ class TestSSLTimeOut(tservers.ServerTestBase): def test_timeout_client(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() c.settimeout(0.1) with pytest.raises(exceptions.TcpTimeout): c.rfile.read(10) @@ -605,7 +605,7 @@ class TestDHParams(tservers.ServerTestBase): def test_dhparams(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() ret = c.get_current_cipher() assert ret[0] == "DHE-RSA-AES256-SHA" @@ -801,5 +801,5 @@ class TestPeekSSL(TestPeek): def _connect(self, c): with c.connect() as conn: - c.convert_to_ssl() + c.convert_to_tls() return conn.pop() diff --git a/test/mitmproxy/net/test_tls.py b/test/mitmproxy/net/test_tls.py index d0583d34..489bf89f 100644 --- a/test/mitmproxy/net/test_tls.py +++ b/test/mitmproxy/net/test_tls.py @@ -1,3 +1,5 @@ +import io + import pytest from mitmproxy import exceptions @@ -6,6 +8,17 @@ from mitmproxy.net.tcp import TCPClient from test.mitmproxy.net.test_tcp import EchoHandler from . import tservers +CLIENT_HELLO_NO_EXTENSIONS = bytes.fromhex( + "03015658a756ab2c2bff55f636814deac086b7ca56b65058c7893ffc6074f5245f70205658a75475103a152637" + "78e1bb6d22e8bbd5b6b0a3a59760ad354e91ba20d353001a0035002f000a000500040009000300060008006000" + "61006200640100" +) +FULL_CLIENT_HELLO_NO_EXTENSIONS = ( + b"\x16\x03\x03\x00\x65" # record layer + b"\x01\x00\x00\x61" + # handshake header + CLIENT_HELLO_NO_EXTENSIONS +) + class TestMasterSecretLogger(tservers.ServerTestBase): handler = EchoHandler @@ -22,7 +35,7 @@ class TestMasterSecretLogger(tservers.ServerTestBase): c = TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() c.wfile.write(testval) c.wfile.flush() assert c.rfile.readline() == testval @@ -53,3 +66,92 @@ class TestTLSInvalid: with pytest.raises(exceptions.TlsException, match="ALPN error"): tls.create_client_context(alpn_select="foo", alpn_select_callback="bar") + + +def test_is_record_magic(): + assert not tls.is_tls_record_magic(b"POST /") + assert not tls.is_tls_record_magic(b"\x16\x03") + assert not tls.is_tls_record_magic(b"\x16\x03\x04") + assert tls.is_tls_record_magic(b"\x16\x03\x00") + assert tls.is_tls_record_magic(b"\x16\x03\x01") + assert tls.is_tls_record_magic(b"\x16\x03\x02") + assert tls.is_tls_record_magic(b"\x16\x03\x03") + + +def test_get_client_hello(): + rfile = io.BufferedReader(io.BytesIO( + FULL_CLIENT_HELLO_NO_EXTENSIONS + )) + assert tls.get_client_hello(rfile) + + rfile = io.BufferedReader(io.BytesIO( + FULL_CLIENT_HELLO_NO_EXTENSIONS[:30] + )) + with pytest.raises(exceptions.TlsProtocolException, message="Unexpected EOF"): + tls.get_client_hello(rfile) + + rfile = io.BufferedReader(io.BytesIO( + b"GET /" + )) + with pytest.raises(exceptions.TlsProtocolException, message="Expected TLS record"): + tls.get_client_hello(rfile) + + +class TestClientHello: + def test_no_extensions(self): + c = tls.ClientHello(CLIENT_HELLO_NO_EXTENSIONS) + assert repr(c) + assert c.sni is None + assert c.cipher_suites == [53, 47, 10, 5, 4, 9, 3, 6, 8, 96, 97, 98, 100] + assert c.alpn_protocols == [] + assert c.extensions == [] + + def test_extensions(self): + data = bytes.fromhex( + "03033b70638d2523e1cba15f8364868295305e9c52aceabda4b5147210abc783e6e1000022c02bc02fc02cc030" + "cca9cca8cc14cc13c009c013c00ac014009c009d002f0035000a0100006cff0100010000000010000e00000b65" + "78616d706c652e636f6d0017000000230000000d00120010060106030501050304010403020102030005000501" + "00000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a00080006001d00" + "170018" + ) + c = tls.ClientHello(data) + assert repr(c) + assert c.sni == 'example.com' + assert c.cipher_suites == [ + 49195, 49199, 49196, 49200, 52393, 52392, 52244, 52243, 49161, + 49171, 49162, 49172, 156, 157, 47, 53, 10 + ] + assert c.alpn_protocols == [b'h2', b'http/1.1'] + assert c.extensions == [ + (65281, b'\x00'), + (0, b'\x00\x0e\x00\x00\x0bexample.com'), + (23, b''), + (35, b''), + (13, b'\x00\x10\x06\x01\x06\x03\x05\x01\x05\x03\x04\x01\x04\x03\x02\x01\x02\x03'), + (5, b'\x01\x00\x00\x00\x00'), + (18, b''), + (16, b'\x00\x0c\x02h2\x08http/1.1'), + (30032, b''), + (11, b'\x01\x00'), + (10, b'\x00\x06\x00\x1d\x00\x17\x00\x18') + ] + + def test_from_file(self): + rfile = io.BufferedReader(io.BytesIO( + FULL_CLIENT_HELLO_NO_EXTENSIONS + )) + assert tls.ClientHello.from_file(rfile) + + rfile = io.BufferedReader(io.BytesIO( + b"" + )) + with pytest.raises(exceptions.TlsProtocolException): + tls.ClientHello.from_file(rfile) + + rfile = io.BufferedReader(io.BytesIO( + b"\x16\x03\x03\x00\x07" # record layer + b"\x01\x00\x00\x03" + # handshake header + b"foo" + )) + with pytest.raises(exceptions.TlsProtocolException, message='Cannot parse Client Hello'): + tls.ClientHello.from_file(rfile) diff --git a/test/mitmproxy/net/tools/getcertnames b/test/mitmproxy/net/tools/getcertnames index d64e5ff5..9349415f 100644 --- a/test/mitmproxy/net/tools/getcertnames +++ b/test/mitmproxy/net/tools/getcertnames @@ -7,7 +7,7 @@ from mitmproxy.net import tcp def get_remote_cert(host, port, sni): c = tcp.TCPClient((host, port)) c.connect() - c.convert_to_ssl(sni=sni) + c.convert_to_tls(sni=sni) return c.cert if len(sys.argv) > 2: diff --git a/test/mitmproxy/net/tservers.py b/test/mitmproxy/net/tservers.py index 44701aa5..22e195e3 100644 --- a/test/mitmproxy/net/tservers.py +++ b/test/mitmproxy/net/tservers.py @@ -60,7 +60,7 @@ class _TServer(tcp.TCPServer): else: method = OpenSSL.SSL.SSLv23_METHOD options = None - h.convert_to_ssl( + h.convert_to_tls( cert, key, method=method, diff --git a/test/mitmproxy/proxy/protocol/test_http2.py b/test/mitmproxy/proxy/protocol/test_http2.py index 4f161ef5..194a57c9 100644 --- a/test/mitmproxy/proxy/protocol/test_http2.py +++ b/test/mitmproxy/proxy/protocol/test_http2.py @@ -141,7 +141,7 @@ class _Http2TestBase: while self.client.rfile.readline() != b"\r\n": pass - self.client.convert_to_ssl(alpn_protos=[b'h2']) + self.client.convert_to_tls(alpn_protos=[b'h2']) config = h2.config.H2Configuration( client_side=True, diff --git a/test/mitmproxy/proxy/protocol/test_tls.py b/test/mitmproxy/proxy/protocol/test_tls.py index e17ee46f..e69de29b 100644 --- a/test/mitmproxy/proxy/protocol/test_tls.py +++ b/test/mitmproxy/proxy/protocol/test_tls.py @@ -1,26 +0,0 @@ -from mitmproxy.proxy.protocol.tls import TlsClientHello - - -class TestClientHello: - - def test_no_extensions(self): - data = bytes.fromhex( - "03015658a756ab2c2bff55f636814deac086b7ca56b65058c7893ffc6074f5245f70205658a75475103a152637" - "78e1bb6d22e8bbd5b6b0a3a59760ad354e91ba20d353001a0035002f000a000500040009000300060008006000" - "61006200640100" - ) - c = TlsClientHello(data) - assert c.sni is None - assert c.alpn_protocols == [] - - def test_extensions(self): - data = bytes.fromhex( - "03033b70638d2523e1cba15f8364868295305e9c52aceabda4b5147210abc783e6e1000022c02bc02fc02cc030" - "cca9cca8cc14cc13c009c013c00ac014009c009d002f0035000a0100006cff0100010000000010000e00000b65" - "78616d706c652e636f6d0017000000230000000d00120010060106030501050304010403020102030005000501" - "00000000001200000010000e000c02683208687474702f312e3175500000000b00020100000a00080006001d00" - "170018" - ) - c = TlsClientHello(data) - assert c.sni == 'example.com' - assert c.alpn_protocols == [b'h2', b'http/1.1'] diff --git a/test/mitmproxy/proxy/protocol/test_websocket.py b/test/mitmproxy/proxy/protocol/test_websocket.py index d9389faf..5cd9601c 100644 --- a/test/mitmproxy/proxy/protocol/test_websocket.py +++ b/test/mitmproxy/proxy/protocol/test_websocket.py @@ -101,8 +101,8 @@ class _WebSocketTestBase: response = http.http1.read_response(self.client.rfile, request) if self.ssl: - self.client.convert_to_ssl() - assert self.client.ssl_established + self.client.convert_to_tls() + assert self.client.tls_established request = http.Request( "relative", diff --git a/test/mitmproxy/proxy/test_server.py b/test/mitmproxy/proxy/test_server.py index 8dce9bcd..87ec443a 100644 --- a/test/mitmproxy/proxy/test_server.py +++ b/test/mitmproxy/proxy/test_server.py @@ -143,9 +143,9 @@ class TcpMixin: # Test that we get the original SSL cert if self.ssl: - i_cert = certs.SSLCert(i.sslinfo.certchain[0]) - i2_cert = certs.SSLCert(i2.sslinfo.certchain[0]) - n_cert = certs.SSLCert(n.sslinfo.certchain[0]) + i_cert = certs.Cert(i.sslinfo.certchain[0]) + i2_cert = certs.Cert(i2.sslinfo.certchain[0]) + n_cert = certs.Cert(n.sslinfo.certchain[0]) assert i_cert == i2_cert assert i_cert != n_cert @@ -188,9 +188,9 @@ class TcpMixin: # Test that we get the original SSL cert if self.ssl: - i_cert = certs.SSLCert(i.sslinfo.certchain[0]) - i2_cert = certs.SSLCert(i2.sslinfo.certchain[0]) - n_cert = certs.SSLCert(n.sslinfo.certchain[0]) + i_cert = certs.Cert(i.sslinfo.certchain[0]) + i2_cert = certs.Cert(i2.sslinfo.certchain[0]) + n_cert = certs.Cert(n.sslinfo.certchain[0]) assert i_cert == i2_cert assert i_cert != n_cert @@ -511,6 +511,14 @@ class TestReverse(tservers.ReverseProxyTest, CommonMixin, TcpMixin): req = self.master.state.flows[0].request assert req.host_header == "127.0.0.1" + def test_selfconnection(self): + self.options.mode = "reverse:http://127.0.0.1:0" + + p = self.pathoc() + with p.connect(): + p.request("get:/") + assert self.master.has_log("The proxy shall not connect to itself.") + class TestReverseSSL(tservers.ReverseProxyTest, CommonMixin, TcpMixin): reverse = True @@ -579,7 +587,7 @@ class TestSocks5SSL(tservers.SocksModeTest): p = self.pathoc_raw() with p.connect(): p.socks_connect(("localhost", self.server.port)) - p.convert_to_ssl() + p.convert_to_tls() f = p.request("get:/p/200") assert f.status_code == 200 @@ -709,7 +717,7 @@ class TestProxy(tservers.HTTPProxyTest): first_flow = self.master.state.flows[0] second_flow = self.master.state.flows[1] assert first_flow.server_conn.timestamp_tcp_setup - assert first_flow.server_conn.timestamp_ssl_setup is None + assert first_flow.server_conn.timestamp_tls_setup is None assert second_flow.server_conn.timestamp_tcp_setup assert first_flow.server_conn.timestamp_tcp_setup == second_flow.server_conn.timestamp_tcp_setup @@ -723,12 +731,13 @@ class TestProxy(tservers.HTTPProxyTest): class TestProxySSL(tservers.HTTPProxyTest): ssl = True - def test_request_ssl_setup_timestamp_presence(self): + def test_request_tls_attribute_presence(self): # tests that the ssl timestamp is present when ssl is used f = self.pathod("304:b@10k") assert f.status_code == 304 first_flow = self.master.state.flows[0] - assert first_flow.server_conn.timestamp_ssl_setup + assert first_flow.server_conn.timestamp_tls_setup + assert first_flow.client_conn.tls_extensions def test_via(self): # tests that the ssl timestamp is present when ssl is used @@ -1149,7 +1158,7 @@ class AddUpstreamCertsToClientChainMixin: def test_add_upstream_certs_to_client_chain(self): with open(self.servercert, "rb") as f: d = f.read() - upstreamCert = certs.SSLCert.from_pem(d) + upstreamCert = certs.Cert.from_pem(d) p = self.pathoc() with p.connect(): upstream_cert_found_in_client_chain = False diff --git a/test/mitmproxy/test_certs.py b/test/mitmproxy/test_certs.py index 693bebc6..dcc185c0 100644 --- a/test/mitmproxy/test_certs.py +++ b/test/mitmproxy/test_certs.py @@ -136,18 +136,18 @@ class TestDummyCert: assert r.altnames == [] -class TestSSLCert: +class TestCert: def test_simple(self): with open(tutils.test_data.path("mitmproxy/net/data/text_cert"), "rb") as f: d = f.read() - c1 = certs.SSLCert.from_pem(d) + c1 = certs.Cert.from_pem(d) assert c1.cn == b"google.com" assert len(c1.altnames) == 436 with open(tutils.test_data.path("mitmproxy/net/data/text_cert_2"), "rb") as f: d = f.read() - c2 = certs.SSLCert.from_pem(d) + c2 = certs.Cert.from_pem(d) assert c2.cn == b"www.inode.co.nz" assert len(c2.altnames) == 2 assert c2.digest("sha1") @@ -165,20 +165,20 @@ class TestSSLCert: def test_err_broken_sans(self): with open(tutils.test_data.path("mitmproxy/net/data/text_cert_weird1"), "rb") as f: d = f.read() - c = certs.SSLCert.from_pem(d) + c = certs.Cert.from_pem(d) # This breaks unless we ignore a decoding error. assert c.altnames is not None def test_der(self): with open(tutils.test_data.path("mitmproxy/net/data/dercert"), "rb") as f: d = f.read() - s = certs.SSLCert.from_der(d) + s = certs.Cert.from_der(d) assert s.cn def test_state(self): with open(tutils.test_data.path("mitmproxy/net/data/text_cert"), "rb") as f: d = f.read() - c = certs.SSLCert.from_pem(d) + c = certs.Cert.from_pem(d) c.get_state() c2 = c.copy() @@ -188,6 +188,6 @@ class TestSSLCert: assert c == c2 assert c is not c2 - x = certs.SSLCert('') + x = certs.Cert('') x.set_state(a) assert x == c diff --git a/test/mitmproxy/test_connections.py b/test/mitmproxy/test_connections.py index 83f0bd34..9e5d89f1 100644 --- a/test/mitmproxy/test_connections.py +++ b/test/mitmproxy/test_connections.py @@ -41,10 +41,10 @@ class TestClientConnection: def test_tls_established_property(self): c = tflow.tclient_conn() c.tls_established = True - assert c.ssl_established + assert c.tls_established assert c.tls_established c.tls_established = False - assert not c.ssl_established + assert not c.tls_established assert not c.tls_established def test_make_dummy(self): @@ -113,10 +113,10 @@ class TestServerConnection: def test_tls_established_property(self): c = tflow.tserver_conn() c.tls_established = True - assert c.ssl_established + assert c.tls_established assert c.tls_established c.tls_established = False - assert not c.ssl_established + assert not c.tls_established assert not c.tls_established def test_make_dummy(self): @@ -155,7 +155,7 @@ class TestServerConnection: def test_sni(self): c = connections.ServerConnection(('', 1234)) with pytest.raises(ValueError, matches='sni must be str, not '): - c.establish_ssl(None, b'foobar') + c.establish_tls(None, b'foobar') def test_state(self): c = tflow.tserver_conn() @@ -206,7 +206,7 @@ class TestClientConnectionTLS: key = OpenSSL.crypto.load_privatekey( OpenSSL.crypto.FILETYPE_PEM, raw_key) - c.convert_to_ssl(cert, key) + c.convert_to_tls(cert, key) assert c.connected() assert c.sni == sni assert c.tls_established @@ -230,7 +230,7 @@ class TestServerConnectionTLS(tservers.ServerTestBase): def test_tls(self, clientcert): c = connections.ServerConnection(("127.0.0.1", self.port)) c.connect() - c.establish_ssl(clientcert, "foo.com") + c.establish_tls(clientcert, "foo.com") assert c.connected() assert c.sni == "foo.com" assert c.tls_established diff --git a/test/mitmproxy/test_stateobject.py b/test/mitmproxy/test_stateobject.py index d8c7a8e9..bd5d1792 100644 --- a/test/mitmproxy/test_stateobject.py +++ b/test/mitmproxy/test_stateobject.py @@ -1,101 +1,146 @@ -from typing import List +import typing + import pytest from mitmproxy.stateobject import StateObject -class Child(StateObject): +class TObject(StateObject): def __init__(self, x): self.x = x - _stateobject_attributes = dict( - x=int - ) - @classmethod def from_state(cls, state): obj = cls(None) obj.set_state(state) return obj + +class Child(TObject): + _stateobject_attributes = dict( + x=int + ) + def __eq__(self, other): return isinstance(other, Child) and self.x == other.x -class Container(StateObject): - def __init__(self): - self.child = None - self.children = None - self.dictionary = None +class TTuple(TObject): + _stateobject_attributes = dict( + x=typing.Tuple[int, Child] + ) + + +class TList(TObject): + _stateobject_attributes = dict( + x=typing.List[Child] + ) + +class TDict(TObject): _stateobject_attributes = dict( - child=Child, - children=List[Child], - dictionary=dict, + x=typing.Dict[str, Child] ) - @classmethod - def from_state(cls, state): - obj = cls() - obj.set_state(state) - return obj + +class TAny(TObject): + _stateobject_attributes = dict( + x=typing.Any + ) + + +class TSerializableChild(TObject): + _stateobject_attributes = dict( + x=Child + ) def test_simple(): a = Child(42) + assert a.get_state() == {"x": 42} b = a.copy() - assert b.get_state() == {"x": 42} a.set_state({"x": 44}) assert a.x == 44 assert b.x == 42 -def test_container(): - a = Container() - a.child = Child(42) +def test_serializable_child(): + child = Child(42) + a = TSerializableChild(child) + assert a.get_state() == { + "x": {"x": 42} + } + a.set_state({ + "x": {"x": 43} + }) + assert a.x.x == 43 + assert a.x is child b = a.copy() - assert a.child.x == b.child.x - b.child.x = 44 - assert a.child.x != b.child.x + assert a.x == b.x + assert a.x is not b.x -def test_container_list(): - a = Container() - a.children = [Child(42), Child(44)] +def test_tuple(): + a = TTuple((42, Child(43))) assert a.get_state() == { - "child": None, - "children": [{"x": 42}, {"x": 44}], - "dictionary": None, + "x": (42, {"x": 43}) } - copy = a.copy() - assert len(copy.children) == 2 - assert copy.children is not a.children - assert copy.children[0] is not a.children[0] - assert Container.from_state(a.get_state()) + b = a.copy() + a.set_state({"x": (44, {"x": 45})}) + assert a.x == (44, Child(45)) + assert b.x == (42, Child(43)) + +def test_tuple_err(): + a = TTuple(None) + with pytest.raises(ValueError, msg="Invalid data"): + a.set_state({"x": (42,)}) -def test_container_dict(): - a = Container() - a.dictionary = dict() - a.dictionary['foo'] = 'bar' - a.dictionary['bar'] = Child(44) + +def test_list(): + a = TList([Child(1), Child(2)]) assert a.get_state() == { - "child": None, - "children": None, - "dictionary": {'bar': {'x': 44}, 'foo': 'bar'}, + "x": [{"x": 1}, {"x": 2}], } copy = a.copy() - assert len(copy.dictionary) == 2 - assert copy.dictionary is not a.dictionary - assert copy.dictionary['bar'] is not a.dictionary['bar'] + assert len(copy.x) == 2 + assert copy.x is not a.x + assert copy.x[0] is not a.x[0] + + +def test_dict(): + a = TDict({"foo": Child(42)}) + assert a.get_state() == { + "x": {"foo": {"x": 42}} + } + b = a.copy() + assert list(a.x.items()) == list(b.x.items()) + assert a.x is not b.x + assert a.x["foo"] is not b.x["foo"] + + +def test_any(): + a = TAny(42) + b = a.copy() + assert a.x == b.x + + a = TAny(object()) + with pytest.raises(AssertionError): + a.get_state() def test_too_much_state(): - a = Container() - a.child = Child(42) + a = Child(42) s = a.get_state() s['foo'] = 'bar' - b = Container() with pytest.raises(RuntimeWarning): - b.set_state(s) + a.set_state(s) + + +def test_none(): + a = Child(None) + assert a.get_state() == {"x": None} + a = Child(42) + a.set_state({"x": None}) + assert a.x is None diff --git a/test/mitmproxy/test_version.py b/test/mitmproxy/test_version.py index f8d646dc..8c176542 100644 --- a/test/mitmproxy/test_version.py +++ b/test/mitmproxy/test_version.py @@ -1,3 +1,4 @@ +import pathlib import runpy import subprocess from unittest import mock @@ -6,7 +7,9 @@ from mitmproxy import version def test_version(capsys): - runpy.run_module('mitmproxy.version', run_name='__main__') + here = pathlib.Path(__file__).absolute().parent + version_file = here / ".." / ".." / "mitmproxy" / "version.py" + runpy.run_path(str(version_file), run_name='__main__') stdout, stderr = capsys.readouterr() assert len(stdout) > 0 assert stdout.strip() == version.VERSION @@ -27,7 +30,7 @@ def test_get_version(): assert version.get_version(True, True) == "3.0.0" m.return_value = b"tag-2-cafecafe" - assert version.get_version(True, True) == "3.0.0.dev0002-0xcafecaf" + assert version.get_version(True, True) == "3.0.0.dev2-0xcafecaf" - m.side_effect = subprocess.CalledProcessError(-1, 'git describe --tags --long') + m.side_effect = subprocess.CalledProcessError(-1, 'git describe --long') assert version.get_version(True, True) == "3.0.0" diff --git a/test/mitmproxy/tools/console/test_common.py b/test/mitmproxy/tools/console/test_common.py index 3ab4fd67..72438c49 100644 --- a/test/mitmproxy/tools/console/test_common.py +++ b/test/mitmproxy/tools/console/test_common.py @@ -1,12 +1,34 @@ +import urwid + from mitmproxy.test import tflow from mitmproxy.tools.console import common -from ....conftest import skip_appveyor - -@skip_appveyor def test_format_flow(): f = tflow.tflow(resp=True) assert common.format_flow(f, True) assert common.format_flow(f, True, hostheader=True) assert common.format_flow(f, True, extended=True) + + +def test_format_keyvals(): + assert common.format_keyvals( + [ + ("aa", "bb"), + ("cc", "dd"), + ("ee", None), + ] + ) + wrapped = urwid.BoxAdapter( + urwid.ListBox( + urwid.SimpleFocusListWalker( + common.format_keyvals([("foo", "bar")]) + ) + ), 1 + ) + assert wrapped.render((30, )) + assert common.format_keyvals( + [ + ("aa", wrapped) + ] + ) diff --git a/test/mitmproxy/tools/console/test_master.py b/test/mitmproxy/tools/console/test_master.py index 3aa0dc54..9779a482 100644 --- a/test/mitmproxy/tools/console/test_master.py +++ b/test/mitmproxy/tools/console/test_master.py @@ -4,22 +4,9 @@ from mitmproxy import options from mitmproxy.test import tflow from mitmproxy.test import tutils from mitmproxy.tools import console -from mitmproxy.tools.console import common from ... import tservers -def test_format_keyvals(): - assert common.format_keyvals( - [ - ("aa", "bb"), - None, - ("cc", "dd"), - (None, "dd"), - (None, "dd"), - ] - ) - - def test_options(): assert options.Options(replay_kill_extra=True) diff --git a/test/mitmproxy/utils/test_typecheck.py b/test/mitmproxy/utils/test_typecheck.py index 5295fff5..9cb4334e 100644 --- a/test/mitmproxy/utils/test_typecheck.py +++ b/test/mitmproxy/utils/test_typecheck.py @@ -93,3 +93,8 @@ def test_typesec_to_str(): assert(typecheck.typespec_to_str(typing.Optional[str])) == "optional str" with pytest.raises(NotImplementedError): typecheck.typespec_to_str(dict) + + +def test_mapping_types(): + # this is not covered by check_option_type, but still belongs in this module + assert (str, int) == typecheck.mapping_types(typing.Mapping[str, int]) diff --git a/test/pathod/protocols/test_http2.py b/test/pathod/protocols/test_http2.py index b1eebc73..95965cee 100644 --- a/test/pathod/protocols/test_http2.py +++ b/test/pathod/protocols/test_http2.py @@ -75,7 +75,7 @@ class TestCheckALPNMatch(net_tservers.ServerTestBase): def test_check_alpn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(alpn_protos=[b'h2']) + c.convert_to_tls(alpn_protos=[b'h2']) protocol = HTTP2StateProtocol(c) assert protocol.check_alpn() @@ -89,7 +89,7 @@ class TestCheckALPNMismatch(net_tservers.ServerTestBase): def test_check_alpn(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl(alpn_protos=[b'h2']) + c.convert_to_tls(alpn_protos=[b'h2']) protocol = HTTP2StateProtocol(c) with pytest.raises(NotImplementedError): protocol.check_alpn() @@ -207,7 +207,7 @@ class TestApplySettings(net_tservers.ServerTestBase): def test_apply_settings(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() protocol = HTTP2StateProtocol(c) protocol._apply_settings({ @@ -302,7 +302,7 @@ class TestReadRequest(net_tservers.ServerTestBase): def test_read_request(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() protocol = HTTP2StateProtocol(c, is_server=True) protocol.connection_preface_performed = True @@ -328,7 +328,7 @@ class TestReadRequestRelative(net_tservers.ServerTestBase): def test_asterisk_form(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() protocol = HTTP2StateProtocol(c, is_server=True) protocol.connection_preface_performed = True @@ -351,7 +351,7 @@ class TestReadRequestAbsolute(net_tservers.ServerTestBase): def test_absolute_form(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() protocol = HTTP2StateProtocol(c, is_server=True) protocol.connection_preface_performed = True @@ -378,7 +378,7 @@ class TestReadResponse(net_tservers.ServerTestBase): def test_read_response(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() protocol = HTTP2StateProtocol(c) protocol.connection_preface_performed = True @@ -404,7 +404,7 @@ class TestReadEmptyResponse(net_tservers.ServerTestBase): def test_read_empty_response(self): c = tcp.TCPClient(("127.0.0.1", self.port)) with c.connect(): - c.convert_to_ssl() + c.convert_to_tls() protocol = HTTP2StateProtocol(c) protocol.connection_preface_performed = True diff --git a/test/pathod/test_pathoc.py b/test/pathod/test_pathoc.py index 4b50e2a7..297b54d4 100644 --- a/test/pathod/test_pathoc.py +++ b/test/pathod/test_pathoc.py @@ -238,11 +238,11 @@ class TestDaemonHTTP2(PathocTestDaemon): http2_skip_connection_preface=True, ) - tmp_convert_to_ssl = c.convert_to_ssl - c.convert_to_ssl = Mock() - c.convert_to_ssl.side_effect = tmp_convert_to_ssl + tmp_convert_to_tls = c.convert_to_tls + c.convert_to_tls = Mock() + c.convert_to_tls.side_effect = tmp_convert_to_tls with c.connect(): - _, kwargs = c.convert_to_ssl.call_args + _, kwargs = c.convert_to_tls.call_args assert set(kwargs['alpn_protos']) == set([b'http/1.1', b'h2']) def test_request(self): diff --git a/test/pathod/test_pathod.py b/test/pathod/test_pathod.py index c0011952..d6522cb6 100644 --- a/test/pathod/test_pathod.py +++ b/test/pathod/test_pathod.py @@ -153,7 +153,7 @@ class CommonTests(tservers.DaemonTests): c = tcp.TCPClient(("localhost", self.d.port)) with c.connect(): if self.ssl: - c.convert_to_ssl() + c.convert_to_tls() c.wfile.write(b"foo\n\n\n") c.wfile.flush() l = self.d.last_log() @@ -241,7 +241,7 @@ class TestDaemonSSL(CommonTests): with c.connect(): c.wfile.write(b"\0\0\0\0") with pytest.raises(exceptions.TlsException): - c.convert_to_ssl() + c.convert_to_tls() l = self.d.last_log() assert l["type"] == "error" assert "SSL" in l["msg"] @@ -56,7 +56,7 @@ deps = -rrequirements.txt pyinstaller==3.3.1 twine==1.9.1 - pysftp==0.2.8 + pysftp==0.2.9 commands = mitmdump --version diff --git a/web/src/js/filt/filt.js b/web/src/js/filt/filt.js index 26058649..19a41af2 100644 --- a/web/src/js/filt/filt.js +++ b/web/src/js/filt/filt.js @@ -1929,7 +1929,7 @@ module.exports = (function() { function body(regex){ regex = new RegExp(regex, "i"); function bodyFilter(flow){ - return True; + return true; } bodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; return bodyFilter; @@ -1937,7 +1937,7 @@ module.exports = (function() { function requestBody(regex){ regex = new RegExp(regex, "i"); function requestBodyFilter(flow){ - return True; + return true; } requestBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; return requestBodyFilter; @@ -1945,7 +1945,7 @@ module.exports = (function() { function responseBody(regex){ regex = new RegExp(regex, "i"); function responseBodyFilter(flow){ - return True; + return true; } responseBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; return responseBodyFilter; @@ -2104,4 +2104,4 @@ module.exports = (function() { SyntaxError: peg$SyntaxError, parse: peg$parse }; -})();
\ No newline at end of file +})(); diff --git a/web/src/js/filt/filt.peg b/web/src/js/filt/filt.peg index 12959474..e4b151ad 100644 --- a/web/src/js/filt/filt.peg +++ b/web/src/js/filt/filt.peg @@ -1,4 +1,4 @@ -// PEG.js filter rules - see http://pegjs.majda.cz/online +// PEG.js filter rules - see https://pegjs.org/ { var flowutils = require("../flow/utils.js"); @@ -72,7 +72,7 @@ function responseCode(code){ function body(regex){ regex = new RegExp(regex, "i"); function bodyFilter(flow){ - return True; + return true; } bodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; return bodyFilter; @@ -80,7 +80,7 @@ function body(regex){ function requestBody(regex){ regex = new RegExp(regex, "i"); function requestBodyFilter(flow){ - return True; + return true; } requestBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; return requestBodyFilter; @@ -88,7 +88,7 @@ function requestBody(regex){ function responseBody(regex){ regex = new RegExp(regex, "i"); function responseBodyFilter(flow){ - return True; + return true; } responseBodyFilter.desc = "body filters are not implemented yet, see https://github.com/mitmproxy/mitmweb/issues/10"; return responseBodyFilter; |
