diff options
| -rw-r--r-- | mitmproxy/io/compat.py | 19 | ||||
| -rw-r--r-- | mitmproxy/net/http/url.py | 24 | ||||
| -rw-r--r-- | setup.cfg | 5 | ||||
| -rw-r--r-- | test/mitmproxy/data/dumpfile-018 | 40 | ||||
| -rw-r--r-- | test/mitmproxy/net/http/test_cookies.py | 3 | ||||
| -rw-r--r-- | test/mitmproxy/net/http/test_message.py | 6 |
6 files changed, 67 insertions, 30 deletions
diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py index 9d95f602..99da496d 100644 --- a/mitmproxy/io/compat.py +++ b/mitmproxy/io/compat.py @@ -75,6 +75,8 @@ def convert_018_019(data): data["client_conn"]["cipher_name"] = None data["client_conn"]["tls_version"] = None data["server_conn"]["alpn_proto_negotiated"] = None + if data["server_conn"]["via"]: + data["server_conn"]["via"]["alpn_proto_negotiated"] = None data["mode"] = "regular" data["metadata"] = dict() data["version"] = (0, 19) @@ -96,6 +98,13 @@ def convert_100_200(data): data["server_conn"]["source_address"] = data["server_conn"]["source_address"]["address"] if data["server_conn"]["ip_address"]: data["server_conn"]["ip_address"] = data["server_conn"]["ip_address"]["address"] + + if data["server_conn"]["via"]: + data["server_conn"]["via"]["address"] = data["server_conn"]["via"]["address"]["address"] + data["server_conn"]["via"]["source_address"] = data["server_conn"]["via"]["source_address"]["address"] + if data["server_conn"]["via"]["ip_address"]: + data["server_conn"]["via"]["ip_address"] = data["server_conn"]["via"]["ip_address"]["address"] + return data @@ -129,6 +138,14 @@ def convert_4_5(data): ) data["client_conn"]["id"] = client_connections.setdefault(client_conn_key, str(uuid.uuid4())) data["server_conn"]["id"] = server_connections.setdefault(server_conn_key, str(uuid.uuid4())) + + if data["server_conn"]["via"]: + server_conn_key = ( + data["server_conn"]["via"]["timestamp_start"], + *data["server_conn"]["via"]["source_address"] + ) + data["server_conn"]["via"]["id"] = server_connections.setdefault(server_conn_key, str(uuid.uuid4())) + return data @@ -142,7 +159,7 @@ def _convert_dict_keys(o: Any) -> Any: def _convert_dict_vals(o: dict, values_to_convert: dict) -> dict: for k, v in values_to_convert.items(): if not o or k not in o: - continue + continue # pragma: no cover if v is True: o[k] = strutils.always_str(o[k]) else: diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py index f2c8c473..86f65cfd 100644 --- a/mitmproxy/net/http/url.py +++ b/mitmproxy/net/http/url.py @@ -5,22 +5,6 @@ from typing import Tuple from mitmproxy.net import check -# PY2 workaround -def decode_parse_result(result, enc): - if hasattr(result, "decode"): - return result.decode(enc) - else: - return urllib.parse.ParseResult(*[x.decode(enc) for x in result]) - - -# PY2 workaround -def encode_parse_result(result, enc): - if hasattr(result, "encode"): - return result.encode(enc) - else: - return urllib.parse.ParseResult(*[x.encode(enc) for x in result]) - - def parse(url): """ URL-parsing function that checks that @@ -47,12 +31,12 @@ def parse(url): # this should not raise a ValueError, # but we try to be very forgiving here and accept just everything. - # decode_parse_result(parsed, "ascii") else: host = parsed.hostname.encode("idna") - parsed = encode_parse_result(parsed, "ascii") + if isinstance(parsed, urllib.parse.ParseResult): + parsed = parsed.encode("ascii") - port = parsed.port + port = parsed.port # Returns None if port number invalid in Py3.5. Will throw ValueError in Py3.6 if not port: port = 443 if parsed.scheme == b"https" else 80 @@ -64,8 +48,6 @@ def parse(url): if not check.is_valid_host(host): raise ValueError("Invalid Host") - if not check.is_valid_port(port): - raise ValueError("Invalid Port") return parsed.scheme, host, port, full_path @@ -25,10 +25,7 @@ exclude = mitmproxy/contentviews/wbxml.py mitmproxy/contentviews/xml_html.py mitmproxy/net/tcp.py - mitmproxy/net/http/cookies.py mitmproxy/net/http/encoding.py - mitmproxy/net/http/message.py - mitmproxy/net/http/url.py mitmproxy/proxy/protocol/ mitmproxy/proxy/config.py mitmproxy/proxy/root_context.py @@ -36,7 +33,6 @@ exclude = mitmproxy/tools/ mitmproxy/controller.py mitmproxy/flow.py - mitmproxy/io/compat.py mitmproxy/master.py pathod/pathoc.py pathod/pathod.py @@ -55,7 +51,6 @@ exclude = mitmproxy/exceptions.py mitmproxy/flow.py mitmproxy/io/io.py - mitmproxy/io/compat.py mitmproxy/io/tnetstring.py mitmproxy/log.py mitmproxy/master.py diff --git a/test/mitmproxy/data/dumpfile-018 b/test/mitmproxy/data/dumpfile-018 index abe8b0b1..6a27b5a6 100644 --- a/test/mitmproxy/data/dumpfile-018 +++ b/test/mitmproxy/data/dumpfile-018 @@ -1,4 +1,4 @@ -5243:5:error;0:~11:intercepted;5:false!6:marked;5:false!2:id;36:55367415-10f5-4938-b69f-8a523394f947;7:request;396:10:stickyauth;5:false!7:content;0:,15:timestamp_start;18:1482157523.9086578^9:is_replay;5:false!4:path;1:/,4:host;15:www.example.com,17:first_line_format;8:relative;12:stickycookie;5:false!12:http_version;8:HTTP/1.1,6:method;3:GET,4:port;3:443#13:timestamp_end;18:1482157523.9086578^6:scheme;5:https,7:headers;82:29:10:User-Agent,11:curl/7.35.0,]26:4:Host,15:www.example.com,]15:6:Accept,3:*/*,]]}8:response;1851:6:reason;2:OK,12:http_version;8:HTTP/1.1,13:timestamp_end;17:1482157524.361187^11:status_code;3:200#7:content;1270:<!doctype html> +7816:4:type;4:http;2:id;36:55367415-10f5-4938-b69f-8a523394f947;8:response;1851:15:timestamp_start;17:1482157524.361187^12:http_version;8:HTTP/1.1,7:content;1270:<!doctype html> <html> <head> <title>Example Domain</title> @@ -48,7 +48,7 @@ </div> </body> </html> -,7:headers;410:25:13:Accept-Ranges,5:bytes,]35:13:Cache-Control,14:max-age=604800,]28:12:Content-Type,9:text/html,]40:4:Date,29:Mon, 19 Dec 2016 14:25:24 GMT,]22:4:Etag,11:"359670651",]43:7:Expires,29:Mon, 26 Dec 2016 14:25:24 GMT,]50:13:Last-Modified,29:Fri, 09 Aug 2013 23:54:35 GMT,]27:6:Server,14:ECS (iad/18CB),]26:4:Vary,15:Accept-Encoding,]16:7:X-Cache,3:HIT,]25:17:x-ec-custom-error,1:1,]25:14:Content-Length,4:1270,]]15:timestamp_start;17:1482157524.361187^}4:type;4:http;11:server_conn;2570:15:ssl_established;4:true!7:address;58:7:address;25:15:www.example.com;3:443#]8:use_ipv6;5:false!}10:ip_address;56:7:address;23:13:93.184.216.34;3:443#]8:use_ipv6;5:false!}3:via;0:~14:source_address;57:7:address;24:12:10.67.53.133;5:52775#]8:use_ipv6;5:false!}13:timestamp_end;0:~4:cert;2122:-----BEGIN CERTIFICATE----- +,13:timestamp_end;17:1482157524.361187^11:status_code;3:200#6:reason;2:OK,7:headers;410:25:13:Accept-Ranges,5:bytes,]35:13:Cache-Control,14:max-age=604800,]28:12:Content-Type,9:text/html,]40:4:Date,29:Mon, 19 Dec 2016 14:25:24 GMT,]22:4:Etag,11:"359670651",]43:7:Expires,29:Mon, 26 Dec 2016 14:25:24 GMT,]50:13:Last-Modified,29:Fri, 09 Aug 2013 23:54:35 GMT,]27:6:Server,14:ECS (iad/18CB),]26:4:Vary,15:Accept-Encoding,]16:7:X-Cache,3:HIT,]25:17:x-ec-custom-error,1:1,]25:14:Content-Length,4:1270,]]}7:request;396:9:is_replay;5:false!17:first_line_format;8:relative;4:port;3:443#7:content;0:,12:stickycookie;5:false!6:method;3:GET,7:headers;82:29:10:User-Agent,11:curl/7.35.0,]26:4:Host,15:www.example.com,]15:6:Accept,3:*/*,]]15:timestamp_start;18:1482157523.9086578^12:http_version;8:HTTP/1.1,13:timestamp_end;18:1482157523.9086578^4:path;1:/,10:stickyauth;5:false!4:host;15:www.example.com,6:scheme;5:https,}7:version;13:1:0#2:18#1:2#]5:error;0:~11:intercepted;5:false!11:server_conn;5143:10:ip_address;56:8:use_ipv6;5:false!7:address;23:13:93.184.216.34;3:443#]}15:timestamp_start;18:1482157523.9086578^19:timestamp_tcp_setup;18:1482157524.0081189^15:ssl_established;4:true!14:source_address;57:8:use_ipv6;5:false!7:address;24:12:10.67.53.133;5:52775#]}19:timestamp_ssl_setup;17:1482157524.260993^4:cert;2122:-----BEGIN CERTIFICATE----- MIIF8jCCBNqgAwIBAgIQDmTF+8I2reFLFyrrQceMsDANBgkqhkiG9w0BAQsFADBw MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 d3cuZGlnaWNlcnQuY29tMS8wLQYDVQQDEyZEaWdpQ2VydCBTSEEyIEhpZ2ggQXNz @@ -82,4 +82,38 @@ ieqRbcuFjmqfyPmUv1U9QoI4TQikpw7TZU0zYZANP4C/gj4Ry48/znmUaRvy2kvI l7gRQ21qJTK5suoiYoYNo3J9T+pXPGU7Lydz/HwW+w0DpArtAaukI8aNX4ohFUKS wDSiIIWIWJiJGbEeIO0TIFwEVWTOnbNl/faPXpk5IRXicapqiII= -----END CERTIFICATE----- -,15:timestamp_start;18:1482157523.9086578^3:sni;15:www.example.com;19:timestamp_ssl_setup;17:1482157524.260993^19:timestamp_tcp_setup;18:1482157524.0081189^}11:client_conn;216:15:ssl_established;4:true!7:address;53:7:address;20:9:127.0.0.1;5:52774#]8:use_ipv6;5:false!}10:clientcert;0:~13:timestamp_end;0:~15:timestamp_start;18:1482157522.8949482^19:timestamp_ssl_setup;18:1482157523.9086578^}7:version;13:1:0#2:18#1:2#]}
\ No newline at end of file +,13:timestamp_end;0:~3:via;2570:15:ssl_established;4:true!19:timestamp_tcp_setup;18:1482157524.0081189^19:timestamp_ssl_setup;17:1482157524.260993^3:via;0:~3:sni;15:www.example.com;10:ip_address;56:8:use_ipv6;5:false!7:address;23:13:93.184.216.34;3:443#]}15:timestamp_start;18:1482157523.9086578^14:source_address;57:8:use_ipv6;5:false!7:address;24:12:10.67.53.133;5:52775#]}4:cert;2122:-----BEGIN CERTIFICATE----- +MIIF8jCCBNqgAwIBAgIQDmTF+8I2reFLFyrrQceMsDANBgkqhkiG9w0BAQsFADBw +MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3 +d3cuZGlnaWNlcnQuY29tMS8wLQYDVQQDEyZEaWdpQ2VydCBTSEEyIEhpZ2ggQXNz +dXJhbmNlIFNlcnZlciBDQTAeFw0xNTExMDMwMDAwMDBaFw0xODExMjgxMjAwMDBa +MIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxML +TG9zIEFuZ2VsZXMxPDA6BgNVBAoTM0ludGVybmV0IENvcnBvcmF0aW9uIGZvciBB +c3NpZ25lZCBOYW1lcyBhbmQgTnVtYmVyczETMBEGA1UECxMKVGVjaG5vbG9neTEY +MBYGA1UEAxMPd3d3LmV4YW1wbGUub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A +MIIBCgKCAQEAs0CWL2FjPiXBl61lRfvvE0KzLJmG9LWAC3bcBjgsH6NiVVo2dt6u +Xfzi5bTm7F3K7srfUBYkLO78mraM9qizrHoIeyofrV/n+pZZJauQsPjCPxMEJnRo +D8Z4KpWKX0LyDu1SputoI4nlQ/htEhtiQnuoBfNZxF7WxcxGwEsZuS1KcXIkHl5V +RJOreKFHTaXcB1qcZ/QRaBIv0yhxvK1yBTwWddT4cli6GfHcCe3xGMaSL328Fgs3 +jYrvG29PueB6VJi/tbbPu6qTfwp/H1brqdjh29U52Bhb0fJkM9DWxCP/Cattcc7a +z8EXnCO+LK8vkhw/kAiJWPKx4RBvgy73nwIDAQABo4ICUDCCAkwwHwYDVR0jBBgw +FoAUUWj/kK8CB3U8zNllZGKiErhZcjswHQYDVR0OBBYEFKZPYB4fLdHn8SOgKpUW +5Oia6m5IMIGBBgNVHREEejB4gg93d3cuZXhhbXBsZS5vcmeCC2V4YW1wbGUuY29t +ggtleGFtcGxlLmVkdYILZXhhbXBsZS5uZXSCC2V4YW1wbGUub3Jngg93d3cuZXhh +bXBsZS5jb22CD3d3dy5leGFtcGxlLmVkdYIPd3d3LmV4YW1wbGUubmV0MA4GA1Ud +DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0f +BG4wbDA0oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItaGEtc2Vy +dmVyLWc0LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTIt +aGEtc2VydmVyLWc0LmNybDBMBgNVHSAERTBDMDcGCWCGSAGG/WwBATAqMCgGCCsG +AQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAgGBmeBDAECAjCB +gwYIKwYBBQUHAQEEdzB1MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2Vy +dC5jb20wTQYIKwYBBQUHMAKGQWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9E +aWdpQ2VydFNIQTJIaWdoQXNzdXJhbmNlU2VydmVyQ0EuY3J0MAwGA1UdEwEB/wQC +MAAwDQYJKoZIhvcNAQELBQADggEBAISomhGn2L0LJn5SJHuyVZ3qMIlRCIdvqe0Q +6ls+C8ctRwRO3UU3x8q8OH+2ahxlQmpzdC5al4XQzJLiLjiJ2Q1p+hub8MFiMmVP +PZjb2tZm2ipWVuMRM+zgpRVM6nVJ9F3vFfUSHOb4/JsEIUvPY+d8/Krc+kPQwLvy +ieqRbcuFjmqfyPmUv1U9QoI4TQikpw7TZU0zYZANP4C/gj4Ry48/znmUaRvy2kvI +l7gRQ21qJTK5suoiYoYNo3J9T+pXPGU7Lydz/HwW+w0DpArtAaukI8aNX4ohFUKS +wDSiIIWIWJiJGbEeIO0TIFwEVWTOnbNl/faPXpk5IRXicapqiII= +-----END CERTIFICATE----- +,13:timestamp_end;0:~7:address;58:8:use_ipv6;5:false!7:address;25:15:www.example.com;3:443#]}}7:address;58:8:use_ipv6;5:false!7:address;25:15:www.example.com;3:443#]}3:sni;15:www.example.com;}11:client_conn;216:15:timestamp_start;18:1482157522.8949482^15:ssl_established;4:true!13:timestamp_end;0:~10:clientcert;0:~7:address;53:8:use_ipv6;5:false!7:address;20:9:127.0.0.1;5:52774#]}19:timestamp_ssl_setup;18:1482157523.9086578^}6:marked;5:false!}
\ No newline at end of file diff --git a/test/mitmproxy/net/http/test_cookies.py b/test/mitmproxy/net/http/test_cookies.py index 680a5033..77549d9e 100644 --- a/test/mitmproxy/net/http/test_cookies.py +++ b/test/mitmproxy/net/http/test_cookies.py @@ -269,6 +269,9 @@ def test_refresh_cookie(): c = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure" assert "00:21:38" in cookies.refresh_set_cookie_header(c, 60) + c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2037; Path=/" + assert "expires" not in cookies.refresh_set_cookie_header(c, 60) + c = "foo,bar" with pytest.raises(ValueError): cookies.refresh_set_cookie_header(c, 60) diff --git a/test/mitmproxy/net/http/test_message.py b/test/mitmproxy/net/http/test_message.py index c6036697..512f3199 100644 --- a/test/mitmproxy/net/http/test_message.py +++ b/test/mitmproxy/net/http/test_message.py @@ -48,6 +48,12 @@ class TestMessageData: assert data != 0 + def test_serializable(self): + data1 = tutils.tresp(timestamp_start=42, timestamp_end=42).data + data2 = tutils.tresp().data.from_state(data1.get_state()) # ResponseData.from_state() + + assert data1 == data2 + class TestMessage: |