Merge pull request #2331 from ujjwal96/test-coverage

Improve Test coverage

6 files changed, 67 insertions, 30 deletions

diff --git a/mitmproxy/io/compat.py b/mitmproxy/io/compat.py
index 9d95f602..99da496d 100644
--- a/mitmproxy/io/compat.py
+++ b/mitmproxy/io/compat.py
@@ -75,6 +75,8 @@ def convert_018_019(data):
     data["client_conn"]["cipher_name"] = None
     data["client_conn"]["tls_version"] = None
     data["server_conn"]["alpn_proto_negotiated"] = None
+    if data["server_conn"]["via"]:
+        data["server_conn"]["via"]["alpn_proto_negotiated"] = None
     data["mode"] = "regular"
     data["metadata"] = dict()
     data["version"] = (0, 19)
@@ -96,6 +98,13 @@ def convert_100_200(data):
     data["server_conn"]["source_address"] = data["server_conn"]["source_address"]["address"]
     if data["server_conn"]["ip_address"]:
         data["server_conn"]["ip_address"] = data["server_conn"]["ip_address"]["address"]
+
+    if data["server_conn"]["via"]:
+        data["server_conn"]["via"]["address"] = data["server_conn"]["via"]["address"]["address"]
+        data["server_conn"]["via"]["source_address"] = data["server_conn"]["via"]["source_address"]["address"]
+        if data["server_conn"]["via"]["ip_address"]:
+            data["server_conn"]["via"]["ip_address"] = data["server_conn"]["via"]["ip_address"]["address"]
+
     return data
 
 
@@ -129,6 +138,14 @@ def convert_4_5(data):
     )
     data["client_conn"]["id"] = client_connections.setdefault(client_conn_key, str(uuid.uuid4()))
     data["server_conn"]["id"] = server_connections.setdefault(server_conn_key, str(uuid.uuid4()))
+
+    if data["server_conn"]["via"]:
+        server_conn_key = (
+            data["server_conn"]["via"]["timestamp_start"],
+            *data["server_conn"]["via"]["source_address"]
+        )
+        data["server_conn"]["via"]["id"] = server_connections.setdefault(server_conn_key, str(uuid.uuid4()))
+
     return data
 
 
@@ -142,7 +159,7 @@ def _convert_dict_keys(o: Any) -> Any:
 def _convert_dict_vals(o: dict, values_to_convert: dict) -> dict:
     for k, v in values_to_convert.items():
         if not o or k not in o:
-            continue
+            continue  # pragma: no cover
         if v is True:
             o[k] = strutils.always_str(o[k])
         else:
diff --git a/mitmproxy/net/http/url.py b/mitmproxy/net/http/url.py
index f2c8c473..86f65cfd 100644
--- a/mitmproxy/net/http/url.py
+++ b/mitmproxy/net/http/url.py
@@ -5,22 +5,6 @@ from typing import Tuple
 from mitmproxy.net import check
 
 
-# PY2 workaround
-def decode_parse_result(result, enc):
-    if hasattr(result, "decode"):
-        return result.decode(enc)
-    else:
-        return urllib.parse.ParseResult(*[x.decode(enc) for x in result])
-
-
-# PY2 workaround
-def encode_parse_result(result, enc):
-    if hasattr(result, "encode"):
-        return result.encode(enc)
-    else:
-        return urllib.parse.ParseResult(*[x.encode(enc) for x in result])
-
-
 def parse(url):
     """
         URL-parsing function that checks that
@@ -47,12 +31,12 @@ def parse(url):
 
         # this should not raise a ValueError,
         # but we try to be very forgiving here and accept just everything.
-        # decode_parse_result(parsed, "ascii")
     else:
         host = parsed.hostname.encode("idna")
-        parsed = encode_parse_result(parsed, "ascii")
+        if isinstance(parsed, urllib.parse.ParseResult):
+            parsed = parsed.encode("ascii")
 
-    port = parsed.port
+    port = parsed.port  # Returns None if port number invalid in Py3.5. Will throw ValueError in Py3.6
     if not port:
         port = 443 if parsed.scheme == b"https" else 80
 
@@ -64,8 +48,6 @@ def parse(url):
 
     if not check.is_valid_host(host):
         raise ValueError("Invalid Host")
-    if not check.is_valid_port(port):
-        raise ValueError("Invalid Port")
 
     return parsed.scheme, host, port, full_path
 
diff --git a/setup.cfg b/setup.cfg
index 993cad31..ebe76efd 100644
--- a/setup.cfg
+++ b/setup.cfg
@@ -25,10 +25,7 @@ exclude =
     mitmproxy/contentviews/wbxml.py
     mitmproxy/contentviews/xml_html.py
     mitmproxy/net/tcp.py
-    mitmproxy/net/http/cookies.py
     mitmproxy/net/http/encoding.py
-    mitmproxy/net/http/message.py
-    mitmproxy/net/http/url.py
     mitmproxy/proxy/protocol/
     mitmproxy/proxy/config.py
     mitmproxy/proxy/root_context.py
@@ -36,7 +33,6 @@ exclude =
     mitmproxy/tools/
     mitmproxy/controller.py
     mitmproxy/flow.py
-    mitmproxy/io/compat.py
     mitmproxy/master.py
     pathod/pathoc.py
     pathod/pathod.py
@@ -55,7 +51,6 @@ exclude =
     mitmproxy/exceptions.py
     mitmproxy/flow.py
     mitmproxy/io/io.py
-    mitmproxy/io/compat.py
     mitmproxy/io/tnetstring.py
     mitmproxy/log.py
     mitmproxy/master.py
diff --git a/test/mitmproxy/data/dumpfile-018 b/test/mitmproxy/data/dumpfile-018
index abe8b0b1..6a27b5a6 100644
--- a/test/mitmproxy/data/dumpfile-018
+++ b/test/mitmproxy/data/dumpfile-018
@@ -1,4 +1,4 @@
-5243:5:error;0:~11:intercepted;5:false!6:marked;5:false!2:id;36:55367415-10f5-4938-b69f-8a523394f947;7:request;396:10:stickyauth;5:false!7:content;0:,15:timestamp_start;18:1482157523.9086578^9:is_replay;5:false!4:path;1:/,4:host;15:www.example.com,17:first_line_format;8:relative;12:stickycookie;5:false!12:http_version;8:HTTP/1.1,6:method;3:GET,4:port;3:443#13:timestamp_end;18:1482157523.9086578^6:scheme;5:https,7:headers;82:29:10:User-Agent,11:curl/7.35.0,]26:4:Host,15:www.example.com,]15:6:Accept,3:*/*,]]}8:response;1851:6:reason;2:OK,12:http_version;8:HTTP/1.1,13:timestamp_end;17:1482157524.361187^11:status_code;3:200#7:content;1270:<!doctype html>
+7816:4:type;4:http;2:id;36:55367415-10f5-4938-b69f-8a523394f947;8:response;1851:15:timestamp_start;17:1482157524.361187^12:http_version;8:HTTP/1.1,7:content;1270:<!doctype html>
 <html>
 <head>
     <title>Example Domain</title>
@@ -48,7 +48,7 @@
 </div>
 </body>
 </html>
-,7:headers;410:25:13:Accept-Ranges,5:bytes,]35:13:Cache-Control,14:max-age=604800,]28:12:Content-Type,9:text/html,]40:4:Date,29:Mon, 19 Dec 2016 14:25:24 GMT,]22:4:Etag,11:"359670651",]43:7:Expires,29:Mon, 26 Dec 2016 14:25:24 GMT,]50:13:Last-Modified,29:Fri, 09 Aug 2013 23:54:35 GMT,]27:6:Server,14:ECS (iad/18CB),]26:4:Vary,15:Accept-Encoding,]16:7:X-Cache,3:HIT,]25:17:x-ec-custom-error,1:1,]25:14:Content-Length,4:1270,]]15:timestamp_start;17:1482157524.361187^}4:type;4:http;11:server_conn;2570:15:ssl_established;4:true!7:address;58:7:address;25:15:www.example.com;3:443#]8:use_ipv6;5:false!}10:ip_address;56:7:address;23:13:93.184.216.34;3:443#]8:use_ipv6;5:false!}3:via;0:~14:source_address;57:7:address;24:12:10.67.53.133;5:52775#]8:use_ipv6;5:false!}13:timestamp_end;0:~4:cert;2122:-----BEGIN CERTIFICATE-----
+,13:timestamp_end;17:1482157524.361187^11:status_code;3:200#6:reason;2:OK,7:headers;410:25:13:Accept-Ranges,5:bytes,]35:13:Cache-Control,14:max-age=604800,]28:12:Content-Type,9:text/html,]40:4:Date,29:Mon, 19 Dec 2016 14:25:24 GMT,]22:4:Etag,11:"359670651",]43:7:Expires,29:Mon, 26 Dec 2016 14:25:24 GMT,]50:13:Last-Modified,29:Fri, 09 Aug 2013 23:54:35 GMT,]27:6:Server,14:ECS (iad/18CB),]26:4:Vary,15:Accept-Encoding,]16:7:X-Cache,3:HIT,]25:17:x-ec-custom-error,1:1,]25:14:Content-Length,4:1270,]]}7:request;396:9:is_replay;5:false!17:first_line_format;8:relative;4:port;3:443#7:content;0:,12:stickycookie;5:false!6:method;3:GET,7:headers;82:29:10:User-Agent,11:curl/7.35.0,]26:4:Host,15:www.example.com,]15:6:Accept,3:*/*,]]15:timestamp_start;18:1482157523.9086578^12:http_version;8:HTTP/1.1,13:timestamp_end;18:1482157523.9086578^4:path;1:/,10:stickyauth;5:false!4:host;15:www.example.com,6:scheme;5:https,}7:version;13:1:0#2:18#1:2#]5:error;0:~11:intercepted;5:false!11:server_conn;5143:10:ip_address;56:8:use_ipv6;5:false!7:address;23:13:93.184.216.34;3:443#]}15:timestamp_start;18:1482157523.9086578^19:timestamp_tcp_setup;18:1482157524.0081189^15:ssl_established;4:true!14:source_address;57:8:use_ipv6;5:false!7:address;24:12:10.67.53.133;5:52775#]}19:timestamp_ssl_setup;17:1482157524.260993^4:cert;2122:-----BEGIN CERTIFICATE-----
 MIIF8jCCBNqgAwIBAgIQDmTF+8I2reFLFyrrQceMsDANBgkqhkiG9w0BAQsFADBw
 MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
 d3cuZGlnaWNlcnQuY29tMS8wLQYDVQQDEyZEaWdpQ2VydCBTSEEyIEhpZ2ggQXNz
@@ -82,4 +82,38 @@ ieqRbcuFjmqfyPmUv1U9QoI4TQikpw7TZU0zYZANP4C/gj4Ry48/znmUaRvy2kvI
 l7gRQ21qJTK5suoiYoYNo3J9T+pXPGU7Lydz/HwW+w0DpArtAaukI8aNX4ohFUKS
 wDSiIIWIWJiJGbEeIO0TIFwEVWTOnbNl/faPXpk5IRXicapqiII=
 -----END CERTIFICATE-----
-,15:timestamp_start;18:1482157523.9086578^3:sni;15:www.example.com;19:timestamp_ssl_setup;17:1482157524.260993^19:timestamp_tcp_setup;18:1482157524.0081189^}11:client_conn;216:15:ssl_established;4:true!7:address;53:7:address;20:9:127.0.0.1;5:52774#]8:use_ipv6;5:false!}10:clientcert;0:~13:timestamp_end;0:~15:timestamp_start;18:1482157522.8949482^19:timestamp_ssl_setup;18:1482157523.9086578^}7:version;13:1:0#2:18#1:2#]}
\ No newline at end of file
+,13:timestamp_end;0:~3:via;2570:15:ssl_established;4:true!19:timestamp_tcp_setup;18:1482157524.0081189^19:timestamp_ssl_setup;17:1482157524.260993^3:via;0:~3:sni;15:www.example.com;10:ip_address;56:8:use_ipv6;5:false!7:address;23:13:93.184.216.34;3:443#]}15:timestamp_start;18:1482157523.9086578^14:source_address;57:8:use_ipv6;5:false!7:address;24:12:10.67.53.133;5:52775#]}4:cert;2122:-----BEGIN CERTIFICATE-----
+MIIF8jCCBNqgAwIBAgIQDmTF+8I2reFLFyrrQceMsDANBgkqhkiG9w0BAQsFADBw
+MQswCQYDVQQGEwJVUzEVMBMGA1UEChMMRGlnaUNlcnQgSW5jMRkwFwYDVQQLExB3
+d3cuZGlnaWNlcnQuY29tMS8wLQYDVQQDEyZEaWdpQ2VydCBTSEEyIEhpZ2ggQXNz
+dXJhbmNlIFNlcnZlciBDQTAeFw0xNTExMDMwMDAwMDBaFw0xODExMjgxMjAwMDBa
+MIGlMQswCQYDVQQGEwJVUzETMBEGA1UECBMKQ2FsaWZvcm5pYTEUMBIGA1UEBxML
+TG9zIEFuZ2VsZXMxPDA6BgNVBAoTM0ludGVybmV0IENvcnBvcmF0aW9uIGZvciBB
+c3NpZ25lZCBOYW1lcyBhbmQgTnVtYmVyczETMBEGA1UECxMKVGVjaG5vbG9neTEY
+MBYGA1UEAxMPd3d3LmV4YW1wbGUub3JnMIIBIjANBgkqhkiG9w0BAQEFAAOCAQ8A
+MIIBCgKCAQEAs0CWL2FjPiXBl61lRfvvE0KzLJmG9LWAC3bcBjgsH6NiVVo2dt6u
+Xfzi5bTm7F3K7srfUBYkLO78mraM9qizrHoIeyofrV/n+pZZJauQsPjCPxMEJnRo
+D8Z4KpWKX0LyDu1SputoI4nlQ/htEhtiQnuoBfNZxF7WxcxGwEsZuS1KcXIkHl5V
+RJOreKFHTaXcB1qcZ/QRaBIv0yhxvK1yBTwWddT4cli6GfHcCe3xGMaSL328Fgs3
+jYrvG29PueB6VJi/tbbPu6qTfwp/H1brqdjh29U52Bhb0fJkM9DWxCP/Cattcc7a
+z8EXnCO+LK8vkhw/kAiJWPKx4RBvgy73nwIDAQABo4ICUDCCAkwwHwYDVR0jBBgw
+FoAUUWj/kK8CB3U8zNllZGKiErhZcjswHQYDVR0OBBYEFKZPYB4fLdHn8SOgKpUW
+5Oia6m5IMIGBBgNVHREEejB4gg93d3cuZXhhbXBsZS5vcmeCC2V4YW1wbGUuY29t
+ggtleGFtcGxlLmVkdYILZXhhbXBsZS5uZXSCC2V4YW1wbGUub3Jngg93d3cuZXhh
+bXBsZS5jb22CD3d3dy5leGFtcGxlLmVkdYIPd3d3LmV4YW1wbGUubmV0MA4GA1Ud
+DwEB/wQEAwIFoDAdBgNVHSUEFjAUBggrBgEFBQcDAQYIKwYBBQUHAwIwdQYDVR0f
+BG4wbDA0oDKgMIYuaHR0cDovL2NybDMuZGlnaWNlcnQuY29tL3NoYTItaGEtc2Vy
+dmVyLWc0LmNybDA0oDKgMIYuaHR0cDovL2NybDQuZGlnaWNlcnQuY29tL3NoYTIt
+aGEtc2VydmVyLWc0LmNybDBMBgNVHSAERTBDMDcGCWCGSAGG/WwBATAqMCgGCCsG
+AQUFBwIBFhxodHRwczovL3d3dy5kaWdpY2VydC5jb20vQ1BTMAgGBmeBDAECAjCB
+gwYIKwYBBQUHAQEEdzB1MCQGCCsGAQUFBzABhhhodHRwOi8vb2NzcC5kaWdpY2Vy
+dC5jb20wTQYIKwYBBQUHMAKGQWh0dHA6Ly9jYWNlcnRzLmRpZ2ljZXJ0LmNvbS9E
+aWdpQ2VydFNIQTJIaWdoQXNzdXJhbmNlU2VydmVyQ0EuY3J0MAwGA1UdEwEB/wQC
+MAAwDQYJKoZIhvcNAQELBQADggEBAISomhGn2L0LJn5SJHuyVZ3qMIlRCIdvqe0Q
+6ls+C8ctRwRO3UU3x8q8OH+2ahxlQmpzdC5al4XQzJLiLjiJ2Q1p+hub8MFiMmVP
+PZjb2tZm2ipWVuMRM+zgpRVM6nVJ9F3vFfUSHOb4/JsEIUvPY+d8/Krc+kPQwLvy
+ieqRbcuFjmqfyPmUv1U9QoI4TQikpw7TZU0zYZANP4C/gj4Ry48/znmUaRvy2kvI
+l7gRQ21qJTK5suoiYoYNo3J9T+pXPGU7Lydz/HwW+w0DpArtAaukI8aNX4ohFUKS
+wDSiIIWIWJiJGbEeIO0TIFwEVWTOnbNl/faPXpk5IRXicapqiII=
+-----END CERTIFICATE-----
+,13:timestamp_end;0:~7:address;58:8:use_ipv6;5:false!7:address;25:15:www.example.com;3:443#]}}7:address;58:8:use_ipv6;5:false!7:address;25:15:www.example.com;3:443#]}3:sni;15:www.example.com;}11:client_conn;216:15:timestamp_start;18:1482157522.8949482^15:ssl_established;4:true!13:timestamp_end;0:~10:clientcert;0:~7:address;53:8:use_ipv6;5:false!7:address;20:9:127.0.0.1;5:52774#]}19:timestamp_ssl_setup;18:1482157523.9086578^}6:marked;5:false!}
\ No newline at end of file
diff --git a/test/mitmproxy/net/http/test_cookies.py b/test/mitmproxy/net/http/test_cookies.py
index 680a5033..77549d9e 100644
--- a/test/mitmproxy/net/http/test_cookies.py
+++ b/test/mitmproxy/net/http/test_cookies.py
@@ -269,6 +269,9 @@ def test_refresh_cookie():
     c = "MOO=BAR; Expires=Tue, 08-Mar-2011 00:20:38 GMT; Path=foo.com; Secure"
     assert "00:21:38" in cookies.refresh_set_cookie_header(c, 60)
 
+    c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2037; Path=/"
+    assert "expires" not in cookies.refresh_set_cookie_header(c, 60)
+
     c = "foo,bar"
     with pytest.raises(ValueError):
         cookies.refresh_set_cookie_header(c, 60)
diff --git a/test/mitmproxy/net/http/test_message.py b/test/mitmproxy/net/http/test_message.py
index c6036697..512f3199 100644
--- a/test/mitmproxy/net/http/test_message.py
+++ b/test/mitmproxy/net/http/test_message.py
@@ -48,6 +48,12 @@ class TestMessageData:
 
         assert data != 0
 
+    def test_serializable(self):
+        data1 = tutils.tresp(timestamp_start=42, timestamp_end=42).data
+        data2 = tutils.tresp().data.from_state(data1.get_state())  # ResponseData.from_state()
+
+        assert data1 == data2
+
 
 class TestMessage: