add MultiDict

This commit introduces MultiDict, a multi-dictionary similar to
ODict, but with improved semantics (as in the Headers class).
MultiDict fixes a few issues that were present in the Request/Response
API. In particular, `request.cookies["foo"] = "bar"` has previously been a
no-op, as the cookies property returned a mutable _copy_ of the cookies.

9 files changed, 56 insertions, 103 deletions

diff --git a/test/mitmproxy/test_examples.py b/test/mitmproxy/test_examples.py
index c401a6b9..d0a258e9 100644
--- a/test/mitmproxy/test_examples.py
+++ b/test/mitmproxy/test_examples.py
@@ -94,14 +94,22 @@ def test_modify_form():
     flow = tutils.tflow(req=netutils.treq(headers=form_header))
     with example("modify_form.py") as ex:
         ex.run("request", flow)
-        assert flow.request.urlencoded_form["mitmproxy"] == ["rocks"]
+        assert flow.request.urlencoded_form["mitmproxy"] == "rocks"
+
+        flow.request.headers["content-type"] = ""
+        ex.run("request", flow)
+        assert list(flow.request.urlencoded_form.items()) == [("foo","bar")]
 
 
 def test_modify_querystring():
     flow = tutils.tflow(req=netutils.treq(path="/search?q=term"))
     with example("modify_querystring.py") as ex:
         ex.run("request", flow)
-        assert flow.request.query["mitmproxy"] == ["rocks"]
+        assert flow.request.query["mitmproxy"] == "rocks"
+
+        flow.request.path = "/"
+        ex.run("request", flow)
+        assert flow.request.query["mitmproxy"] == "rocks"
 
 
 def test_modify_response_body():
diff --git a/test/mitmproxy/test_flow.py b/test/mitmproxy/test_flow.py
index b9c6a2f6..bf417423 100644
--- a/test/mitmproxy/test_flow.py
+++ b/test/mitmproxy/test_flow.py
@@ -1067,60 +1067,6 @@ class TestRequest:
         assert r.url == "https://address:22/path"
         assert r.pretty_url == "https://foo.com:22/path"
 
-    def test_path_components(self):
-        r = HTTPRequest.wrap(netlib.tutils.treq())
-        r.path = "/"
-        assert r.get_path_components() == []
-        r.path = "/foo/bar"
-        assert r.get_path_components() == ["foo", "bar"]
-        q = odict.ODict()
-        q["test"] = ["123"]
-        r.set_query(q)
-        assert r.get_path_components() == ["foo", "bar"]
-
-        r.set_path_components([])
-        assert r.get_path_components() == []
-        r.set_path_components(["foo"])
-        assert r.get_path_components() == ["foo"]
-        r.set_path_components(["/oo"])
-        assert r.get_path_components() == ["/oo"]
-        assert "%2F" in r.path
-
-    def test_getset_form_urlencoded(self):
-        d = odict.ODict([("one", "two"), ("three", "four")])
-        r = HTTPRequest.wrap(netlib.tutils.treq(content=netlib.utils.urlencode(d.lst)))
-        r.headers["content-type"] = "application/x-www-form-urlencoded"
-        assert r.get_form_urlencoded() == d
-
-        d = odict.ODict([("x", "y")])
-        r.set_form_urlencoded(d)
-        assert r.get_form_urlencoded() == d
-
-        r.headers["content-type"] = "foo"
-        assert not r.get_form_urlencoded()
-
-    def test_getset_query(self):
-        r = HTTPRequest.wrap(netlib.tutils.treq())
-        r.path = "/foo?x=y&a=b"
-        q = r.get_query()
-        assert q.lst == [("x", "y"), ("a", "b")]
-
-        r.path = "/"
-        q = r.get_query()
-        assert not q
-
-        r.path = "/?adsfa"
-        q = r.get_query()
-        assert q.lst == [("adsfa", "")]
-
-        r.path = "/foo?x=y&a=b"
-        assert r.get_query()
-        r.set_query(odict.ODict([]))
-        assert not r.get_query()
-        qv = odict.ODict([("a", "b"), ("c", "d")])
-        r.set_query(qv)
-        assert r.get_query() == qv
-
     def test_anticache(self):
         r = HTTPRequest.wrap(netlib.tutils.treq())
         r.headers = Headers()
diff --git a/test/mitmproxy/test_flow_export.py b/test/mitmproxy/test_flow_export.py
index 035f07b7..2b1f897c 100644
--- a/test/mitmproxy/test_flow_export.py
+++ b/test/mitmproxy/test_flow_export.py
@@ -21,7 +21,7 @@ def python_equals(testdata, text):
     assert clean_blanks(text).rstrip() == clean_blanks(d).rstrip()
 
 
-req_get = lambda: netlib.tutils.treq(method='GET', content='')
+req_get = lambda: netlib.tutils.treq(method='GET', content='', path=b"/")
 
 req_post = lambda: netlib.tutils.treq(method='POST', headers=None)
 
diff --git a/test/netlib/http/http1/test_read.py b/test/netlib/http/http1/test_read.py
index 90234070..d8106904 100644
--- a/test/netlib/http/http1/test_read.py
+++ b/test/netlib/http/http1/test_read.py
@@ -261,7 +261,7 @@ class TestReadHeaders(object):
             b"\r\n"
         )
         headers = self._read(data)
-        assert headers.fields == [[b"Header", b"one"], [b"Header2", b"two"]]
+        assert headers.fields == ((b"Header", b"one"), (b"Header2", b"two"))
 
     def test_read_multi(self):
         data = (
@@ -270,7 +270,7 @@ class TestReadHeaders(object):
             b"\r\n"
         )
         headers = self._read(data)
-        assert headers.fields == [[b"Header", b"one"], [b"Header", b"two"]]
+        assert headers.fields == ((b"Header", b"one"), (b"Header", b"two"))
 
     def test_read_continued(self):
         data = (
@@ -280,7 +280,7 @@ class TestReadHeaders(object):
             b"\r\n"
         )
         headers = self._read(data)
-        assert headers.fields == [[b"Header", b"one\r\n two"], [b"Header2", b"three"]]
+        assert headers.fields == ((b"Header", b"one\r\n two"), (b"Header2", b"three"))
 
     def test_read_continued_err(self):
         data = b"\tfoo: bar\r\n"
@@ -300,7 +300,7 @@ class TestReadHeaders(object):
     def test_read_empty_value(self):
         data = b"bar:"
         headers = self._read(data)
-        assert headers.fields == [[b"bar", b""]]
+        assert headers.fields == ((b"bar", b""),)
 
 def test_read_chunked():
     req = treq(content=None)
diff --git a/test/netlib/http/http2/test_connections.py b/test/netlib/http/http2/test_connections.py
index 7b003067..7d240c0e 100644
--- a/test/netlib/http/http2/test_connections.py
+++ b/test/netlib/http/http2/test_connections.py
@@ -312,7 +312,7 @@ class TestReadRequest(tservers.ServerTestBase):
         req = protocol.read_request(NotImplemented)
 
         assert req.stream_id
-        assert req.headers.fields == [[b':method', b'GET'], [b':path', b'/'], [b':scheme', b'https']]
+        assert req.headers.fields == ((b':method', b'GET'), (b':path', b'/'), (b':scheme', b'https'))
         assert req.content == b'foobar'
 
 
@@ -418,7 +418,7 @@ class TestReadResponse(tservers.ServerTestBase):
         assert resp.http_version == "HTTP/2.0"
         assert resp.status_code == 200
         assert resp.reason == ''
-        assert resp.headers.fields == [[b':status', b'200'], [b'etag', b'foobar']]
+        assert resp.headers.fields == ((b':status', b'200'), (b'etag', b'foobar'))
         assert resp.content == b'foobar'
         assert resp.timestamp_end
 
@@ -445,7 +445,7 @@ class TestReadEmptyResponse(tservers.ServerTestBase):
         assert resp.http_version == "HTTP/2.0"
         assert resp.status_code == 200
         assert resp.reason == ''
-        assert resp.headers.fields == [[b':status', b'200'], [b'etag', b'foobar']]
+        assert resp.headers.fields == ((b':status', b'200'), (b'etag', b'foobar'))
         assert resp.content == b''
 
 
diff --git a/test/netlib/http/test_cookies.py b/test/netlib/http/test_cookies.py
index da28850f..e2cee57f 100644
--- a/test/netlib/http/test_cookies.py
+++ b/test/netlib/http/test_cookies.py
@@ -128,10 +128,10 @@ def test_cookie_roundtrips():
     ]
     for s, lst in pairs:
         ret = cookies.parse_cookie_header(s)
-        assert ret.lst == lst
+        assert ret == lst
         s2 = cookies.format_cookie_header(ret)
         ret = cookies.parse_cookie_header(s2)
-        assert ret.lst == lst
+        assert ret == lst
 
 
 def test_parse_set_cookie_pairs():
diff --git a/test/netlib/http/test_headers.py b/test/netlib/http/test_headers.py
index 8c1db9dc..48d3b323 100644
--- a/test/netlib/http/test_headers.py
+++ b/test/netlib/http/test_headers.py
@@ -5,10 +5,10 @@ from netlib.tutils import raises
 class TestHeaders(object):
     def _2host(self):
         return Headers(
-            [
-                [b"Host", b"example.com"],
-                [b"host", b"example.org"]
-            ]
+            (
+                (b"Host", b"example.com"),
+                (b"host", b"example.org")
+            )
         )
 
     def test_init(self):
@@ -38,7 +38,7 @@ class TestHeaders(object):
         assert headers["Host"] == "example.com"
         assert headers["Accept"] == "text/plain"
 
-        with raises(ValueError):
+        with raises(TypeError):
             Headers([[b"Host", u"not-bytes"]])
 
     def test_getitem(self):
diff --git a/test/netlib/http/test_request.py b/test/netlib/http/test_request.py
index 7ed6bd0f..26593ee1 100644
--- a/test/netlib/http/test_request.py
+++ b/test/netlib/http/test_request.py
@@ -12,7 +12,7 @@ from .test_message import _test_decoded_attr, _test_passthrough_attr
 
 class TestRequestData(object):
     def test_init(self):
-        with raises(ValueError if six.PY2 else TypeError):
+        with raises(ValueError):
             treq(headers="foobar")
 
         assert isinstance(treq(headers=None).headers, Headers)
@@ -158,16 +158,17 @@ class TestRequestUtils(object):
 
     def test_get_query(self):
         request = treq()
-        assert request.query is None
+        assert not request.query
 
         request.url = "http://localhost:80/foo?bar=42"
-        assert request.query.lst == [("bar", "42")]
+        assert dict(request.query) == {"bar": "42"}
 
     def test_set_query(self):
-        request = treq(host=b"foo", headers = Headers(host=b"bar"))
-        request.query = ODict([])
-        assert request.host == "foo"
-        assert request.headers["host"] == "bar"
+        request = treq()
+        assert not request.query
+        request.query["foo"] = "bar"
+        assert request.query["foo"] == "bar"
+        assert request.path == "/path?foo=bar"
 
     def test_get_cookies_none(self):
         request = treq()
@@ -177,47 +178,50 @@ class TestRequestUtils(object):
     def test_get_cookies_single(self):
         request = treq()
         request.headers = Headers(cookie="cookiename=cookievalue")
-        result = request.cookies
-        assert len(result) == 1
-        assert result['cookiename'] == ['cookievalue']
+        assert len(request.cookies) == 1
+        assert request.cookies['cookiename'] == 'cookievalue'
 
     def test_get_cookies_double(self):
         request = treq()
         request.headers = Headers(cookie="cookiename=cookievalue;othercookiename=othercookievalue")
         result = request.cookies
         assert len(result) == 2
-        assert result['cookiename'] == ['cookievalue']
-        assert result['othercookiename'] == ['othercookievalue']
+        assert result['cookiename'] == 'cookievalue'
+        assert result['othercookiename'] == 'othercookievalue'
 
     def test_get_cookies_withequalsign(self):
         request = treq()
         request.headers = Headers(cookie="cookiename=coo=kievalue;othercookiename=othercookievalue")
         result = request.cookies
         assert len(result) == 2
-        assert result['cookiename'] == ['coo=kievalue']
-        assert result['othercookiename'] == ['othercookievalue']
+        assert result['cookiename'] == 'coo=kievalue'
+        assert result['othercookiename'] == 'othercookievalue'
 
     def test_set_cookies(self):
         request = treq()
         request.headers = Headers(cookie="cookiename=cookievalue")
         result = request.cookies
-        result["cookiename"] = ["foo"]
-        request.cookies = result
-        assert request.cookies["cookiename"] == ["foo"]
+        result["cookiename"] = "foo"
+        assert request.cookies["cookiename"] == "foo"
 
     def test_get_path_components(self):
         request = treq(path=b"/foo/bar")
-        assert request.path_components == ["foo", "bar"]
+        assert request.path_components == ("foo", "bar")
 
     def test_set_path_components(self):
-        request = treq(host=b"foo", headers = Headers(host=b"bar"))
+        request = treq()
         request.path_components = ["foo", "baz"]
         assert request.path == "/foo/baz"
+
         request.path_components = []
         assert request.path == "/"
-        request.query = ODict([])
-        assert request.host == "foo"
-        assert request.headers["host"] == "bar"
+
+        request.path_components = ["foo", "baz"]
+        request.query["hello"] = "hello"
+        assert request.path_components == ("foo", "baz")
+
+        request.path_components = ["abc"]
+        assert request.path == "/abc?hello=hello"
 
     def test_anticache(self):
         request = treq()
@@ -246,15 +250,15 @@ class TestRequestUtils(object):
         assert "gzip" in request.headers["Accept-Encoding"]
 
     def test_get_urlencoded_form(self):
-        request = treq(content="foobar")
+        request = treq(content="foobar=baz")
         assert request.urlencoded_form is None
 
         request.headers["Content-Type"] = "application/x-www-form-urlencoded"
-        assert request.urlencoded_form == ODict(utils.urldecode(request.content))
+        assert list(request.urlencoded_form.items()) == [("foobar", "baz")]
 
     def test_set_urlencoded_form(self):
         request = treq()
-        request.urlencoded_form = ODict([('foo', 'bar'), ('rab', 'oof')])
+        request.urlencoded_form = [('foo', 'bar'), ('rab', 'oof')]
         assert request.headers["Content-Type"] == "application/x-www-form-urlencoded"
         assert request.content
 
@@ -263,9 +267,4 @@ class TestRequestUtils(object):
         assert request.multipart_form is None
 
         request.headers["Content-Type"] = "multipart/form-data"
-        assert request.multipart_form == ODict(
-            utils.multipartdecode(
-                request.headers,
-                request.content
-            )
-        )
+        assert list(request.multipart_form.items()) == []
diff --git a/test/netlib/http/test_response.py b/test/netlib/http/test_response.py
index 5440176c..37273541 100644
--- a/test/netlib/http/test_response.py
+++ b/test/netlib/http/test_response.py
@@ -13,7 +13,7 @@ from .test_message import _test_passthrough_attr, _test_decoded_attr
 
 class TestResponseData(object):
     def test_init(self):
-        with raises(ValueError if six.PY2 else TypeError):
+        with raises(ValueError):
             tresp(headers="foobar")
 
         assert isinstance(tresp(headers=None).headers, Headers)