finish netlib.http.http1 refactor

3 files changed, 404 insertions, 466 deletions

diff --git a/test/http/http1/test_assemble.py b/test/http/http1/test_assemble.py
new file mode 100644
index 00000000..8a0a54f1
--- /dev/null
+++ b/test/http/http1/test_assemble.py
@@ -0,0 +1,91 @@
+from __future__ import absolute_import, print_function, division
+from netlib.exceptions import HttpException
+from netlib.http import CONTENT_MISSING, Headers
+from netlib.http.http1.assemble import (
+    assemble_request, assemble_request_head, assemble_response,
+    assemble_response_head, _assemble_request_line, _assemble_request_headers,
+    _assemble_response_headers
+)
+from netlib.tutils import treq, raises, tresp
+
+
+def test_assemble_request():
+    c = assemble_request(treq()) == (
+        b"GET /path HTTP/1.1\r\n"
+        b"header: qvalue\r\n"
+        b"Host: address:22\r\n"
+        b"Content-Length: 7\r\n"
+        b"\r\n"
+        b"content"
+    )
+
+    with raises(HttpException):
+        assemble_request(treq(body=CONTENT_MISSING))
+
+
+def test_assemble_request_head():
+    c = assemble_request_head(treq())
+    assert b"GET" in c
+    assert b"qvalue" in c
+    assert b"content" not in c
+
+
+def test_assemble_response():
+    c = assemble_response(tresp()) == (
+        b"HTTP/1.1 200 OK\r\n"
+        b"header-response: svalue\r\n"
+        b"Content-Length: 7\r\n"
+        b"\r\n"
+        b"message"
+    )
+
+    with raises(HttpException):
+        assemble_response(tresp(body=CONTENT_MISSING))
+
+
+def test_assemble_response_head():
+    c = assemble_response_head(tresp())
+    assert b"200" in c
+    assert b"svalue" in c
+    assert b"message" not in c
+
+
+def test_assemble_request_line():
+    assert _assemble_request_line(treq()) == b"GET /path HTTP/1.1"
+
+    authority_request = treq(method=b"CONNECT", form_in="authority")
+    assert _assemble_request_line(authority_request) == b"CONNECT address:22 HTTP/1.1"
+
+    absolute_request = treq(form_in="absolute")
+    assert _assemble_request_line(absolute_request) == b"GET http://address:22/path HTTP/1.1"
+
+    with raises(RuntimeError):
+        _assemble_request_line(treq(), "invalid_form")
+
+
+def test_assemble_request_headers():
+    # https://github.com/mitmproxy/mitmproxy/issues/186
+    r = treq(body=b"")
+    r.headers[b"Transfer-Encoding"] = b"chunked"
+    c = _assemble_request_headers(r)
+    assert b"Content-Length" in c
+    assert b"Transfer-Encoding" not in c
+
+    assert b"Host" in _assemble_request_headers(treq(headers=Headers()))
+
+    assert b"Proxy-Connection" not in _assemble_request_headers(
+        treq(headers=Headers(Proxy_Connection="42"))
+    )
+
+
+def test_assemble_response_headers():
+    # https://github.com/mitmproxy/mitmproxy/issues/186
+    r = tresp(body=b"")
+    r.headers["Transfer-Encoding"] = b"chunked"
+    c = _assemble_response_headers(r)
+    assert b"Content-Length" in c
+    assert b"Transfer-Encoding" not in c
+
+    assert b"Proxy-Connection" not in _assemble_response_headers(
+        tresp(headers=Headers(Proxy_Connection=b"42"))
+    )
diff --git a/test/http/http1/test_protocol.py b/test/http/http1/test_protocol.py
index bdcba5cb..e69de29b 100644
--- a/test/http/http1/test_protocol.py
+++ b/test/http/http1/test_protocol.py
@@ -1,466 +0,0 @@
-from io import BytesIO
-import textwrap
-from http.http1.protocol import _parse_authority_form
-from netlib.exceptions import HttpSyntaxException, HttpReadDisconnect, HttpException
-
-from netlib import http, tcp, tutils
-from netlib.http import semantics, Headers
-from netlib.http.http1 import HTTP1Protocol, read_message_body, read_request, \
-    read_message_body_chunked, expected_http_body_size
-from ... import tservers
-
-
-class NoContentLengthHTTPHandler(tcp.BaseHandler):
-    def handle(self):
-        self.wfile.write("HTTP/1.1 200 OK\r\n\r\nbar\r\n\r\n")
-        self.wfile.flush()
-
-
-def mock_protocol(data=''):
-    rfile = BytesIO(data)
-    wfile = BytesIO()
-    return HTTP1Protocol(rfile=rfile, wfile=wfile)
-
-
-def match_http_string(data):
-    return textwrap.dedent(data).strip().replace('\n', '\r\n')
-
-
-def test_stripped_chunked_encoding_no_content():
-    """
-    https://github.com/mitmproxy/mitmproxy/issues/186
-    """
-
-    r = tutils.treq(content="")
-    r.headers["Transfer-Encoding"] = "chunked"
-    assert "Content-Length" in mock_protocol()._assemble_request_headers(r)
-
-    r = tutils.tresp(content="")
-    r.headers["Transfer-Encoding"] = "chunked"
-    assert "Content-Length" in mock_protocol()._assemble_response_headers(r)
-
-
-def test_read_chunked():
-    req = tutils.treq(None)
-    req.headers["Transfer-Encoding"] = "chunked"
-
-    data = b"1\r\na\r\n0\r\n"
-    with tutils.raises(HttpSyntaxException):
-        read_message_body(BytesIO(data), req)
-
-    data = b"1\r\na\r\n0\r\n\r\n"
-    assert read_message_body(BytesIO(data), req) == b"a"
-
-    data = b"\r\n\r\n1\r\na\r\n1\r\nb\r\n0\r\n\r\n"
-    assert read_message_body(BytesIO(data), req) == b"ab"
-
-    data = b"\r\n"
-    with tutils.raises("closed prematurely"):
-        read_message_body(BytesIO(data), req)
-
-    data = b"1\r\nfoo"
-    with tutils.raises("malformed chunked body"):
-        read_message_body(BytesIO(data), req)
-
-    data = b"foo\r\nfoo"
-    with tutils.raises(HttpSyntaxException):
-        read_message_body(BytesIO(data), req)
-
-    data = b"5\r\naaaaa\r\n0\r\n\r\n"
-    with tutils.raises("too large"):
-        read_message_body(BytesIO(data), req, limit=2)
-
-
-def test_connection_close():
-    headers = Headers()
-    assert HTTP1Protocol.connection_close((1, 0), headers)
-    assert not HTTP1Protocol.connection_close((1, 1), headers)
-
-    headers["connection"] = "keep-alive"
-    assert not HTTP1Protocol.connection_close((1, 1), headers)
-
-    headers["connection"] = "close"
-    assert HTTP1Protocol.connection_close((1, 1), headers)
-
-
-def test_read_http_body_request():
-    headers = Headers()
-    data = "testing"
-    assert mock_protocol(data).read_http_body(headers, None, "GET", None, True) == ""
-
-
-def test_read_http_body_response():
-    headers = Headers()
-    data = "testing"
-    assert mock_protocol(data).read_http_body(headers, None, "GET", 200, False) == "testing"
-
-
-def test_read_http_body():
-    # test default case
-    headers = Headers()
-    headers["content-length"] = "7"
-    data = "testing"
-    assert mock_protocol(data).read_http_body(headers, None, "GET", 200, False) == "testing"
-
-    # test content length: invalid header
-    headers["content-length"] = "foo"
-    data = "testing"
-    tutils.raises(
-        http.HttpError,
-        mock_protocol(data).read_http_body,
-        headers, None, "GET", 200, False
-    )
-
-    # test content length: invalid header #2
-    headers["content-length"] = "-1"
-    data = "testing"
-    tutils.raises(
-        http.HttpError,
-        mock_protocol(data).read_http_body,
-        headers, None, "GET", 200, False
-    )
-
-    # test content length: content length > actual content
-    headers["content-length"] = "5"
-    data = "testing"
-    tutils.raises(
-        http.HttpError,
-        mock_protocol(data).read_http_body,
-        headers, 4, "GET", 200, False
-    )
-
-    # test content length: content length < actual content
-    data = "testing"
-    assert len(mock_protocol(data).read_http_body(headers, None, "GET", 200, False)) == 5
-
-    # test no content length: limit > actual content
-    headers = Headers()
-    data = "testing"
-    assert len(mock_protocol(data).read_http_body(headers, 100, "GET", 200, False)) == 7
-
-    # test no content length: limit < actual content
-    data = "testing"
-    tutils.raises(
-        http.HttpError,
-        mock_protocol(data).read_http_body,
-        headers, 4, "GET", 200, False
-    )
-
-    # test chunked
-    headers = Headers()
-    headers["transfer-encoding"] = "chunked"
-    data = "5\r\naaaaa\r\n0\r\n\r\n"
-    assert mock_protocol(data).read_http_body(headers, 100, "GET", 200, False) == "aaaaa"
-
-
-def test_expected_http_body_size():
-    # gibber in the content-length field
-    headers = Headers(content_length="foo")
-    with tutils.raises(HttpSyntaxException):
-        expected_http_body_size(headers, False, "GET", 200) is None
-    # negative number in the content-length field
-    headers = Headers(content_length="-7")
-    with tutils.raises(HttpSyntaxException):
-        expected_http_body_size(headers, False, "GET", 200) is None
-    # explicit length
-    headers = Headers(content_length="5")
-    assert expected_http_body_size(headers, False, "GET", 200) == 5
-    # no length
-    headers = Headers()
-    assert expected_http_body_size(headers, False, "GET", 200) == -1
-    # no length request
-    headers = Headers()
-    assert expected_http_body_size(headers, True, "GET", None) == 0
-    # expect header
-    headers = Headers(content_length="5", expect="100-continue")
-    assert expected_http_body_size(headers, True, "GET", None) == 0
-
-
-def test_parse_init_connect():
-    assert _parse_authority_form(b"CONNECT host.com:443 HTTP/1.0")
-    tutils.raises(ValueError,_parse_authority_form, b"\0host.com:443")
-    tutils.raises(ValueError,_parse_authority_form, b"host.com:444444")
-    tutils.raises(ValueError,_parse_authority_form, b"CONNECT host.com443 HTTP/1.0")
-    tutils.raises(ValueError,_parse_authority_form, b"CONNECT host.com:foo HTTP/1.0")
-
-
-def test_parse_init_proxy():
-    u = b"GET http://foo.com:8888/test HTTP/1.1"
-    m, s, h, po, pa, httpversion = HTTP1Protocol._parse_absolute_form(u)
-    assert m == "GET"
-    assert s == "http"
-    assert h == "foo.com"
-    assert po == 8888
-    assert pa == "/test"
-    assert httpversion == (1, 1)
-
-    u = "G\xfeET http://foo.com:8888/test HTTP/1.1"
-    assert not HTTP1Protocol._parse_absolute_form(u)
-
-    with tutils.raises(ValueError):
-        assert not HTTP1Protocol._parse_absolute_form("invalid")
-    with tutils.raises(ValueError):
-        assert not HTTP1Protocol._parse_absolute_form("GET invalid HTTP/1.1")
-    with tutils.raises(ValueError):
-        assert not HTTP1Protocol._parse_absolute_form("GET http://foo.com:8888/test foo/1.1")
-
-
-def test_parse_init_http():
-    u = "GET /test HTTP/1.1"
-    m, u, httpversion = HTTP1Protocol._parse_init_http(u)
-    assert m == "GET"
-    assert u == "/test"
-    assert httpversion == (1, 1)
-
-    u = "G\xfeET /test HTTP/1.1"
-    assert not HTTP1Protocol._parse_init_http(u)
-
-    assert not HTTP1Protocol._parse_init_http("invalid")
-    assert not HTTP1Protocol._parse_init_http("GET invalid HTTP/1.1")
-    assert not HTTP1Protocol._parse_init_http("GET /test foo/1.1")
-    assert not HTTP1Protocol._parse_init_http("GET /test\xc0 HTTP/1.1")
-
-
-class TestReadHeaders:
-
-    def _read(self, data, verbatim=False):
-        if not verbatim:
-            data = textwrap.dedent(data)
-            data = data.strip()
-        return mock_protocol(data).read_headers()
-
-    def test_read_simple(self):
-        data = """
-            Header: one
-            Header2: two
-            \r\n
-        """
-        headers = self._read(data)
-        assert headers.fields == [["Header", "one"], ["Header2", "two"]]
-
-    def test_read_multi(self):
-        data = """
-            Header: one
-            Header: two
-            \r\n
-        """
-        headers = self._read(data)
-        assert headers.fields == [["Header", "one"], ["Header", "two"]]
-
-    def test_read_continued(self):
-        data = """
-            Header: one
-            \ttwo
-            Header2: three
-            \r\n
-        """
-        headers = self._read(data)
-        assert headers.fields == [["Header", "one\r\n two"], ["Header2", "three"]]
-
-    def test_read_continued_err(self):
-        data = "\tfoo: bar\r\n"
-        assert self._read(data, True) is None
-
-    def test_read_err(self):
-        data = """
-            foo
-        """
-        assert self._read(data) is None
-
-
-class TestReadRequest(object):
-
-    def tst(self, data, **kwargs):
-        return mock_protocol(data).read_request(**kwargs)
-
-    def test_invalid(self):
-        tutils.raises(
-            "bad http request",
-            self.tst,
-            "xxx"
-        )
-        tutils.raises(
-            "bad http request line",
-            self.tst,
-            "get /\xff HTTP/1.1"
-        )
-        tutils.raises(
-            "invalid headers",
-            self.tst,
-            "get / HTTP/1.1\r\nfoo"
-        )
-        tutils.raises(
-            HttpReadDisconnect,
-            self.tst,
-            "\r\n"
-        )
-
-    def test_asterisk_form_in(self):
-        v = self.tst("OPTIONS * HTTP/1.1")
-        assert v.form_in == "relative"
-        assert v.method == "OPTIONS"
-
-    def test_absolute_form_in(self):
-        tutils.raises(
-            "Bad HTTP request line",
-            self.tst,
-            "GET oops-no-protocol.com HTTP/1.1"
-        )
-        v = self.tst("GET http://address:22/ HTTP/1.1")
-        assert v.form_in == "absolute"
-        assert v.port == 22
-        assert v.host == "address"
-        assert v.scheme == "http"
-
-    def test_connect(self):
-        tutils.raises(
-            "Bad HTTP request line",
-            self.tst,
-            "CONNECT oops-no-port.com HTTP/1.1"
-        )
-        v = self.tst("CONNECT foo.com:443 HTTP/1.1")
-        assert v.form_in == "authority"
-        assert v.method == "CONNECT"
-        assert v.port == 443
-        assert v.host == "foo.com"
-
-    def test_expect(self):
-        data = (
-            b"GET / HTTP/1.1\r\n"
-            b"Content-Length: 3\r\n"
-            b"Expect: 100-continue\r\n"
-            b"\r\n"
-            b"foobar"
-        )
-
-        rfile = BytesIO(data)
-        r = read_request(rfile)
-        assert r.body == b""
-        assert rfile.read(-1) == b"foobar"
-
-
-class TestReadResponse(object):
-    def tst(self, data, method, body_size_limit, include_body=True):
-        data = textwrap.dedent(data)
-        return mock_protocol(data).read_response(
-            method, body_size_limit, include_body=include_body
-        )
-
-    def test_errors(self):
-        tutils.raises("server disconnect", self.tst, "", "GET", None)
-        tutils.raises("invalid server response", self.tst, "foo", "GET", None)
-
-    def test_simple(self):
-        data = """
-            HTTP/1.1 200
-        """
-        assert self.tst(data, "GET", None) == http.Response(
-            (1, 1), 200, '', Headers(), ''
-        )
-
-    def test_simple_message(self):
-        data = """
-            HTTP/1.1 200 OK
-        """
-        assert self.tst(data, "GET", None) == http.Response(
-            (1, 1), 200, 'OK', Headers(), ''
-        )
-
-    def test_invalid_http_version(self):
-        data = """
-            HTTP/x 200 OK
-        """
-        tutils.raises("invalid http version", self.tst, data, "GET", None)
-
-    def test_invalid_status_code(self):
-        data = """
-            HTTP/1.1 xx OK
-        """
-        tutils.raises("invalid server response", self.tst, data, "GET", None)
-
-    def test_valid_with_continue(self):
-        data = """
-            HTTP/1.1 100 CONTINUE
-
-            HTTP/1.1 200 OK
-        """
-        assert self.tst(data, "GET", None) == http.Response(
-            (1, 1), 100, 'CONTINUE', Headers(), ''
-        )
-
-    def test_simple_body(self):
-        data = """
-            HTTP/1.1 200 OK
-            Content-Length: 3
-
-            foo
-        """
-        assert self.tst(data, "GET", None).body == 'foo'
-        assert self.tst(data, "HEAD", None).body == ''
-
-    def test_invalid_headers(self):
-        data = """
-            HTTP/1.1 200 OK
-            \tContent-Length: 3
-
-            foo
-        """
-        tutils.raises("invalid headers", self.tst, data, "GET", None)
-
-    def test_without_body(self):
-        data = """
-            HTTP/1.1 200 OK
-            Content-Length: 3
-
-            foo
-        """
-        assert self.tst(data, "GET", None, include_body=False).body is None
-
-
-class TestReadResponseNoContentLength(tservers.ServerTestBase):
-    handler = NoContentLengthHTTPHandler
-
-    def test_no_content_length(self):
-        c = tcp.TCPClient(("127.0.0.1", self.port))
-        c.connect()
-        resp = HTTP1Protocol(c).read_response("GET", None)
-        assert resp.body == "bar\r\n\r\n"
-
-
-class TestAssembleRequest(object):
-    def test_simple(self):
-        req = tutils.treq()
-        b = HTTP1Protocol().assemble_request(req)
-        assert b == match_http_string("""
-            GET /path HTTP/1.1
-            header: qvalue
-            Host: address:22
-            Content-Length: 7
-
-            content""")
-
-    def test_body_missing(self):
-        req = tutils.treq(content=semantics.CONTENT_MISSING)
-        tutils.raises(http.HttpError, HTTP1Protocol().assemble_request, req)
-
-    def test_not_a_request(self):
-        tutils.raises(AssertionError, HTTP1Protocol().assemble_request, 'foo')
-
-
-class TestAssembleResponse(object):
-    def test_simple(self):
-        resp = tutils.tresp()
-        b = HTTP1Protocol().assemble_response(resp)
-        assert b == match_http_string("""
-            HTTP/1.1 200 OK
-            header_response: svalue
-            Content-Length: 7
-
-            message""")
-
-    def test_body_missing(self):
-        resp = tutils.tresp(content=semantics.CONTENT_MISSING)
-        tutils.raises(http.HttpError, HTTP1Protocol().assemble_response, resp)
-
-    def test_not_a_request(self):
-        tutils.raises(AssertionError, HTTP1Protocol().assemble_response, 'foo')
diff --git a/test/http/http1/test_read.py b/test/http/http1/test_read.py
new file mode 100644
index 00000000..5e6680af
--- /dev/null
+++ b/test/http/http1/test_read.py
@@ -0,0 +1,313 @@
+from __future__ import absolute_import, print_function, division
+from io import BytesIO
+import textwrap
+
+from mock import Mock
+
+from netlib.exceptions import HttpException, HttpSyntaxException, HttpReadDisconnect
+from netlib.http import Headers
+from netlib.http.http1.read import (
+    read_request, read_response, read_request_head,
+    read_response_head, read_body, connection_close, expected_http_body_size, _get_first_line,
+    _read_request_line, _parse_authority_form, _read_response_line, _check_http_version,
+    _read_headers, _read_chunked
+)
+from netlib.tutils import treq, tresp, raises
+
+
+def test_read_request():
+    rfile = BytesIO(b"GET / HTTP/1.1\r\n\r\nskip")
+    r = read_request(rfile)
+    assert r.method == b"GET"
+    assert r.body == b""
+    assert r.timestamp_end
+    assert rfile.read() == b"skip"
+
+
+def test_read_request_head():
+    rfile = BytesIO(
+        b"GET / HTTP/1.1\r\n"
+        b"Content-Length: 4\r\n"
+        b"\r\n"
+        b"skip"
+    )
+    rfile.reset_timestamps = Mock()
+    rfile.first_byte_timestamp = 42
+    r = read_request_head(rfile)
+    assert r.method == b"GET"
+    assert r.headers["Content-Length"] == b"4"
+    assert r.body is None
+    assert rfile.reset_timestamps.called
+    assert r.timestamp_start == 42
+    assert rfile.read() == b"skip"
+
+
+def test_read_response():
+    req = treq()
+    rfile = BytesIO(b"HTTP/1.1 418 I'm a teapot\r\n\r\nbody")
+    r = read_response(rfile, req)
+    assert r.status_code == 418
+    assert r.body == b"body"
+    assert r.timestamp_end
+
+
+def test_read_response_head():
+    rfile = BytesIO(
+        b"HTTP/1.1 418 I'm a teapot\r\n"
+        b"Content-Length: 4\r\n"
+        b"\r\n"
+        b"skip"
+    )
+    rfile.reset_timestamps = Mock()
+    rfile.first_byte_timestamp = 42
+    r = read_response_head(rfile)
+    assert r.status_code == 418
+    assert r.headers["Content-Length"] == b"4"
+    assert r.body is None
+    assert rfile.reset_timestamps.called
+    assert r.timestamp_start == 42
+    assert rfile.read() == b"skip"
+
+
+class TestReadBody(object):
+    def test_chunked(self):
+        rfile = BytesIO(b"3\r\nfoo\r\n0\r\n\r\nbar")
+        body = b"".join(read_body(rfile, None))
+        assert body == b"foo"
+        assert rfile.read() == b"bar"
+
+
+    def test_known_size(self):
+        rfile = BytesIO(b"foobar")
+        body = b"".join(read_body(rfile, 3))
+        assert body == b"foo"
+        assert rfile.read() == b"bar"
+
+
+    def test_known_size_limit(self):
+        rfile = BytesIO(b"foobar")
+        with raises(HttpException):
+            b"".join(read_body(rfile, 3, 2))
+
+    def test_known_size_too_short(self):
+        rfile = BytesIO(b"foo")
+        with raises(HttpException):
+            b"".join(read_body(rfile, 6))
+
+    def test_unknown_size(self):
+        rfile = BytesIO(b"foobar")
+        body = b"".join(read_body(rfile, -1))
+        assert body == b"foobar"
+
+
+    def test_unknown_size_limit(self):
+        rfile = BytesIO(b"foobar")
+        with raises(HttpException):
+            b"".join(read_body(rfile, -1, 3))
+
+
+def test_connection_close():
+    headers = Headers()
+    assert connection_close((1, 0), headers)
+    assert not connection_close((1, 1), headers)
+
+    headers["connection"] = "keep-alive"
+    assert not connection_close((1, 1), headers)
+
+    headers["connection"] = "close"
+    assert connection_close((1, 1), headers)
+
+
+def test_expected_http_body_size():
+    # Expect: 100-continue
+    assert expected_http_body_size(
+        treq(headers=Headers(expect=b"100-continue", content_length=b"42"))
+    ) == 0
+
+    # http://tools.ietf.org/html/rfc7230#section-3.3
+    assert expected_http_body_size(
+        treq(method=b"HEAD"),
+        tresp(headers=Headers(content_length=b"42"))
+    ) == 0
+    assert expected_http_body_size(
+        treq(method=b"CONNECT"),
+        tresp()
+    ) == 0
+    for code in (100, 204, 304):
+        assert expected_http_body_size(
+            treq(),
+            tresp(status_code=code)
+        ) == 0
+
+    # chunked
+    assert expected_http_body_size(
+        treq(headers=Headers(transfer_encoding=b"chunked")),
+    ) is None
+
+    # explicit length
+    for l in (b"foo", b"-7"):
+        with raises(HttpSyntaxException):
+            expected_http_body_size(
+                treq(headers=Headers(content_length=l))
+            )
+    assert expected_http_body_size(
+        treq(headers=Headers(content_length=b"42"))
+    ) == 42
+
+    # no length
+    assert expected_http_body_size(
+        treq()
+    ) == 0
+    assert expected_http_body_size(
+        treq(), tresp()
+    ) == -1
+
+
+def test_get_first_line():
+    rfile = BytesIO(b"foo\r\nbar")
+    assert _get_first_line(rfile) == b"foo"
+
+    rfile = BytesIO(b"\r\nfoo\r\nbar")
+    assert _get_first_line(rfile) == b"foo"
+
+    with raises(HttpReadDisconnect):
+        rfile = BytesIO(b"")
+        _get_first_line(rfile)
+
+    with raises(HttpSyntaxException):
+        rfile = BytesIO(b"GET /\xff HTTP/1.1")
+        _get_first_line(rfile)
+
+
+def test_read_request_line():
+    def t(b):
+        return _read_request_line(BytesIO(b))
+
+    assert (t(b"GET / HTTP/1.1") ==
+            ("relative", b"GET", None, None, None, b"/", b"HTTP/1.1"))
+    assert (t(b"OPTIONS * HTTP/1.1") ==
+            ("relative", b"OPTIONS", None, None, None, b"*", b"HTTP/1.1"))
+    assert (t(b"CONNECT foo:42 HTTP/1.1") ==
+            ("authority", b"CONNECT", None, b"foo", 42, None, b"HTTP/1.1"))
+    assert (t(b"GET http://foo:42/bar HTTP/1.1") ==
+            ("absolute", b"GET", b"http", b"foo", 42, b"/bar", b"HTTP/1.1"))
+
+    with raises(HttpSyntaxException):
+        t(b"GET / WTF/1.1")
+    with raises(HttpSyntaxException):
+        t(b"this is not http")
+
+
+def test_parse_authority_form():
+    assert _parse_authority_form(b"foo:42") == (b"foo", 42)
+    with raises(HttpSyntaxException):
+        _parse_authority_form(b"foo")
+    with raises(HttpSyntaxException):
+        _parse_authority_form(b"foo:bar")
+    with raises(HttpSyntaxException):
+        _parse_authority_form(b"foo:99999999")
+    with raises(HttpSyntaxException):
+        _parse_authority_form(b"f\x00oo:80")
+
+
+def test_read_response_line():
+    def t(b):
+        return _read_response_line(BytesIO(b))
+
+    assert t(b"HTTP/1.1 200 OK") == (b"HTTP/1.1", 200, b"OK")
+    assert t(b"HTTP/1.1 200") == (b"HTTP/1.1", 200, b"")
+    with raises(HttpSyntaxException):
+        assert t(b"HTTP/1.1")
+
+    with raises(HttpSyntaxException):
+        t(b"HTTP/1.1 OK OK")
+    with raises(HttpSyntaxException):
+        t(b"WTF/1.1 200 OK")
+
+
+def test_check_http_version():
+    _check_http_version(b"HTTP/0.9")
+    _check_http_version(b"HTTP/1.0")
+    _check_http_version(b"HTTP/1.1")
+    _check_http_version(b"HTTP/2.0")
+    with raises(HttpSyntaxException):
+        _check_http_version(b"WTF/1.0")
+    with raises(HttpSyntaxException):
+        _check_http_version(b"HTTP/1.10")
+    with raises(HttpSyntaxException):
+        _check_http_version(b"HTTP/1.b")
+
+
+class TestReadHeaders(object):
+    @staticmethod
+    def _read(data):
+        return _read_headers(BytesIO(data))
+
+    def test_read_simple(self):
+        data = (
+            b"Header: one\r\n"
+            b"Header2: two\r\n"
+            b"\r\n"
+        )
+        headers = self._read(data)
+        assert headers.fields == [[b"Header", b"one"], [b"Header2", b"two"]]
+
+    def test_read_multi(self):
+        data = (
+            b"Header: one\r\n"
+            b"Header: two\r\n"
+            b"\r\n"
+        )
+        headers = self._read(data)
+        assert headers.fields == [[b"Header", b"one"], [b"Header", b"two"]]
+
+    def test_read_continued(self):
+        data = (
+            b"Header: one\r\n"
+            b"\ttwo\r\n"
+            b"Header2: three\r\n"
+            b"\r\n"
+        )
+        headers = self._read(data)
+        assert headers.fields == [[b"Header", b"one\r\n two"], [b"Header2", b"three"]]
+
+    def test_read_continued_err(self):
+        data = b"\tfoo: bar\r\n"
+        with raises(HttpSyntaxException):
+            self._read(data)
+
+    def test_read_err(self):
+        data = b"foo"
+        with raises(HttpSyntaxException):
+            self._read(data)
+
+
+def test_read_chunked():
+    req = treq(body=None)
+    req.headers["Transfer-Encoding"] = "chunked"
+
+    data = b"1\r\na\r\n0\r\n"
+    with raises(HttpSyntaxException):
+        b"".join(_read_chunked(BytesIO(data)))
+
+    data = b"1\r\na\r\n0\r\n\r\n"
+    assert b"".join(_read_chunked(BytesIO(data))) == b"a"
+
+    data = b"\r\n\r\n1\r\na\r\n1\r\nb\r\n0\r\n\r\n"
+    assert b"".join(_read_chunked(BytesIO(data))) == b"ab"
+
+    data = b"\r\n"
+    with raises("closed prematurely"):
+        b"".join(_read_chunked(BytesIO(data)))
+
+    data = b"1\r\nfoo"
+    with raises("malformed chunked body"):
+        b"".join(_read_chunked(BytesIO(data)))
+
+    data = b"foo\r\nfoo"
+    with raises(HttpSyntaxException):
+        b"".join(_read_chunked(BytesIO(data)))
+
+    data = b"5\r\naaaaa\r\n0\r\n\r\n"
+    with raises("too large"):
+        b"".join(_read_chunked(BytesIO(data), limit=2))