From d3611777539471e53d4fdedf352ed755a4092415 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sun, 3 Jul 2016 18:03:34 +0530 Subject: h2: move header parsing to netlib --- netlib/http/http2/__init__.py | 2 ++ netlib/http/http2/utils.py | 37 +++++++++++++++++++++++++++++++++++++ 2 files changed, 39 insertions(+) create mode 100644 netlib/http/http2/utils.py (limited to 'netlib/http') diff --git a/netlib/http/http2/__init__.py b/netlib/http/http2/__init__.py index 6a979a0d..60064190 100644 --- a/netlib/http/http2/__init__.py +++ b/netlib/http/http2/__init__.py @@ -1,6 +1,8 @@ from __future__ import absolute_import, print_function, division from netlib.http.http2 import framereader +from netlib.http.http2.utils import parse_headers __all__ = [ "framereader", + "parse_headers", ] diff --git a/netlib/http/http2/utils.py b/netlib/http/http2/utils.py new file mode 100644 index 00000000..4d5f580c --- /dev/null +++ b/netlib/http/http2/utils.py @@ -0,0 +1,37 @@ +from netlib.http import url + + +def parse_headers(headers): + authority = headers.get(':authority', b'') + method = headers.get(':method', b'GET') + scheme = headers.get(':scheme', b'https') + path = headers.get(':path', b'/') + + headers.clear(":method") + headers.clear(":scheme") + headers.clear(":path") + + host = None + port = None + + if path == b'*' or path.startswith(b"/"): + first_line_format = "relative" + elif method == b'CONNECT': # pragma: no cover + raise NotImplementedError("CONNECT over HTTP/2 is not implemented.") + else: # pragma: no cover + first_line_format = "absolute" + # FIXME: verify if path or :host contains what we need + scheme, host, port, _ = url.parse(path) + + if authority: + host, _, port = authority.partition(b':') + + if not host: + host = b'localhost' + + if not port: + port = 443 if scheme == b'https' else 80 + + port = int(port) + + return first_line_format, method, scheme, host, port, path -- cgit v1.2.3 From 23e295b37eb3aee9ebe7c88cc1ebb1dc2ffa9cf2 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 3 Jul 2016 19:04:16 +0200 Subject: py3: fix bytes vs. str --- netlib/http/http2/utils.py | 8 ++++---- 1 file changed, 4 insertions(+), 4 deletions(-) (limited to 'netlib/http') diff --git a/netlib/http/http2/utils.py b/netlib/http/http2/utils.py index 4d5f580c..4c01952d 100644 --- a/netlib/http/http2/utils.py +++ b/netlib/http/http2/utils.py @@ -2,10 +2,10 @@ from netlib.http import url def parse_headers(headers): - authority = headers.get(':authority', b'') - method = headers.get(':method', b'GET') - scheme = headers.get(':scheme', b'https') - path = headers.get(':path', b'/') + authority = headers.get(':authority', '').encode() + method = headers.get(':method', 'GET').encode() + scheme = headers.get(':scheme', 'https').encode() + path = headers.get(':path', '/').encode() headers.clear(":method") headers.clear(":scheme") -- cgit v1.2.3 From 9c873d63f4ede1b2470f8e7ea838909e60efe998 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 6 Jul 2016 19:50:06 -0700 Subject: py3++, multidict fixes This commit improves Python 3 compatibility and fixes two multidict issues: 1. Headers.items(multi=True) now decodes fields 2. MultiDict.clear(item) has been removed, as Python's MutableMapping already defines .clear() with different semantics. This is confusing for everyone who expects a dict-like object. `.pop("attr", None)` is not fantastic, but it's the Python way to do it. --- netlib/http/headers.py | 9 +++++++++ netlib/http/http2/utils.py | 6 +++--- 2 files changed, 12 insertions(+), 3 deletions(-) (limited to 'netlib/http') diff --git a/netlib/http/headers.py b/netlib/http/headers.py index f052a53b..413add87 100644 --- a/netlib/http/headers.py +++ b/netlib/http/headers.py @@ -148,6 +148,15 @@ class Headers(multidict.MultiDict): value = _always_bytes(value) super(Headers, self).insert(index, key, value) + def items(self, multi=False): + if multi: + return ( + (_native(k), _native(v)) + for k, v in self.fields + ) + else: + return super(Headers, self).items() + def replace(self, pattern, repl, flags=0): """ Replaces a regular expression pattern with repl in each "name: value" diff --git a/netlib/http/http2/utils.py b/netlib/http/http2/utils.py index 4c01952d..164bacc8 100644 --- a/netlib/http/http2/utils.py +++ b/netlib/http/http2/utils.py @@ -7,9 +7,9 @@ def parse_headers(headers): scheme = headers.get(':scheme', 'https').encode() path = headers.get(':path', '/').encode() - headers.clear(":method") - headers.clear(":scheme") - headers.clear(":path") + headers.pop(":method", None) + headers.pop(":scheme", None) + headers.pop(":path", None) host = None port = None -- cgit v1.2.3 From f62e976e1e0245665aeeb08fa801661d6c766ba8 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 7 Jul 2016 17:29:22 -0700 Subject: py3++ --- netlib/http/message.py | 3 +++ netlib/http/request.py | 14 +++++++++++++- netlib/http/response.py | 7 +++++++ 3 files changed, 23 insertions(+), 1 deletion(-) (limited to 'netlib/http') diff --git a/netlib/http/message.py b/netlib/http/message.py index 0583c246..b268fec9 100644 --- a/netlib/http/message.py +++ b/netlib/http/message.py @@ -100,7 +100,10 @@ class Message(basetypes.Serializable): @content.setter def content(self, content): + # type: (Optional[bytes]) -> None self.data.content = content + if isinstance(content, six.text_type): + raise ValueError("Message content must be bytes, not {}".format(type(content).__name__)) if isinstance(content, bytes): self.headers["content-length"] = str(len(content)) diff --git a/netlib/http/request.py b/netlib/http/request.py index d9f4ed00..c4c39942 100644 --- a/netlib/http/request.py +++ b/netlib/http/request.py @@ -23,8 +23,20 @@ host_header_re = re.compile(r"^(?P[^:]+|\[.+\])(?::(?P\d+))?$") class RequestData(message.MessageData): def __init__(self, first_line_format, method, scheme, host, port, path, http_version, headers=(), content=None, timestamp_start=None, timestamp_end=None): + if isinstance(method, six.text_type): + method = method.encode("ascii", "strict") + if isinstance(scheme, six.text_type): + scheme = scheme.encode("ascii", "strict") + if isinstance(host, six.text_type): + host = host.encode("idna", "strict") + if isinstance(path, six.text_type): + path = path.encode("ascii", "strict") + if isinstance(http_version, six.text_type): + http_version = http_version.encode("ascii", "strict") if not isinstance(headers, nheaders.Headers): headers = nheaders.Headers(headers) + if isinstance(content, six.text_type): + raise ValueError("Content must be bytes, not {}".format(type(content).__name__)) self.first_line_format = first_line_format self.method = method @@ -356,7 +368,7 @@ class Request(message.Message): This will overwrite the existing content if there is one. """ self.headers["content-type"] = "application/x-www-form-urlencoded" - self.content = netlib.http.url.encode(value) + self.content = netlib.http.url.encode(value).encode() @urlencoded_form.setter def urlencoded_form(self, value): diff --git a/netlib/http/response.py b/netlib/http/response.py index 17d69418..7cfb55c8 100644 --- a/netlib/http/response.py +++ b/netlib/http/response.py @@ -2,6 +2,7 @@ from __future__ import absolute_import, print_function, division from email.utils import parsedate_tz, formatdate, mktime_tz import time +import six from netlib.http import cookies from netlib.http import headers as nheaders @@ -13,8 +14,14 @@ from netlib import human class ResponseData(message.MessageData): def __init__(self, http_version, status_code, reason=None, headers=(), content=None, timestamp_start=None, timestamp_end=None): + if isinstance(http_version, six.text_type): + http_version = http_version.encode("ascii", "strict") + if isinstance(reason, six.text_type): + reason = reason.encode("ascii", "strict") if not isinstance(headers, nheaders.Headers): headers = nheaders.Headers(headers) + if isinstance(content, six.text_type): + raise ValueError("Content must be bytes, not {}".format(type(content).__name__)) self.http_version = http_version self.status_code = status_code -- cgit v1.2.3 From c92992f03bba6553ec39fc42e6716beb942967e3 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Fri, 8 Jul 2016 14:16:29 +0530 Subject: Move cookie expiry detection to separate function --- netlib/http/cookies.py | 24 +++++++++++++++++++++++- 1 file changed, 23 insertions(+), 1 deletion(-) (limited to 'netlib/http') diff --git a/netlib/http/cookies.py b/netlib/http/cookies.py index 768a85df..90789365 100644 --- a/netlib/http/cookies.py +++ b/netlib/http/cookies.py @@ -1,7 +1,8 @@ import collections +import email.utils import re +import time -import email.utils from netlib import multidict """ @@ -260,3 +261,24 @@ def refresh_set_cookie_header(c, delta): if not ret: raise ValueError("Invalid Cookie") return ret + +def is_expired(cookie_attrs): + """ + Determines whether a cookie has expired. + + Returns: boolean + """ + expired = False + + # See if 'expires' time is in the past + if 'expires' in cookie_attrs: + e = email.utils.parsedate_tz(cookie_attrs["expires"]) + if e: + exp_ts = email.utils.mktime_tz(e) + now_ts = time.time() + expired = exp_ts < now_ts + + # or if Max-Age is 0 + expired = expired or (int(cookie_attrs.get('Max-Age', 1)) == 0) + + return expired -- cgit v1.2.3 From 39f51084003b93a2e9868f7a56acfc29c12ed79e Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sun, 10 Jul 2016 01:06:50 +0530 Subject: Test cookies.is_expired separately --- netlib/http/cookies.py | 13 +++++++++---- 1 file changed, 9 insertions(+), 4 deletions(-) (limited to 'netlib/http') diff --git a/netlib/http/cookies.py b/netlib/http/cookies.py index 90789365..dd0af99c 100644 --- a/netlib/http/cookies.py +++ b/netlib/http/cookies.py @@ -262,23 +262,28 @@ def refresh_set_cookie_header(c, delta): raise ValueError("Invalid Cookie") return ret + def is_expired(cookie_attrs): """ Determines whether a cookie has expired. Returns: boolean """ - expired = False # See if 'expires' time is in the past + expires = False if 'expires' in cookie_attrs: e = email.utils.parsedate_tz(cookie_attrs["expires"]) if e: exp_ts = email.utils.mktime_tz(e) now_ts = time.time() - expired = exp_ts < now_ts + expires = exp_ts < now_ts # or if Max-Age is 0 - expired = expired or (int(cookie_attrs.get('Max-Age', 1)) == 0) + max_age = False + try: + max_age = int(cookie_attrs.get('Max-Age', 1)) == 0 + except ValueError: + pass - return expired + return expires or max_age -- cgit v1.2.3