diff options
Diffstat (limited to 'netlib')
| -rw-r--r-- | netlib/http/__init__.py | 8 | ||||
| -rw-r--r-- | netlib/http/cookies.py | 43 | ||||
| -rw-r--r-- | netlib/http/headers.py | 4 | ||||
| -rw-r--r-- | netlib/http/message.py | 41 | ||||
| -rw-r--r-- | netlib/http/request.py | 69 | ||||
| -rw-r--r-- | netlib/http/response.py | 45 | ||||
| -rw-r--r-- | netlib/multidict.py | 403 |
7 files changed, 363 insertions, 250 deletions
diff --git a/netlib/http/__init__.py b/netlib/http/__init__.py index 917080f7..9fafa28f 100644 --- a/netlib/http/__init__.py +++ b/netlib/http/__init__.py @@ -2,13 +2,13 @@ from __future__ import absolute_import, print_function, division from .request import Request from .response import Response from .headers import Headers -from .message import decoded -from . import http1, http2 +from .message import MultiDictView, decoded +from . import http1, http2, status_codes __all__ = [ "Request", "Response", "Headers", - "decoded", - "http1", "http2", + "MultiDictView", "decoded", + "http1", "http2", "status_codes", ] diff --git a/netlib/http/cookies.py b/netlib/http/cookies.py index fd531146..c5ac4591 100644 --- a/netlib/http/cookies.py +++ b/netlib/http/cookies.py @@ -1,6 +1,8 @@ +import collections import re from email.utils import parsedate_tz, formatdate, mktime_tz +from netlib.multidict import ImmutableMultiDict from .. import odict """ @@ -155,25 +157,52 @@ def _parse_set_cookie_pairs(s): return pairs +def parse_set_cookie_headers(headers): + ret = [] + for header in headers: + v = parse_set_cookie_header(header) + if v: + name, value, attrs = v + ret.append((name, SetCookie(value, attrs))) + return ret + + +class CookieAttrs(ImmutableMultiDict): + @staticmethod + def _kconv(v): + return v.lower() + + @staticmethod + def _reduce_values(values): + # See the StickyCookieTest for a weird cookie that only makes sense + # if we take the last part. + return values[-1] + + +SetCookie = collections.namedtuple("SetCookie", ["value", "attrs"]) + + def parse_set_cookie_header(line): """ Parse a Set-Cookie header value Returns a (name, value, attrs) tuple, or None, where attrs is an - ODictCaseless set of attributes. No attempt is made to parse attribute + CookieAttrs dict of attributes. No attempt is made to parse attribute values - they are treated purely as strings. """ pairs = _parse_set_cookie_pairs(line) if pairs: - return pairs[0][0], pairs[0][1], odict.ODictCaseless(pairs[1:]) + return pairs[0][0], pairs[0][1], CookieAttrs(tuple(x) for x in pairs[1:]) def format_set_cookie_header(name, value, attrs): """ Formats a Set-Cookie header value. """ - pairs = [[name, value]] - pairs.extend(attrs.lst) + pairs = [(name, value)] + pairs.extend( + attrs.fields if hasattr(attrs, "fields") else attrs + ) return _format_set_cookie_pairs(pairs) @@ -214,10 +243,10 @@ def refresh_set_cookie_header(c, delta): raise ValueError("Invalid Cookie") if "expires" in attrs: - e = parsedate_tz(attrs["expires"][-1]) + e = parsedate_tz(attrs["expires"]) if e: f = mktime_tz(e) + delta - attrs["expires"] = [formatdate(f)] + attrs = attrs.with_set_all("expires", [formatdate(f)]) else: # This can happen when the expires tag is invalid. # reddit.com sends a an expires tag like this: "Thu, 31 Dec @@ -225,7 +254,7 @@ def refresh_set_cookie_header(c, delta): # strictly correct according to the cookie spec. Browsers # appear to parse this tolerantly - maybe we should too. # For now, we just ignore this. - del attrs["expires"] + attrs = attrs.with_delitem("expires") ret = format_set_cookie_header(name, value, attrs) if not ret: diff --git a/netlib/http/headers.py b/netlib/http/headers.py index 7e39c371..8959394c 100644 --- a/netlib/http/headers.py +++ b/netlib/http/headers.py @@ -83,6 +83,10 @@ class Headers(MultiDict): """ super(Headers, self).__init__(fields) + for key, value in self.fields: + if not isinstance(key, bytes) or not isinstance(value, bytes): + raise TypeError("Header fields must be bytes.") + # content_type -> content-type headers = { _always_bytes(name).replace(b"_", b"-"): _always_bytes(value) diff --git a/netlib/http/message.py b/netlib/http/message.py index 262ef3e1..3c731ea6 100644 --- a/netlib/http/message.py +++ b/netlib/http/message.py @@ -238,9 +238,44 @@ class decoded(object): self.message.encode(self.ce) -class MessageMultiDict(MultiDict): +class MultiDictView(MultiDict): """ - A MultiDict that provides a proxy view to the underlying message. + Some parts in HTTP (Cookies, URL query strings, ...) require a specific data structure: A MultiDict. + It behaves mostly like an ordered dict but it can have several values for the same key. + + The MultiDictView provides a MultiDict *view* on an :py:class:`Request` or :py:class:`Response`. + That is, it represents a part of the request as a MultiDict, but doesn't contain state/data themselves. + + For example, ``request.cookies`` provides a view on the ``Cookie: ...`` header. + Any change to ``request.cookies`` will also modify the ``Cookie`` header. + Any change to the ``Cookie`` header will also modify ``request.cookies``. + + Example: + + .. code-block:: python + + # Cookies are represented as a MultiDict. + >>> request.cookies + MultiDictView[("name", "value"), ("a", "false"), ("a", "42")] + + # MultiDicts mostly behave like a normal dict. + >>> request.cookies["name"] + "value" + + # If there is more than one value, only the first value is returned. + >>> request.cookies["a"] + "false" + + # `.get_all(key)` returns a list of all values. + >>> request.cookies.get_all("a") + ["false", "42"] + + # Changes to the headers are immediately reflected in the cookies. + >>> request.cookies + MultiDictView[("name", "value"), ...] + >>> del request.headers["Cookie"] + >>> request.cookies + MultiDictView[] # empty now """ def __init__(self, attr, message): @@ -248,7 +283,7 @@ class MessageMultiDict(MultiDict): # We do not want to call the parent constructor here as that # would cause an unnecessary parse/unparse pass. # This is here to silence linters. Message - super(MessageMultiDict, self).__init__(None) + super(MultiDictView, self).__init__(None) self._attr = attr self._message = message # type: Message diff --git a/netlib/http/request.py b/netlib/http/request.py index 26ec12cf..ae28084b 100644 --- a/netlib/http/request.py +++ b/netlib/http/request.py @@ -11,7 +11,7 @@ from netlib.http import cookies from netlib.odict import ODict from .. import encoding from .headers import Headers -from .message import Message, _native, _always_bytes, MessageData, MessageMultiDict +from .message import Message, _native, _always_bytes, MessageData, MultiDictView # This regex extracts & splits the host header into host and port. # Handles the edge case of IPv6 addresses containing colons. @@ -224,11 +224,11 @@ class Request(Message): @property def query(self): - # type: () -> MessageMultiDict + # type: () -> MultiDictView """ - The request query string as an :py:class:`MessageMultiDict` object. + The request query string as an :py:class:`MultiDictView` object. """ - return MessageMultiDict("query", self) + return MultiDictView("query", self) @property def _query(self): @@ -244,13 +244,13 @@ class Request(Message): @property def cookies(self): - # type: () -> MessageMultiDict + # type: () -> MultiDictView """ The request cookies. - An empty :py:class:`MessageMultiDict` object if the cookie monster ate them all. + An empty :py:class:`MultiDictView` object if the cookie monster ate them all. """ - return MessageMultiDict("cookies", self) + return MultiDictView("cookies", self) @property def _cookies(self): @@ -318,17 +318,18 @@ class Request(Message): @property def urlencoded_form(self): """ - The URL-encoded form data as an :py:class:`MessageMultiDict` object. - None if the content-type indicates non-form data. + The URL-encoded form data as an :py:class:`MultiDictView` object. + An empty MultiDictView if the content-type indicates non-form data + or the content could not be parsed. """ - is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower() - if is_valid_content_type: - return MessageMultiDict("urlencoded_form", self) - return None + return MultiDictView("urlencoded_form", self) @property def _urlencoded_form(self): - return tuple(utils.urldecode(self.content)) + is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower() + if is_valid_content_type: + return tuple(utils.urldecode(self.content)) + return () @urlencoded_form.setter def urlencoded_form(self, value): @@ -345,45 +346,15 @@ class Request(Message): The multipart form data as an :py:class:`MultipartFormDict` object. None if the content-type indicates non-form data. """ - is_valid_content_type = "multipart/form-data" in self.headers.get("content-type", "").lower() - if is_valid_content_type: - return MessageMultiDict("multipart_form", self) - return None + return MultiDictView("multipart_form", self) @property def _multipart_form(self): - return utils.multipartdecode(self.headers, self.content) + is_valid_content_type = "multipart/form-data" in self.headers.get("content-type", "").lower() + if is_valid_content_type: + return utils.multipartdecode(self.headers, self.content) + return () @multipart_form.setter def multipart_form(self, value): raise NotImplementedError() - - # Legacy - - def get_query(self): # pragma: no cover - warnings.warn(".get_query is deprecated, use .query instead.", DeprecationWarning) - return self.query or ODict([]) - - def set_query(self, odict): # pragma: no cover - warnings.warn(".set_query is deprecated, use .query instead.", DeprecationWarning) - self.query = odict - - def get_path_components(self): # pragma: no cover - warnings.warn(".get_path_components is deprecated, use .path_components instead.", DeprecationWarning) - return self.path_components - - def set_path_components(self, lst): # pragma: no cover - warnings.warn(".set_path_components is deprecated, use .path_components instead.", DeprecationWarning) - self.path_components = lst - - def get_form_urlencoded(self): # pragma: no cover - warnings.warn(".get_form_urlencoded is deprecated, use .urlencoded_form instead.", DeprecationWarning) - return self.urlencoded_form or ODict([]) - - def set_form_urlencoded(self, odict): # pragma: no cover - warnings.warn(".set_form_urlencoded is deprecated, use .urlencoded_form instead.", DeprecationWarning) - self.urlencoded_form = odict - - def get_form_multipart(self): # pragma: no cover - warnings.warn(".get_form_multipart is deprecated, use .multipart_form instead.", DeprecationWarning) - return self.multipart_form or ODict([]) diff --git a/netlib/http/response.py b/netlib/http/response.py index 20074dca..6d56fc1f 100644 --- a/netlib/http/response.py +++ b/netlib/http/response.py @@ -1,14 +1,12 @@ from __future__ import absolute_import, print_function, division -import warnings from email.utils import parsedate_tz, formatdate, mktime_tz import time from . import cookies from .headers import Headers -from .message import Message, _native, _always_bytes, MessageData +from .message import Message, _native, _always_bytes, MessageData, MultiDictView from .. import utils -from ..odict import ODict class ResponseData(MessageData): @@ -70,33 +68,32 @@ class Response(Message): def reason(self, reason): self.data.reason = _always_bytes(reason) - # FIXME @property def cookies(self): + # type: () -> MultiDictView """ - Get the contents of all Set-Cookie headers. + The response cookies. A possibly empty :py:class:`MultiDictView`, where the keys are + cookie name strings, and values are (value, attr) tuples. Value is a string, and attr is + an ODictCaseless containing cookie attributes. Within attrs, unary attributes (e.g. HTTPOnly) + are indicated by a Null value. - A possibly empty :py:class:`ODict`, where keys are cookie name strings, - and values are [value, attr] lists. Value is a string, and attr is - an ODictCaseless containing cookie attributes. Within attrs, unary - attributes (e.g. HTTPOnly) are indicated by a Null value. + Caveats: + Updating the attr """ - ret = [] - for header in self.headers.get_all("set-cookie"): - v = cookies.parse_set_cookie_header(header) - if v: - name, value, attrs = v - ret.append([name, [value, attrs]]) - return ODict(ret) - - # FIXME + return MultiDictView("cookies", self) + + @property + def _cookies(self): + h = self.headers.get_all("set-cookie") + return tuple(cookies.parse_set_cookie_headers(h)) + @cookies.setter - def cookies(self, odict): - values = [] - for i in odict.lst: - header = cookies.format_set_cookie_header(i[0], i[1][0], i[1][1]) - values.append(header) - self.headers.set_all("set-cookie", values) + def cookies(self, all_cookies): + cookie_headers = [] + for k, v in all_cookies: + header = cookies.format_set_cookie_header(k, v[0], v[1]) + cookie_headers.append(header) + self.headers.set_all("set-cookie", cookie_headers) def refresh(self, now=None): """ diff --git a/netlib/multidict.py b/netlib/multidict.py index a7158bc5..32d5bfc2 100644 --- a/netlib/multidict.py +++ b/netlib/multidict.py @@ -1,163 +1,240 @@ -from __future__ import absolute_import, print_function, division
-
-from abc import ABCMeta, abstractmethod
-
-from typing import Tuple
-
-try:
- from collections.abc import MutableMapping
-except ImportError: # pragma: no cover
- from collections import MutableMapping # Workaround for Python < 3.3
-
-import six
-
-from .utils import Serializable
-
-
-@six.add_metaclass(ABCMeta)
-class MultiDict(MutableMapping, Serializable):
- def __init__(self, fields=None):
-
- # it is important for us that .fields is immutable, so that we can easily
- # detect changes to it.
- self.fields = tuple(fields) if fields else tuple() # type: Tuple[Tuple[bytes, bytes], ...]
-
- for key, value in self.fields:
- if not isinstance(key, bytes) or not isinstance(value, bytes):
- raise TypeError("MultiDict fields must be bytes.")
-
- def __repr__(self):
- fields = tuple(
- repr(field)
- for field in self.fields
- )
- return "{cls}[{fields}]".format(
- cls=type(self).__name__,
- fields=", ".join(fields)
- )
-
- @staticmethod
- @abstractmethod
- def _reduce_values(values):
- pass
-
- @staticmethod
- @abstractmethod
- def _kconv(v):
- pass
-
- def __getitem__(self, key):
- values = self.get_all(key)
- if not values:
- raise KeyError(key)
- return self._reduce_values(values)
-
- def __setitem__(self, key, value):
- self.set_all(key, [value])
-
- def __delitem__(self, key):
- if key not in self:
- raise KeyError(key)
- key = self._kconv(key)
- self.fields = tuple(
- field for field in self.fields
- if key != self._kconv(field[0])
- )
-
- def __iter__(self):
- seen = set()
- for key, _ in self.fields:
- key_kconv = self._kconv(key)
- if key_kconv not in seen:
- seen.add(key_kconv)
- yield key
-
- def __len__(self):
- return len(set(self._kconv(key) for key, _ in self.fields))
-
- def __eq__(self, other):
- if isinstance(other, MultiDict):
- return self.fields == other.fields
- return False
-
- def __ne__(self, other):
- return not self.__eq__(other)
-
- def get_all(self, key):
- """
- Return the list of items for a given key.
- If that key is not in the MultiDict,
- the return value will be an empty list.
- """
- key = self._kconv(key)
- return [
- value
- for k, value in self.fields
- if self._kconv(k) == key
- ]
-
- def set_all(self, key, values):
- """
- Remove the old values for a key and add new ones.
- """
- key_kconv = self._kconv(key)
-
- new_fields = []
- for field in self.fields:
- if self._kconv(field[0]) == key_kconv:
- if values:
- new_fields.append(
- (key, values.pop(0))
- )
- else:
- new_fields.append(field)
- while values:
- new_fields.append(
- (key, values.pop(0))
- )
- self.fields = tuple(new_fields)
-
- def add(self, key, value):
- self.insert(len(self.fields), key, value)
-
- def insert(self, index, key, value):
- item = (key, value)
- self.fields = self.fields[:index] + (item,) + self.fields[index:]
-
- def keys(self, multi=False):
- return (
- k
- for k, _ in self.items(multi)
- )
-
- def values(self, multi=False):
- return (
- v
- for _, v in self.items(multi)
- )
-
- def items(self, multi=False):
- if multi:
- return self.fields
- else:
- return super(MultiDict, self).items()
-
- def to_dict(self):
- d = {}
- for key in self:
- values = self.get_all(key)
- if len(values) == 1:
- d[key] = values[0]
- else:
- d[key] = values
- return d
-
- def get_state(self):
- return self.fields
-
- def set_state(self, state):
- self.fields = tuple(tuple(x) for x in state)
-
- @classmethod
- def from_state(cls, state):
- return cls(tuple(x) for x in state)
+from __future__ import absolute_import, print_function, division + +from abc import ABCMeta, abstractmethod + +from typing import Tuple, TypeVar + +try: + from collections.abc import MutableMapping +except ImportError: # pragma: no cover + from collections import MutableMapping # Workaround for Python < 3.3 + +import six + +from .utils import Serializable + + +@six.add_metaclass(ABCMeta) +class MultiDict(MutableMapping, Serializable): + def __init__(self, fields=None): + + # it is important for us that .fields is immutable, so that we can easily + # detect changes to it. + self.fields = tuple(fields) if fields else tuple() # type: Tuple[Tuple[bytes, bytes], ...] + + def __repr__(self): + fields = tuple( + repr(field) + for field in self.fields + ) + return "{cls}[{fields}]".format( + cls=type(self).__name__, + fields=", ".join(fields) + ) + + @staticmethod + @abstractmethod + def _reduce_values(values): + pass + + @staticmethod + @abstractmethod + def _kconv(v): + pass + + def __getitem__(self, key): + values = self.get_all(key) + if not values: + raise KeyError(key) + return self._reduce_values(values) + + def __setitem__(self, key, value): + self.set_all(key, [value]) + + def __delitem__(self, key): + if key not in self: + raise KeyError(key) + key = self._kconv(key) + self.fields = tuple( + field for field in self.fields + if key != self._kconv(field[0]) + ) + + def __iter__(self): + seen = set() + for key, _ in self.fields: + key_kconv = self._kconv(key) + if key_kconv not in seen: + seen.add(key_kconv) + yield key + + def __len__(self): + return len(set(self._kconv(key) for key, _ in self.fields)) + + def __eq__(self, other): + if isinstance(other, MultiDict): + return self.fields == other.fields + return False + + def __ne__(self, other): + return not self.__eq__(other) + + def get_all(self, key): + """ + Return the list of all values for a given key. + If that key is not in the MultiDict, the return value will be an empty list. + """ + key = self._kconv(key) + return [ + value + for k, value in self.fields + if self._kconv(k) == key + ] + + def set_all(self, key, values): + """ + Remove the old values for a key and add new ones. + """ + key_kconv = self._kconv(key) + + new_fields = [] + for field in self.fields: + if self._kconv(field[0]) == key_kconv: + if values: + new_fields.append( + (key, values.pop(0)) + ) + else: + new_fields.append(field) + while values: + new_fields.append( + (key, values.pop(0)) + ) + self.fields = tuple(new_fields) + + def add(self, key, value): + """ + Add an additional value for the given key at the bottom. + """ + self.insert(len(self.fields), key, value) + + def insert(self, index, key, value): + """ + Insert an additional value for the given key at the specified position. + """ + item = (key, value) + self.fields = self.fields[:index] + (item,) + self.fields[index:] + + def keys(self, multi=False): + """ + Get all keys. + + Args: + multi(bool): + If True, one key per value will be returned. + If False, duplicate keys will only be returned once. + """ + return ( + k + for k, _ in self.items(multi) + ) + + def values(self, multi=False): + """ + Get all values. + + Args: + multi(bool): + If True, all values will be returned. + If False, only the first value per key will be returned. + """ + return ( + v + for _, v in self.items(multi) + ) + + def items(self, multi=False): + """ + Get all (key, value) tuples. + + Args: + multi(bool): + If True, all (key, value) pairs will be returned + If False, only the first (key, value) pair per unique key will be returned. + """ + if multi: + return self.fields + else: + return super(MultiDict, self).items() + + def to_dict(self): + """ + Get the MultiDict as a plain Python dict. + Keys with multiple values are returned as lists. + + Example: + + .. code-block:: python + + # Simple dict with duplicate values. + >>> d + MultiDictView[("name", "value"), ("a", "false"), ("a", "42")] + >>> d.to_dict() + { + "name": "value", + "a": ["false", "42"] + } + """ + d = {} + for key in self: + values = self.get_all(key) + if len(values) == 1: + d[key] = values[0] + else: + d[key] = values + return d + + def get_state(self): + return self.fields + + def set_state(self, state): + self.fields = tuple(tuple(x) for x in state) + + @classmethod + def from_state(cls, state): + return cls(tuple(x) for x in state) + + +@six.add_metaclass(ABCMeta) +class ImmutableMultiDict(MultiDict): + def _immutable(self, *_): + raise TypeError('{} objects are immutable'.format(self.__class__.__name__)) + + __delitem__ = set_all = insert = _immutable + + def with_delitem(self, key): + """ + Returns: + An updated ImmutableMultiDict. The original object will not be modified. + """ + ret = self.copy() + super(ImmutableMultiDict, ret).__delitem__(key) + return ret + + def with_set_all(self, key, values): + """ + Returns: + An updated ImmutableMultiDict. The original object will not be modified. + """ + ret = self.copy() + super(ImmutableMultiDict, ret).set_all(key, values) + return ret + + def with_insert(self, index, key, value): + """ + Returns: + An updated ImmutableMultiDict. The original object will not be modified. + """ + ret = self.copy() + super(ImmutableMultiDict, ret).insert(index, key, value) + return ret |