aboutsummaryrefslogtreecommitdiffstats
path: root/netlib
diff options
context:
space:
mode:
Diffstat (limited to 'netlib')
-rw-r--r--netlib/basetypes.py34
-rw-r--r--netlib/certutils.py4
-rw-r--r--netlib/http/headers.py27
-rw-r--r--netlib/http/http1/read.py17
-rw-r--r--netlib/http/http2/connections.py77
-rw-r--r--netlib/http/http2/framereader.py21
-rw-r--r--netlib/http/message.py17
-rw-r--r--netlib/http/multipart.py32
-rw-r--r--netlib/http/request.py24
-rw-r--r--netlib/http/response.py4
-rw-r--r--netlib/http/url.py96
-rw-r--r--netlib/human.py50
-rw-r--r--netlib/multidict.py13
-rw-r--r--netlib/odict.py8
-rw-r--r--netlib/tcp.py4
-rw-r--r--netlib/utils.py238
-rw-r--r--netlib/websockets/frame.py3
17 files changed, 358 insertions, 311 deletions
diff --git a/netlib/basetypes.py b/netlib/basetypes.py
new file mode 100644
index 00000000..9d6c60ba
--- /dev/null
+++ b/netlib/basetypes.py
@@ -0,0 +1,34 @@
+import six
+import abc
+
+
+@six.add_metaclass(abc.ABCMeta)
+class Serializable(object):
+ """
+ Abstract Base Class that defines an API to save an object's state and restore it later on.
+ """
+
+ @classmethod
+ @abc.abstractmethod
+ def from_state(cls, state):
+ """
+ Create a new object from the given state.
+ """
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def get_state(self):
+ """
+ Retrieve object state.
+ """
+ raise NotImplementedError()
+
+ @abc.abstractmethod
+ def set_state(self, state):
+ """
+ Set object state to the given state.
+ """
+ raise NotImplementedError()
+
+ def copy(self):
+ return self.from_state(self.get_state())
diff --git a/netlib/certutils.py b/netlib/certutils.py
index 34e01ed3..4a19d170 100644
--- a/netlib/certutils.py
+++ b/netlib/certutils.py
@@ -12,7 +12,7 @@ from pyasn1.codec.der.decoder import decode
from pyasn1.error import PyAsn1Error
import OpenSSL
-from .utils import Serializable
+from . import basetypes
# Default expiry must not be too long: https://github.com/mitmproxy/mitmproxy/issues/815
@@ -364,7 +364,7 @@ class _GeneralNames(univ.SequenceOf):
constraint.ValueSizeConstraint(1, 1024)
-class SSLCert(Serializable):
+class SSLCert(basetypes.Serializable):
def __init__(self, cert):
"""
diff --git a/netlib/http/headers.py b/netlib/http/headers.py
index 6165fd61..fa7b7180 100644
--- a/netlib/http/headers.py
+++ b/netlib/http/headers.py
@@ -175,3 +175,30 @@ class Headers(MultiDict):
fields.append([name, value])
self.fields = fields
return replacements
+
+
+def parse_content_type(c):
+ """
+ A simple parser for content-type values. Returns a (type, subtype,
+ parameters) tuple, where type and subtype are strings, and parameters
+ is a dict. If the string could not be parsed, return None.
+
+ E.g. the following string:
+
+ text/html; charset=UTF-8
+
+ Returns:
+
+ ("text", "html", {"charset": "UTF-8"})
+ """
+ parts = c.split(";", 1)
+ ts = parts[0].split("/", 1)
+ if len(ts) != 2:
+ return None
+ d = {}
+ if len(parts) == 2:
+ for i in parts[1].split(";"):
+ clause = i.split("=", 1)
+ if len(clause) == 2:
+ d[clause[0].strip()] = clause[1].strip()
+ return ts[0].lower(), ts[1].lower(), d
diff --git a/netlib/http/http1/read.py b/netlib/http/http1/read.py
index d30976bd..5783ec67 100644
--- a/netlib/http/http1/read.py
+++ b/netlib/http/http1/read.py
@@ -6,6 +6,19 @@ import re
from ... import utils
from ...exceptions import HttpReadDisconnect, HttpSyntaxException, HttpException, TcpDisconnect
from .. import Request, Response, Headers
+from .. import url
+
+
+def get_header_tokens(headers, key):
+ """
+ Retrieve all tokens for a header key. A number of different headers
+ follow a pattern where each header line can containe comma-separated
+ tokens, and headers can be set multiple times.
+ """
+ if key not in headers:
+ return []
+ tokens = headers[key].split(",")
+ return [token.strip() for token in tokens]
def read_request(rfile, body_size_limit=None):
@@ -147,7 +160,7 @@ def connection_close(http_version, headers):
"""
# At first, check if we have an explicit Connection header.
if "connection" in headers:
- tokens = utils.get_header_tokens(headers, "connection")
+ tokens = get_header_tokens(headers, "connection")
if "close" in tokens:
return True
elif "keep-alive" in tokens:
@@ -240,7 +253,7 @@ def _read_request_line(rfile):
scheme, path = None, None
else:
form = "absolute"
- scheme, host, port, path = utils.parse_url(path)
+ scheme, host, port, path = url.parse(path)
_check_http_version(http_version)
except ValueError:
diff --git a/netlib/http/http2/connections.py b/netlib/http/http2/connections.py
index b988d6ef..16bdf618 100644
--- a/netlib/http/http2/connections.py
+++ b/netlib/http/http2/connections.py
@@ -2,11 +2,12 @@ from __future__ import (absolute_import, print_function, division)
import itertools
import time
+import hyperframe.frame
+
from hpack.hpack import Encoder, Decoder
from ... import utils
-from .. import Headers, Response, Request
-
-from hyperframe import frame
+from .. import Headers, Response, Request, url
+from . import framereader
class TCPHandler(object):
@@ -38,12 +39,12 @@ class HTTP2Protocol(object):
CLIENT_CONNECTION_PREFACE = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n'
HTTP2_DEFAULT_SETTINGS = {
- frame.SettingsFrame.HEADER_TABLE_SIZE: 4096,
- frame.SettingsFrame.ENABLE_PUSH: 1,
- frame.SettingsFrame.MAX_CONCURRENT_STREAMS: None,
- frame.SettingsFrame.INITIAL_WINDOW_SIZE: 2 ** 16 - 1,
- frame.SettingsFrame.MAX_FRAME_SIZE: 2 ** 14,
- frame.SettingsFrame.MAX_HEADER_LIST_SIZE: None,
+ hyperframe.frame.SettingsFrame.HEADER_TABLE_SIZE: 4096,
+ hyperframe.frame.SettingsFrame.ENABLE_PUSH: 1,
+ hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: None,
+ hyperframe.frame.SettingsFrame.INITIAL_WINDOW_SIZE: 2 ** 16 - 1,
+ hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE: 2 ** 14,
+ hyperframe.frame.SettingsFrame.MAX_HEADER_LIST_SIZE: None,
}
def __init__(
@@ -98,6 +99,11 @@ class HTTP2Protocol(object):
method = headers.get(':method', 'GET')
scheme = headers.get(':scheme', 'https')
path = headers.get(':path', '/')
+
+ headers.clear(":method")
+ headers.clear(":scheme")
+ headers.clear(":path")
+
host = None
port = None
@@ -112,7 +118,7 @@ class HTTP2Protocol(object):
else:
first_line_format = "absolute"
# FIXME: verify if path or :host contains what we need
- scheme, host, port, _ = utils.parse_url(path)
+ scheme, host, port, _ = url.parse(path)
scheme = scheme.decode('ascii')
host = host.decode('ascii')
@@ -202,12 +208,9 @@ class HTTP2Protocol(object):
if ':authority' not in headers:
headers.insert(0, b':authority', authority.encode('ascii'))
- if ':scheme' not in headers:
- headers.insert(0, b':scheme', request.scheme.encode('ascii'))
- if ':path' not in headers:
- headers.insert(0, b':path', request.path.encode('ascii'))
- if ':method' not in headers:
- headers.insert(0, b':method', request.method.encode('ascii'))
+ headers.insert(0, b':scheme', request.scheme.encode('ascii'))
+ headers.insert(0, b':path', request.path.encode('ascii'))
+ headers.insert(0, b':method', request.method.encode('ascii'))
if hasattr(request, 'stream_id'):
stream_id = request.stream_id
@@ -251,9 +254,9 @@ class HTTP2Protocol(object):
magic = self.tcp_handler.rfile.safe_read(magic_length)
assert magic == self.CLIENT_CONNECTION_PREFACE
- frm = frame.SettingsFrame(settings={
- frame.SettingsFrame.ENABLE_PUSH: 0,
- frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 1,
+ frm = hyperframe.frame.SettingsFrame(settings={
+ hyperframe.frame.SettingsFrame.ENABLE_PUSH: 0,
+ hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 1,
})
self.send_frame(frm, hide=True)
self._receive_settings(hide=True)
@@ -264,7 +267,7 @@ class HTTP2Protocol(object):
self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE)
- self.send_frame(frame.SettingsFrame(), hide=True)
+ self.send_frame(hyperframe.frame.SettingsFrame(), hide=True)
self._receive_settings(hide=True) # server announces own settings
self._receive_settings(hide=True) # server acks my settings
@@ -277,18 +280,18 @@ class HTTP2Protocol(object):
def read_frame(self, hide=False):
while True:
- frm = utils.http2_read_frame(self.tcp_handler.rfile)
+ frm = framereader.http2_read_frame(self.tcp_handler.rfile)
if not hide and self.dump_frames: # pragma no cover
print(frm.human_readable("<<"))
- if isinstance(frm, frame.PingFrame):
- raw_bytes = frame.PingFrame(flags=['ACK'], payload=frm.payload).serialize()
+ if isinstance(frm, hyperframe.frame.PingFrame):
+ raw_bytes = hyperframe.frame.PingFrame(flags=['ACK'], payload=frm.payload).serialize()
self.tcp_handler.wfile.write(raw_bytes)
self.tcp_handler.wfile.flush()
continue
- if isinstance(frm, frame.SettingsFrame) and 'ACK' not in frm.flags:
+ if isinstance(frm, hyperframe.frame.SettingsFrame) and 'ACK' not in frm.flags:
self._apply_settings(frm.settings, hide)
- if isinstance(frm, frame.DataFrame) and frm.flow_controlled_length > 0:
+ if isinstance(frm, hyperframe.frame.DataFrame) and frm.flow_controlled_length > 0:
self._update_flow_control_window(frm.stream_id, frm.flow_controlled_length)
return frm
@@ -300,7 +303,7 @@ class HTTP2Protocol(object):
return True
def _handle_unexpected_frame(self, frm):
- if isinstance(frm, frame.SettingsFrame):
+ if isinstance(frm, hyperframe.frame.SettingsFrame):
return
if self.unhandled_frame_cb:
self.unhandled_frame_cb(frm)
@@ -308,7 +311,7 @@ class HTTP2Protocol(object):
def _receive_settings(self, hide=False):
while True:
frm = self.read_frame(hide)
- if isinstance(frm, frame.SettingsFrame):
+ if isinstance(frm, hyperframe.frame.SettingsFrame):
break
else:
self._handle_unexpected_frame(frm)
@@ -332,26 +335,26 @@ class HTTP2Protocol(object):
old_value = '-'
self.http2_settings[setting] = value
- frm = frame.SettingsFrame(flags=['ACK'])
+ frm = hyperframe.frame.SettingsFrame(flags=['ACK'])
self.send_frame(frm, hide)
def _update_flow_control_window(self, stream_id, increment):
- frm = frame.WindowUpdateFrame(stream_id=0, window_increment=increment)
+ frm = hyperframe.frame.WindowUpdateFrame(stream_id=0, window_increment=increment)
self.send_frame(frm)
- frm = frame.WindowUpdateFrame(stream_id=stream_id, window_increment=increment)
+ frm = hyperframe.frame.WindowUpdateFrame(stream_id=stream_id, window_increment=increment)
self.send_frame(frm)
def _create_headers(self, headers, stream_id, end_stream=True):
def frame_cls(chunks):
for i in chunks:
if i == 0:
- yield frame.HeadersFrame, i
+ yield hyperframe.frame.HeadersFrame, i
else:
- yield frame.ContinuationFrame, i
+ yield hyperframe.frame.ContinuationFrame, i
header_block_fragment = self.encoder.encode(headers.fields)
- chunk_size = self.http2_settings[frame.SettingsFrame.MAX_FRAME_SIZE]
+ chunk_size = self.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE]
chunks = range(0, len(header_block_fragment), chunk_size)
frms = [frm_cls(
flags=[],
@@ -372,9 +375,9 @@ class HTTP2Protocol(object):
if body is None or len(body) == 0:
return b''
- chunk_size = self.http2_settings[frame.SettingsFrame.MAX_FRAME_SIZE]
+ chunk_size = self.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE]
chunks = range(0, len(body), chunk_size)
- frms = [frame.DataFrame(
+ frms = [hyperframe.frame.DataFrame(
flags=[],
stream_id=stream_id,
data=body[i:i + chunk_size]) for i in chunks]
@@ -398,7 +401,7 @@ class HTTP2Protocol(object):
while True:
frm = self.read_frame()
if (
- (isinstance(frm, frame.HeadersFrame) or isinstance(frm, frame.ContinuationFrame)) and
+ (isinstance(frm, hyperframe.frame.HeadersFrame) or isinstance(frm, hyperframe.frame.ContinuationFrame)) and
(stream_id is None or frm.stream_id == stream_id)
):
stream_id = frm.stream_id
@@ -412,7 +415,7 @@ class HTTP2Protocol(object):
while body_expected:
frm = self.read_frame()
- if isinstance(frm, frame.DataFrame) and frm.stream_id == stream_id:
+ if isinstance(frm, hyperframe.frame.DataFrame) and frm.stream_id == stream_id:
body += frm.data
if 'END_STREAM' in frm.flags:
break
diff --git a/netlib/http/http2/framereader.py b/netlib/http/http2/framereader.py
new file mode 100644
index 00000000..d45be646
--- /dev/null
+++ b/netlib/http/http2/framereader.py
@@ -0,0 +1,21 @@
+import codecs
+
+import hyperframe
+
+
+def http2_read_raw_frame(rfile):
+ header = rfile.safe_read(9)
+ length = int(codecs.encode(header[:3], 'hex_codec'), 16)
+
+ if length == 4740180:
+ raise ValueError("Length field looks more like HTTP/1.1: %s" % rfile.peek(20))
+
+ body = rfile.safe_read(length)
+ return [header, body]
+
+
+def http2_read_frame(rfile):
+ header, body = http2_read_raw_frame(rfile)
+ frame, length = hyperframe.frame.Frame.parse_frame_header(header)
+ frame.parse_body(memoryview(body))
+ return frame
diff --git a/netlib/http/message.py b/netlib/http/message.py
index 13d401a7..d9654f26 100644
--- a/netlib/http/message.py
+++ b/netlib/http/message.py
@@ -4,9 +4,8 @@ import warnings
import six
-from .headers import Headers
-from .. import encoding, utils
-from ..utils import always_bytes
+from .. import encoding, utils, basetypes
+from . import headers
if six.PY2: # pragma: no cover
def _native(x):
@@ -20,10 +19,10 @@ else:
return x.decode("utf-8", "surrogateescape")
def _always_bytes(x):
- return always_bytes(x, "utf-8", "surrogateescape")
+ return utils.always_bytes(x, "utf-8", "surrogateescape")
-class MessageData(utils.Serializable):
+class MessageData(basetypes.Serializable):
def __eq__(self, other):
if isinstance(other, MessageData):
return self.__dict__ == other.__dict__
@@ -38,7 +37,7 @@ class MessageData(utils.Serializable):
def set_state(self, state):
for k, v in state.items():
if k == "headers":
- v = Headers.from_state(v)
+ v = headers.Headers.from_state(v)
setattr(self, k, v)
def get_state(self):
@@ -48,11 +47,11 @@ class MessageData(utils.Serializable):
@classmethod
def from_state(cls, state):
- state["headers"] = Headers.from_state(state["headers"])
+ state["headers"] = headers.Headers.from_state(state["headers"])
return cls(**state)
-class Message(utils.Serializable):
+class Message(basetypes.Serializable):
def __eq__(self, other):
if isinstance(other, Message):
return self.data == other.data
@@ -72,7 +71,7 @@ class Message(utils.Serializable):
@classmethod
def from_state(cls, state):
- state["headers"] = Headers.from_state(state["headers"])
+ state["headers"] = headers.Headers.from_state(state["headers"])
return cls(**state)
@property
diff --git a/netlib/http/multipart.py b/netlib/http/multipart.py
new file mode 100644
index 00000000..a135eb86
--- /dev/null
+++ b/netlib/http/multipart.py
@@ -0,0 +1,32 @@
+import re
+
+from . import headers
+
+
+def decode(hdrs, content):
+ """
+ Takes a multipart boundary encoded string and returns list of (key, value) tuples.
+ """
+ v = hdrs.get("content-type")
+ if v:
+ v = headers.parse_content_type(v)
+ if not v:
+ return []
+ try:
+ boundary = v[2]["boundary"].encode("ascii")
+ except (KeyError, UnicodeError):
+ return []
+
+ rx = re.compile(br'\bname="([^"]+)"')
+ r = []
+
+ for i in content.split(b"--" + boundary):
+ parts = i.splitlines()
+ if len(parts) > 1 and parts[0][0:2] != b"--":
+ match = rx.search(parts[1])
+ if match:
+ key = match.group(1)
+ value = b"".join(parts[3 + parts[2:].index(b""):])
+ r.append((key, value))
+ return r
+ return []
diff --git a/netlib/http/request.py b/netlib/http/request.py
index fa8d54aa..2fcea67d 100644
--- a/netlib/http/request.py
+++ b/netlib/http/request.py
@@ -6,7 +6,9 @@ import six
from six.moves import urllib
from netlib import utils
-from netlib.http import cookies
+import netlib.http.url
+from netlib.http import multipart
+from . import cookies
from .. import encoding
from ..multidict import MultiDictView
from .headers import Headers
@@ -179,11 +181,11 @@ class Request(Message):
"""
if self.first_line_format == "authority":
return "%s:%d" % (self.host, self.port)
- return utils.unparse_url(self.scheme, self.host, self.port, self.path)
+ return netlib.http.url.unparse(self.scheme, self.host, self.port, self.path)
@url.setter
def url(self, url):
- self.scheme, self.host, self.port, self.path = utils.parse_url(url)
+ self.scheme, self.host, self.port, self.path = netlib.http.url.parse(url)
def _parse_host_header(self):
"""Extract the host and port from Host header"""
@@ -219,7 +221,7 @@ class Request(Message):
"""
if self.first_line_format == "authority":
return "%s:%d" % (self.pretty_host, self.port)
- return utils.unparse_url(self.scheme, self.pretty_host, self.port, self.path)
+ return netlib.http.url.unparse(self.scheme, self.pretty_host, self.port, self.path)
@property
def query(self):
@@ -234,12 +236,12 @@ class Request(Message):
def _get_query(self):
_, _, _, _, query, _ = urllib.parse.urlparse(self.url)
- return tuple(utils.urldecode(query))
+ return tuple(netlib.http.url.decode(query))
def _set_query(self, value):
- query = utils.urlencode(value)
+ query = netlib.http.url.encode(value)
scheme, netloc, path, params, _, fragment = urllib.parse.urlparse(self.url)
- _, _, _, self.path = utils.parse_url(
+ _, _, _, self.path = netlib.http.url.parse(
urllib.parse.urlunparse([scheme, netloc, path, params, query, fragment]))
@query.setter
@@ -287,7 +289,7 @@ class Request(Message):
components = map(lambda x: urllib.parse.quote(x, safe=""), components)
path = "/" + "/".join(components)
scheme, netloc, _, params, query, fragment = urllib.parse.urlparse(self.url)
- _, _, _, self.path = utils.parse_url(
+ _, _, _, self.path = netlib.http.url.parse(
urllib.parse.urlunparse([scheme, netloc, path, params, query, fragment]))
def anticache(self):
@@ -339,7 +341,7 @@ class Request(Message):
def _get_urlencoded_form(self):
is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower()
if is_valid_content_type:
- return tuple(utils.urldecode(self.content))
+ return tuple(netlib.http.url.decode(self.content))
return ()
def _set_urlencoded_form(self, value):
@@ -348,7 +350,7 @@ class Request(Message):
This will overwrite the existing content if there is one.
"""
self.headers["content-type"] = "application/x-www-form-urlencoded"
- self.content = utils.urlencode(value)
+ self.content = netlib.http.url.encode(value)
@urlencoded_form.setter
def urlencoded_form(self, value):
@@ -368,7 +370,7 @@ class Request(Message):
def _get_multipart_form(self):
is_valid_content_type = "multipart/form-data" in self.headers.get("content-type", "").lower()
if is_valid_content_type:
- return utils.multipartdecode(self.headers, self.content)
+ return multipart.decode(self.headers, self.content)
return ()
def _set_multipart_form(self, value):
diff --git a/netlib/http/response.py b/netlib/http/response.py
index a6a5bf47..858b3aea 100644
--- a/netlib/http/response.py
+++ b/netlib/http/response.py
@@ -7,7 +7,7 @@ from . import cookies
from .headers import Headers
from .message import Message, _native, _always_bytes, MessageData
from ..multidict import MultiDictView
-from .. import utils
+from .. import human
class ResponseData(MessageData):
@@ -36,7 +36,7 @@ class Response(Message):
if self.content:
details = "{}, {}".format(
self.headers.get("content-type", "unknown content type"),
- utils.pretty_size(len(self.content))
+ human.pretty_size(len(self.content))
)
else:
details = "no content"
diff --git a/netlib/http/url.py b/netlib/http/url.py
new file mode 100644
index 00000000..8ce28578
--- /dev/null
+++ b/netlib/http/url.py
@@ -0,0 +1,96 @@
+import six
+from six.moves import urllib
+
+from .. import utils
+
+
+# PY2 workaround
+def decode_parse_result(result, enc):
+ if hasattr(result, "decode"):
+ return result.decode(enc)
+ else:
+ return urllib.parse.ParseResult(*[x.decode(enc) for x in result])
+
+
+# PY2 workaround
+def encode_parse_result(result, enc):
+ if hasattr(result, "encode"):
+ return result.encode(enc)
+ else:
+ return urllib.parse.ParseResult(*[x.encode(enc) for x in result])
+
+
+def parse(url):
+ """
+ URL-parsing function that checks that
+ - port is an integer 0-65535
+ - host is a valid IDNA-encoded hostname with no null-bytes
+ - path is valid ASCII
+
+ Args:
+ A URL (as bytes or as unicode)
+
+ Returns:
+ A (scheme, host, port, path) tuple
+
+ Raises:
+ ValueError, if the URL is not properly formatted.
+ """
+ parsed = urllib.parse.urlparse(url)
+
+ if not parsed.hostname:
+ raise ValueError("No hostname given")
+
+ if isinstance(url, six.binary_type):
+ host = parsed.hostname
+
+ # this should not raise a ValueError,
+ # but we try to be very forgiving here and accept just everything.
+ # decode_parse_result(parsed, "ascii")
+ else:
+ host = parsed.hostname.encode("idna")
+ parsed = encode_parse_result(parsed, "ascii")
+
+ port = parsed.port
+ if not port:
+ port = 443 if parsed.scheme == b"https" else 80
+
+ full_path = urllib.parse.urlunparse(
+ (b"", b"", parsed.path, parsed.params, parsed.query, parsed.fragment)
+ )
+ if not full_path.startswith(b"/"):
+ full_path = b"/" + full_path
+
+ if not utils.is_valid_host(host):
+ raise ValueError("Invalid Host")
+ if not utils.is_valid_port(port):
+ raise ValueError("Invalid Port")
+
+ return parsed.scheme, host, port, full_path
+
+
+def unparse(scheme, host, port, path=""):
+ """
+ Returns a URL string, constructed from the specified components.
+
+ Args:
+ All args must be str.
+ """
+ if path == "*":
+ path = ""
+ return "%s://%s%s" % (scheme, utils.hostport(scheme, host, port), path)
+
+
+def encode(s):
+ """
+ Takes a list of (key, value) tuples and returns a urlencoded string.
+ """
+ s = [tuple(i) for i in s]
+ return urllib.parse.urlencode(s, False)
+
+
+def decode(s):
+ """
+ Takes a urlencoded string and returns a list of (key, value) tuples.
+ """
+ return urllib.parse.parse_qsl(s, keep_blank_values=True)
diff --git a/netlib/human.py b/netlib/human.py
new file mode 100644
index 00000000..a007adc7
--- /dev/null
+++ b/netlib/human.py
@@ -0,0 +1,50 @@
+
+SIZE_TABLE = [
+ ("b", 1024 ** 0),
+ ("k", 1024 ** 1),
+ ("m", 1024 ** 2),
+ ("g", 1024 ** 3),
+ ("t", 1024 ** 4),
+]
+
+SIZE_UNITS = dict(SIZE_TABLE)
+
+
+def pretty_size(size):
+ for bottom, top in zip(SIZE_TABLE, SIZE_TABLE[1:]):
+ if bottom[1] <= size < top[1]:
+ suf = bottom[0]
+ lim = bottom[1]
+ x = round(size / lim, 2)
+ if x == int(x):
+ x = int(x)
+ return str(x) + suf
+ return "%s%s" % (size, SIZE_TABLE[0][0])
+
+
+def parse_size(s):
+ try:
+ return int(s)
+ except ValueError:
+ pass
+ for i in SIZE_UNITS.keys():
+ if s.endswith(i):
+ try:
+ return int(s[:-1]) * SIZE_UNITS[i]
+ except ValueError:
+ break
+ raise ValueError("Invalid size specification.")
+
+
+def pretty_duration(secs):
+ formatters = [
+ (100, "{:.0f}s"),
+ (10, "{:2.1f}s"),
+ (1, "{:1.2f}s"),
+ ]
+
+ for limit, formatter in formatters:
+ if secs >= limit:
+ return formatter.format(secs)
+ # less than 1 sec
+ return "{:.0f}ms".format(secs * 1000)
diff --git a/netlib/multidict.py b/netlib/multidict.py
index 98fde7e3..6139d60a 100644
--- a/netlib/multidict.py
+++ b/netlib/multidict.py
@@ -9,12 +9,11 @@ except ImportError: # pragma: no cover
from collections import MutableMapping # Workaround for Python < 3.3
import six
-
-from .utils import Serializable
+from . import basetypes
@six.add_metaclass(ABCMeta)
-class _MultiDict(MutableMapping, Serializable):
+class _MultiDict(MutableMapping, basetypes.Serializable):
def __repr__(self):
fields = (
repr(field)
@@ -171,6 +170,14 @@ class _MultiDict(MutableMapping, Serializable):
else:
return super(_MultiDict, self).items()
+ def clear(self, key):
+ """
+ Removes all items with the specified key, and does not raise an
+ exception if the key does not exist.
+ """
+ if key in self:
+ del self[key]
+
def to_dict(self):
"""
Get the MultiDict as a plain Python dict.
diff --git a/netlib/odict.py b/netlib/odict.py
index 8a638dab..87887a29 100644
--- a/netlib/odict.py
+++ b/netlib/odict.py
@@ -3,10 +3,10 @@ import copy
import six
-from .utils import Serializable, safe_subn
+from . import basetypes, utils
-class ODict(Serializable):
+class ODict(basetypes.Serializable):
"""
A dictionary-like object for managing ordered (key, value) data. Think
@@ -139,9 +139,9 @@ class ODict(Serializable):
"""
new, count = [], 0
for k, v in self.lst:
- k, c = safe_subn(pattern, repl, k, *args, **kwargs)
+ k, c = utils.safe_subn(pattern, repl, k, *args, **kwargs)
count += c
- v, c = safe_subn(pattern, repl, v, *args, **kwargs)
+ v, c = utils.safe_subn(pattern, repl, v, *args, **kwargs)
count += c
new.append([k, v])
self.lst = new
diff --git a/netlib/tcp.py b/netlib/tcp.py
index c7231dbb..5662c973 100644
--- a/netlib/tcp.py
+++ b/netlib/tcp.py
@@ -16,7 +16,7 @@ import six
import OpenSSL
from OpenSSL import SSL
-from . import certutils, version_check, utils
+from . import certutils, version_check, basetypes
# This is a rather hackish way to make sure that
# the latest version of pyOpenSSL is actually installed.
@@ -302,7 +302,7 @@ class Reader(_FileLike):
raise NotImplementedError("Can only peek into (pyOpenSSL) sockets")
-class Address(utils.Serializable):
+class Address(basetypes.Serializable):
"""
This class wraps an IPv4/IPv6 tuple to provide named attributes and
diff --git a/netlib/utils.py b/netlib/utils.py
index 174f616d..b8408d1d 100644
--- a/netlib/utils.py
+++ b/netlib/utils.py
@@ -3,47 +3,11 @@ import os.path
import re
import codecs
import unicodedata
-from abc import ABCMeta, abstractmethod
import importlib
import inspect
import six
-from six.moves import urllib
-import hyperframe
-
-
-@six.add_metaclass(ABCMeta)
-class Serializable(object):
- """
- Abstract Base Class that defines an API to save an object's state and restore it later on.
- """
-
- @classmethod
- @abstractmethod
- def from_state(cls, state):
- """
- Create a new object from the given state.
- """
- raise NotImplementedError()
-
- @abstractmethod
- def get_state(self):
- """
- Retrieve object state.
- """
- raise NotImplementedError()
-
- @abstractmethod
- def set_state(self, state):
- """
- Set object state to the given state.
- """
- raise NotImplementedError()
-
- def copy(self):
- return self.from_state(self.get_state())
-
def always_bytes(unicode_or_bytes, *encode_args):
if isinstance(unicode_or_bytes, six.text_type):
@@ -69,14 +33,6 @@ def native(s, *encoding_opts):
return s
-def isascii(bytes):
- try:
- bytes.decode("ascii")
- except ValueError:
- return False
- return True
-
-
def clean_bin(s, keep_spacing=True):
"""
Cleans binary data to make it safe to display.
@@ -161,22 +117,6 @@ class BiDi(object):
return self.values.get(n, default)
-def pretty_size(size):
- suffixes = [
- ("B", 2 ** 10),
- ("kB", 2 ** 20),
- ("MB", 2 ** 30),
- ]
- for suf, lim in suffixes:
- if size >= lim:
- continue
- else:
- x = round(size / float(lim / 2 ** 10), 2)
- if x == int(x):
- x = int(x)
- return str(x) + suf
-
-
class Data(object):
def __init__(self, name):
@@ -222,83 +162,6 @@ def is_valid_port(port):
return 0 <= port <= 65535
-# PY2 workaround
-def decode_parse_result(result, enc):
- if hasattr(result, "decode"):
- return result.decode(enc)
- else:
- return urllib.parse.ParseResult(*[x.decode(enc) for x in result])
-
-
-# PY2 workaround
-def encode_parse_result(result, enc):
- if hasattr(result, "encode"):
- return result.encode(enc)
- else:
- return urllib.parse.ParseResult(*[x.encode(enc) for x in result])
-
-
-def parse_url(url):
- """
- URL-parsing function that checks that
- - port is an integer 0-65535
- - host is a valid IDNA-encoded hostname with no null-bytes
- - path is valid ASCII
-
- Args:
- A URL (as bytes or as unicode)
-
- Returns:
- A (scheme, host, port, path) tuple
-
- Raises:
- ValueError, if the URL is not properly formatted.
- """
- parsed = urllib.parse.urlparse(url)
-
- if not parsed.hostname:
- raise ValueError("No hostname given")
-
- if isinstance(url, six.binary_type):
- host = parsed.hostname
-
- # this should not raise a ValueError,
- # but we try to be very forgiving here and accept just everything.
- # decode_parse_result(parsed, "ascii")
- else:
- host = parsed.hostname.encode("idna")
- parsed = encode_parse_result(parsed, "ascii")
-
- port = parsed.port
- if not port:
- port = 443 if parsed.scheme == b"https" else 80
-
- full_path = urllib.parse.urlunparse(
- (b"", b"", parsed.path, parsed.params, parsed.query, parsed.fragment)
- )
- if not full_path.startswith(b"/"):
- full_path = b"/" + full_path
-
- if not is_valid_host(host):
- raise ValueError("Invalid Host")
- if not is_valid_port(port):
- raise ValueError("Invalid Port")
-
- return parsed.scheme, host, port, full_path
-
-
-def get_header_tokens(headers, key):
- """
- Retrieve all tokens for a header key. A number of different headers
- follow a pattern where each header line can containe comma-separated
- tokens, and headers can be set multiple times.
- """
- if key not in headers:
- return []
- tokens = headers[key].split(",")
- return [token.strip() for token in tokens]
-
-
def hostport(scheme, host, port):
"""
Returns the host component, with a port specifcation if needed.
@@ -312,107 +175,6 @@ def hostport(scheme, host, port):
return "%s:%d" % (host, port)
-def unparse_url(scheme, host, port, path=""):
- """
- Returns a URL string, constructed from the specified components.
-
- Args:
- All args must be str.
- """
- if path == "*":
- path = ""
- return "%s://%s%s" % (scheme, hostport(scheme, host, port), path)
-
-
-def urlencode(s):
- """
- Takes a list of (key, value) tuples and returns a urlencoded string.
- """
- s = [tuple(i) for i in s]
- return urllib.parse.urlencode(s, False)
-
-
-def urldecode(s):
- """
- Takes a urlencoded string and returns a list of (key, value) tuples.
- """
- return urllib.parse.parse_qsl(s, keep_blank_values=True)
-
-
-def parse_content_type(c):
- """
- A simple parser for content-type values. Returns a (type, subtype,
- parameters) tuple, where type and subtype are strings, and parameters
- is a dict. If the string could not be parsed, return None.
-
- E.g. the following string:
-
- text/html; charset=UTF-8
-
- Returns:
-
- ("text", "html", {"charset": "UTF-8"})
- """
- parts = c.split(";", 1)
- ts = parts[0].split("/", 1)
- if len(ts) != 2:
- return None
- d = {}
- if len(parts) == 2:
- for i in parts[1].split(";"):
- clause = i.split("=", 1)
- if len(clause) == 2:
- d[clause[0].strip()] = clause[1].strip()
- return ts[0].lower(), ts[1].lower(), d
-
-
-def multipartdecode(headers, content):
- """
- Takes a multipart boundary encoded string and returns list of (key, value) tuples.
- """
- v = headers.get("content-type")
- if v:
- v = parse_content_type(v)
- if not v:
- return []
- try:
- boundary = v[2]["boundary"].encode("ascii")
- except (KeyError, UnicodeError):
- return []
-
- rx = re.compile(br'\bname="([^"]+)"')
- r = []
-
- for i in content.split(b"--" + boundary):
- parts = i.splitlines()
- if len(parts) > 1 and parts[0][0:2] != b"--":
- match = rx.search(parts[1])
- if match:
- key = match.group(1)
- value = b"".join(parts[3 + parts[2:].index(b""):])
- r.append((key, value))
- return r
- return []
-
-
-def http2_read_raw_frame(rfile):
- header = rfile.safe_read(9)
- length = int(codecs.encode(header[:3], 'hex_codec'), 16)
-
- if length == 4740180:
- raise ValueError("Length field looks more like HTTP/1.1: %s" % rfile.peek(20))
-
- body = rfile.safe_read(length)
- return [header, body]
-
-
-def http2_read_frame(rfile):
- header, body = http2_read_raw_frame(rfile)
- frame, length = hyperframe.frame.Frame.parse_frame_header(header)
- frame.parse_body(memoryview(body))
- return frame
-
-
def safe_subn(pattern, repl, target, *args, **kwargs):
"""
There are Unicode conversion problems with re.subn. We try to smooth
diff --git a/netlib/websockets/frame.py b/netlib/websockets/frame.py
index da5a97f3..cf8917c1 100644
--- a/netlib/websockets/frame.py
+++ b/netlib/websockets/frame.py
@@ -9,6 +9,7 @@ import six
from .protocol import Masker
from netlib import tcp
from netlib import utils
+from netlib import human
MAX_16_BIT_INT = (1 << 16)
@@ -98,7 +99,7 @@ class FrameHeader(object):
if self.masking_key:
vals.append(":key=%s" % repr(self.masking_key))
if self.payload_length:
- vals.append(" %s" % utils.pretty_size(self.payload_length))
+ vals.append(" %s" % human.pretty_size(self.payload_length))
return "".join(vals)
def human_readable(self):
generated by cgit v1.2.3 (git 2.25.1) at 2025-10-31 02:47:12 +0000