diff options
author | Aldo Cortesi <aldo@corte.si> | 2016-05-31 21:03:42 +1200 |
---|---|---|
committer | Aldo Cortesi <aldo@corte.si> | 2016-05-31 21:03:42 +1200 |
commit | a7abf8b731658b4e7ed8705f7a94a6a62f08d51d (patch) | |
tree | cb86e2e483530f5e1e8b0c5d60839de21fcf7390 /netlib/http | |
parent | 2f526393d29b6a03e43d1f6240175b4dfb13dc7d (diff) | |
parent | 4da125b6a098cc0fd8b1fd2878584beb5df75c6c (diff) | |
download | mitmproxy-a7abf8b731658b4e7ed8705f7a94a6a62f08d51d.tar.gz mitmproxy-a7abf8b731658b4e7ed8705f7a94a6a62f08d51d.tar.bz2 mitmproxy-a7abf8b731658b4e7ed8705f7a94a6a62f08d51d.zip |
Merge pull request #1179 from cortesi/reorg
Start reorganising */utils.py
Diffstat (limited to 'netlib/http')
-rw-r--r-- | netlib/http/headers.py | 27 | ||||
-rw-r--r-- | netlib/http/http1/read.py | 17 | ||||
-rw-r--r-- | netlib/http/http2/connections.py | 63 | ||||
-rw-r--r-- | netlib/http/http2/framereader.py | 21 | ||||
-rw-r--r-- | netlib/http/message.py | 17 | ||||
-rw-r--r-- | netlib/http/multipart.py | 32 | ||||
-rw-r--r-- | netlib/http/request.py | 24 | ||||
-rw-r--r-- | netlib/http/response.py | 4 | ||||
-rw-r--r-- | netlib/http/url.py | 96 |
9 files changed, 246 insertions, 55 deletions
diff --git a/netlib/http/headers.py b/netlib/http/headers.py index 6165fd61..fa7b7180 100644 --- a/netlib/http/headers.py +++ b/netlib/http/headers.py @@ -175,3 +175,30 @@ class Headers(MultiDict): fields.append([name, value]) self.fields = fields return replacements + + +def parse_content_type(c): + """ + A simple parser for content-type values. Returns a (type, subtype, + parameters) tuple, where type and subtype are strings, and parameters + is a dict. If the string could not be parsed, return None. + + E.g. the following string: + + text/html; charset=UTF-8 + + Returns: + + ("text", "html", {"charset": "UTF-8"}) + """ + parts = c.split(";", 1) + ts = parts[0].split("/", 1) + if len(ts) != 2: + return None + d = {} + if len(parts) == 2: + for i in parts[1].split(";"): + clause = i.split("=", 1) + if len(clause) == 2: + d[clause[0].strip()] = clause[1].strip() + return ts[0].lower(), ts[1].lower(), d diff --git a/netlib/http/http1/read.py b/netlib/http/http1/read.py index d30976bd..5783ec67 100644 --- a/netlib/http/http1/read.py +++ b/netlib/http/http1/read.py @@ -6,6 +6,19 @@ import re from ... import utils from ...exceptions import HttpReadDisconnect, HttpSyntaxException, HttpException, TcpDisconnect from .. import Request, Response, Headers +from .. import url + + +def get_header_tokens(headers, key): + """ + Retrieve all tokens for a header key. A number of different headers + follow a pattern where each header line can containe comma-separated + tokens, and headers can be set multiple times. + """ + if key not in headers: + return [] + tokens = headers[key].split(",") + return [token.strip() for token in tokens] def read_request(rfile, body_size_limit=None): @@ -147,7 +160,7 @@ def connection_close(http_version, headers): """ # At first, check if we have an explicit Connection header. if "connection" in headers: - tokens = utils.get_header_tokens(headers, "connection") + tokens = get_header_tokens(headers, "connection") if "close" in tokens: return True elif "keep-alive" in tokens: @@ -240,7 +253,7 @@ def _read_request_line(rfile): scheme, path = None, None else: form = "absolute" - scheme, host, port, path = utils.parse_url(path) + scheme, host, port, path = url.parse(path) _check_http_version(http_version) except ValueError: diff --git a/netlib/http/http2/connections.py b/netlib/http/http2/connections.py index 6b91f2ff..16bdf618 100644 --- a/netlib/http/http2/connections.py +++ b/netlib/http/http2/connections.py @@ -2,11 +2,12 @@ from __future__ import (absolute_import, print_function, division) import itertools import time +import hyperframe.frame + from hpack.hpack import Encoder, Decoder from ... import utils -from .. import Headers, Response, Request - -from hyperframe import frame +from .. import Headers, Response, Request, url +from . import framereader class TCPHandler(object): @@ -38,12 +39,12 @@ class HTTP2Protocol(object): CLIENT_CONNECTION_PREFACE = b'PRI * HTTP/2.0\r\n\r\nSM\r\n\r\n' HTTP2_DEFAULT_SETTINGS = { - frame.SettingsFrame.HEADER_TABLE_SIZE: 4096, - frame.SettingsFrame.ENABLE_PUSH: 1, - frame.SettingsFrame.MAX_CONCURRENT_STREAMS: None, - frame.SettingsFrame.INITIAL_WINDOW_SIZE: 2 ** 16 - 1, - frame.SettingsFrame.MAX_FRAME_SIZE: 2 ** 14, - frame.SettingsFrame.MAX_HEADER_LIST_SIZE: None, + hyperframe.frame.SettingsFrame.HEADER_TABLE_SIZE: 4096, + hyperframe.frame.SettingsFrame.ENABLE_PUSH: 1, + hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: None, + hyperframe.frame.SettingsFrame.INITIAL_WINDOW_SIZE: 2 ** 16 - 1, + hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE: 2 ** 14, + hyperframe.frame.SettingsFrame.MAX_HEADER_LIST_SIZE: None, } def __init__( @@ -117,7 +118,7 @@ class HTTP2Protocol(object): else: first_line_format = "absolute" # FIXME: verify if path or :host contains what we need - scheme, host, port, _ = utils.parse_url(path) + scheme, host, port, _ = url.parse(path) scheme = scheme.decode('ascii') host = host.decode('ascii') @@ -253,9 +254,9 @@ class HTTP2Protocol(object): magic = self.tcp_handler.rfile.safe_read(magic_length) assert magic == self.CLIENT_CONNECTION_PREFACE - frm = frame.SettingsFrame(settings={ - frame.SettingsFrame.ENABLE_PUSH: 0, - frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 1, + frm = hyperframe.frame.SettingsFrame(settings={ + hyperframe.frame.SettingsFrame.ENABLE_PUSH: 0, + hyperframe.frame.SettingsFrame.MAX_CONCURRENT_STREAMS: 1, }) self.send_frame(frm, hide=True) self._receive_settings(hide=True) @@ -266,7 +267,7 @@ class HTTP2Protocol(object): self.tcp_handler.wfile.write(self.CLIENT_CONNECTION_PREFACE) - self.send_frame(frame.SettingsFrame(), hide=True) + self.send_frame(hyperframe.frame.SettingsFrame(), hide=True) self._receive_settings(hide=True) # server announces own settings self._receive_settings(hide=True) # server acks my settings @@ -279,18 +280,18 @@ class HTTP2Protocol(object): def read_frame(self, hide=False): while True: - frm = utils.http2_read_frame(self.tcp_handler.rfile) + frm = framereader.http2_read_frame(self.tcp_handler.rfile) if not hide and self.dump_frames: # pragma no cover print(frm.human_readable("<<")) - if isinstance(frm, frame.PingFrame): - raw_bytes = frame.PingFrame(flags=['ACK'], payload=frm.payload).serialize() + if isinstance(frm, hyperframe.frame.PingFrame): + raw_bytes = hyperframe.frame.PingFrame(flags=['ACK'], payload=frm.payload).serialize() self.tcp_handler.wfile.write(raw_bytes) self.tcp_handler.wfile.flush() continue - if isinstance(frm, frame.SettingsFrame) and 'ACK' not in frm.flags: + if isinstance(frm, hyperframe.frame.SettingsFrame) and 'ACK' not in frm.flags: self._apply_settings(frm.settings, hide) - if isinstance(frm, frame.DataFrame) and frm.flow_controlled_length > 0: + if isinstance(frm, hyperframe.frame.DataFrame) and frm.flow_controlled_length > 0: self._update_flow_control_window(frm.stream_id, frm.flow_controlled_length) return frm @@ -302,7 +303,7 @@ class HTTP2Protocol(object): return True def _handle_unexpected_frame(self, frm): - if isinstance(frm, frame.SettingsFrame): + if isinstance(frm, hyperframe.frame.SettingsFrame): return if self.unhandled_frame_cb: self.unhandled_frame_cb(frm) @@ -310,7 +311,7 @@ class HTTP2Protocol(object): def _receive_settings(self, hide=False): while True: frm = self.read_frame(hide) - if isinstance(frm, frame.SettingsFrame): + if isinstance(frm, hyperframe.frame.SettingsFrame): break else: self._handle_unexpected_frame(frm) @@ -334,26 +335,26 @@ class HTTP2Protocol(object): old_value = '-' self.http2_settings[setting] = value - frm = frame.SettingsFrame(flags=['ACK']) + frm = hyperframe.frame.SettingsFrame(flags=['ACK']) self.send_frame(frm, hide) def _update_flow_control_window(self, stream_id, increment): - frm = frame.WindowUpdateFrame(stream_id=0, window_increment=increment) + frm = hyperframe.frame.WindowUpdateFrame(stream_id=0, window_increment=increment) self.send_frame(frm) - frm = frame.WindowUpdateFrame(stream_id=stream_id, window_increment=increment) + frm = hyperframe.frame.WindowUpdateFrame(stream_id=stream_id, window_increment=increment) self.send_frame(frm) def _create_headers(self, headers, stream_id, end_stream=True): def frame_cls(chunks): for i in chunks: if i == 0: - yield frame.HeadersFrame, i + yield hyperframe.frame.HeadersFrame, i else: - yield frame.ContinuationFrame, i + yield hyperframe.frame.ContinuationFrame, i header_block_fragment = self.encoder.encode(headers.fields) - chunk_size = self.http2_settings[frame.SettingsFrame.MAX_FRAME_SIZE] + chunk_size = self.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE] chunks = range(0, len(header_block_fragment), chunk_size) frms = [frm_cls( flags=[], @@ -374,9 +375,9 @@ class HTTP2Protocol(object): if body is None or len(body) == 0: return b'' - chunk_size = self.http2_settings[frame.SettingsFrame.MAX_FRAME_SIZE] + chunk_size = self.http2_settings[hyperframe.frame.SettingsFrame.MAX_FRAME_SIZE] chunks = range(0, len(body), chunk_size) - frms = [frame.DataFrame( + frms = [hyperframe.frame.DataFrame( flags=[], stream_id=stream_id, data=body[i:i + chunk_size]) for i in chunks] @@ -400,7 +401,7 @@ class HTTP2Protocol(object): while True: frm = self.read_frame() if ( - (isinstance(frm, frame.HeadersFrame) or isinstance(frm, frame.ContinuationFrame)) and + (isinstance(frm, hyperframe.frame.HeadersFrame) or isinstance(frm, hyperframe.frame.ContinuationFrame)) and (stream_id is None or frm.stream_id == stream_id) ): stream_id = frm.stream_id @@ -414,7 +415,7 @@ class HTTP2Protocol(object): while body_expected: frm = self.read_frame() - if isinstance(frm, frame.DataFrame) and frm.stream_id == stream_id: + if isinstance(frm, hyperframe.frame.DataFrame) and frm.stream_id == stream_id: body += frm.data if 'END_STREAM' in frm.flags: break diff --git a/netlib/http/http2/framereader.py b/netlib/http/http2/framereader.py new file mode 100644 index 00000000..d45be646 --- /dev/null +++ b/netlib/http/http2/framereader.py @@ -0,0 +1,21 @@ +import codecs + +import hyperframe + + +def http2_read_raw_frame(rfile): + header = rfile.safe_read(9) + length = int(codecs.encode(header[:3], 'hex_codec'), 16) + + if length == 4740180: + raise ValueError("Length field looks more like HTTP/1.1: %s" % rfile.peek(20)) + + body = rfile.safe_read(length) + return [header, body] + + +def http2_read_frame(rfile): + header, body = http2_read_raw_frame(rfile) + frame, length = hyperframe.frame.Frame.parse_frame_header(header) + frame.parse_body(memoryview(body)) + return frame diff --git a/netlib/http/message.py b/netlib/http/message.py index 13d401a7..d9654f26 100644 --- a/netlib/http/message.py +++ b/netlib/http/message.py @@ -4,9 +4,8 @@ import warnings import six -from .headers import Headers -from .. import encoding, utils -from ..utils import always_bytes +from .. import encoding, utils, basetypes +from . import headers if six.PY2: # pragma: no cover def _native(x): @@ -20,10 +19,10 @@ else: return x.decode("utf-8", "surrogateescape") def _always_bytes(x): - return always_bytes(x, "utf-8", "surrogateescape") + return utils.always_bytes(x, "utf-8", "surrogateescape") -class MessageData(utils.Serializable): +class MessageData(basetypes.Serializable): def __eq__(self, other): if isinstance(other, MessageData): return self.__dict__ == other.__dict__ @@ -38,7 +37,7 @@ class MessageData(utils.Serializable): def set_state(self, state): for k, v in state.items(): if k == "headers": - v = Headers.from_state(v) + v = headers.Headers.from_state(v) setattr(self, k, v) def get_state(self): @@ -48,11 +47,11 @@ class MessageData(utils.Serializable): @classmethod def from_state(cls, state): - state["headers"] = Headers.from_state(state["headers"]) + state["headers"] = headers.Headers.from_state(state["headers"]) return cls(**state) -class Message(utils.Serializable): +class Message(basetypes.Serializable): def __eq__(self, other): if isinstance(other, Message): return self.data == other.data @@ -72,7 +71,7 @@ class Message(utils.Serializable): @classmethod def from_state(cls, state): - state["headers"] = Headers.from_state(state["headers"]) + state["headers"] = headers.Headers.from_state(state["headers"]) return cls(**state) @property diff --git a/netlib/http/multipart.py b/netlib/http/multipart.py new file mode 100644 index 00000000..a135eb86 --- /dev/null +++ b/netlib/http/multipart.py @@ -0,0 +1,32 @@ +import re + +from . import headers + + +def decode(hdrs, content): + """ + Takes a multipart boundary encoded string and returns list of (key, value) tuples. + """ + v = hdrs.get("content-type") + if v: + v = headers.parse_content_type(v) + if not v: + return [] + try: + boundary = v[2]["boundary"].encode("ascii") + except (KeyError, UnicodeError): + return [] + + rx = re.compile(br'\bname="([^"]+)"') + r = [] + + for i in content.split(b"--" + boundary): + parts = i.splitlines() + if len(parts) > 1 and parts[0][0:2] != b"--": + match = rx.search(parts[1]) + if match: + key = match.group(1) + value = b"".join(parts[3 + parts[2:].index(b""):]) + r.append((key, value)) + return r + return [] diff --git a/netlib/http/request.py b/netlib/http/request.py index fa8d54aa..2fcea67d 100644 --- a/netlib/http/request.py +++ b/netlib/http/request.py @@ -6,7 +6,9 @@ import six from six.moves import urllib from netlib import utils -from netlib.http import cookies +import netlib.http.url +from netlib.http import multipart +from . import cookies from .. import encoding from ..multidict import MultiDictView from .headers import Headers @@ -179,11 +181,11 @@ class Request(Message): """ if self.first_line_format == "authority": return "%s:%d" % (self.host, self.port) - return utils.unparse_url(self.scheme, self.host, self.port, self.path) + return netlib.http.url.unparse(self.scheme, self.host, self.port, self.path) @url.setter def url(self, url): - self.scheme, self.host, self.port, self.path = utils.parse_url(url) + self.scheme, self.host, self.port, self.path = netlib.http.url.parse(url) def _parse_host_header(self): """Extract the host and port from Host header""" @@ -219,7 +221,7 @@ class Request(Message): """ if self.first_line_format == "authority": return "%s:%d" % (self.pretty_host, self.port) - return utils.unparse_url(self.scheme, self.pretty_host, self.port, self.path) + return netlib.http.url.unparse(self.scheme, self.pretty_host, self.port, self.path) @property def query(self): @@ -234,12 +236,12 @@ class Request(Message): def _get_query(self): _, _, _, _, query, _ = urllib.parse.urlparse(self.url) - return tuple(utils.urldecode(query)) + return tuple(netlib.http.url.decode(query)) def _set_query(self, value): - query = utils.urlencode(value) + query = netlib.http.url.encode(value) scheme, netloc, path, params, _, fragment = urllib.parse.urlparse(self.url) - _, _, _, self.path = utils.parse_url( + _, _, _, self.path = netlib.http.url.parse( urllib.parse.urlunparse([scheme, netloc, path, params, query, fragment])) @query.setter @@ -287,7 +289,7 @@ class Request(Message): components = map(lambda x: urllib.parse.quote(x, safe=""), components) path = "/" + "/".join(components) scheme, netloc, _, params, query, fragment = urllib.parse.urlparse(self.url) - _, _, _, self.path = utils.parse_url( + _, _, _, self.path = netlib.http.url.parse( urllib.parse.urlunparse([scheme, netloc, path, params, query, fragment])) def anticache(self): @@ -339,7 +341,7 @@ class Request(Message): def _get_urlencoded_form(self): is_valid_content_type = "application/x-www-form-urlencoded" in self.headers.get("content-type", "").lower() if is_valid_content_type: - return tuple(utils.urldecode(self.content)) + return tuple(netlib.http.url.decode(self.content)) return () def _set_urlencoded_form(self, value): @@ -348,7 +350,7 @@ class Request(Message): This will overwrite the existing content if there is one. """ self.headers["content-type"] = "application/x-www-form-urlencoded" - self.content = utils.urlencode(value) + self.content = netlib.http.url.encode(value) @urlencoded_form.setter def urlencoded_form(self, value): @@ -368,7 +370,7 @@ class Request(Message): def _get_multipart_form(self): is_valid_content_type = "multipart/form-data" in self.headers.get("content-type", "").lower() if is_valid_content_type: - return utils.multipartdecode(self.headers, self.content) + return multipart.decode(self.headers, self.content) return () def _set_multipart_form(self, value): diff --git a/netlib/http/response.py b/netlib/http/response.py index a6a5bf47..858b3aea 100644 --- a/netlib/http/response.py +++ b/netlib/http/response.py @@ -7,7 +7,7 @@ from . import cookies from .headers import Headers from .message import Message, _native, _always_bytes, MessageData from ..multidict import MultiDictView -from .. import utils +from .. import human class ResponseData(MessageData): @@ -36,7 +36,7 @@ class Response(Message): if self.content: details = "{}, {}".format( self.headers.get("content-type", "unknown content type"), - utils.pretty_size(len(self.content)) + human.pretty_size(len(self.content)) ) else: details = "no content" diff --git a/netlib/http/url.py b/netlib/http/url.py new file mode 100644 index 00000000..8ce28578 --- /dev/null +++ b/netlib/http/url.py @@ -0,0 +1,96 @@ +import six +from six.moves import urllib + +from .. import utils + + +# PY2 workaround +def decode_parse_result(result, enc): + if hasattr(result, "decode"): + return result.decode(enc) + else: + return urllib.parse.ParseResult(*[x.decode(enc) for x in result]) + + +# PY2 workaround +def encode_parse_result(result, enc): + if hasattr(result, "encode"): + return result.encode(enc) + else: + return urllib.parse.ParseResult(*[x.encode(enc) for x in result]) + + +def parse(url): + """ + URL-parsing function that checks that + - port is an integer 0-65535 + - host is a valid IDNA-encoded hostname with no null-bytes + - path is valid ASCII + + Args: + A URL (as bytes or as unicode) + + Returns: + A (scheme, host, port, path) tuple + + Raises: + ValueError, if the URL is not properly formatted. + """ + parsed = urllib.parse.urlparse(url) + + if not parsed.hostname: + raise ValueError("No hostname given") + + if isinstance(url, six.binary_type): + host = parsed.hostname + + # this should not raise a ValueError, + # but we try to be very forgiving here and accept just everything. + # decode_parse_result(parsed, "ascii") + else: + host = parsed.hostname.encode("idna") + parsed = encode_parse_result(parsed, "ascii") + + port = parsed.port + if not port: + port = 443 if parsed.scheme == b"https" else 80 + + full_path = urllib.parse.urlunparse( + (b"", b"", parsed.path, parsed.params, parsed.query, parsed.fragment) + ) + if not full_path.startswith(b"/"): + full_path = b"/" + full_path + + if not utils.is_valid_host(host): + raise ValueError("Invalid Host") + if not utils.is_valid_port(port): + raise ValueError("Invalid Port") + + return parsed.scheme, host, port, full_path + + +def unparse(scheme, host, port, path=""): + """ + Returns a URL string, constructed from the specified components. + + Args: + All args must be str. + """ + if path == "*": + path = "" + return "%s://%s%s" % (scheme, utils.hostport(scheme, host, port), path) + + +def encode(s): + """ + Takes a list of (key, value) tuples and returns a urlencoded string. + """ + s = [tuple(i) for i in s] + return urllib.parse.urlencode(s, False) + + +def decode(s): + """ + Takes a urlencoded string and returns a list of (key, value) tuples. + """ + return urllib.parse.parse_qsl(s, keep_blank_values=True) |