diff options
Diffstat (limited to 'netlib/http')
| -rw-r--r-- | netlib/http/authentication.py | 2 | ||||
| -rw-r--r-- | netlib/http/exceptions.py | 3 | ||||
| -rw-r--r-- | netlib/http/http1/protocol.py | 68 | ||||
| -rw-r--r-- | netlib/http/http2/frame.py | 2 | ||||
| -rw-r--r-- | netlib/http/http2/protocol.py | 27 | ||||
| -rw-r--r-- | netlib/http/semantics.py | 65 |
6 files changed, 86 insertions, 81 deletions
diff --git a/netlib/http/authentication.py b/netlib/http/authentication.py index 9a227010..29b9eb3c 100644 --- a/netlib/http/authentication.py +++ b/netlib/http/authentication.py @@ -2,7 +2,6 @@ from __future__ import (absolute_import, print_function, division) from argparse import Action, ArgumentTypeError import binascii -from .. import http def parse_http_basic_auth(s): words = s.split() @@ -37,7 +36,6 @@ class NullProxyAuth(object): """ Clean up authentication headers, so they're not passed upstream. """ - pass def authenticate(self, headers_): """ diff --git a/netlib/http/exceptions.py b/netlib/http/exceptions.py index 7cd26c12..987a7908 100644 --- a/netlib/http/exceptions.py +++ b/netlib/http/exceptions.py @@ -1,6 +1,8 @@ from netlib import odict + class HttpError(Exception): + def __init__(self, code, message): super(HttpError, self).__init__(message) self.code = code @@ -11,6 +13,7 @@ class HttpErrorConnClosed(HttpError): class HttpAuthenticationError(Exception): + def __init__(self, auth_headers=None): super(HttpAuthenticationError, self).__init__( "Proxy Authentication Required" diff --git a/netlib/http/http1/protocol.py b/netlib/http/http1/protocol.py index 2e85a762..8eeb7744 100644 --- a/netlib/http/http1/protocol.py +++ b/netlib/http/http1/protocol.py @@ -1,28 +1,31 @@ from __future__ import (absolute_import, print_function, division) -import binascii -import collections import string import sys -import urlparse import time from netlib import odict, utils, tcp, http from netlib.http import semantics -from .. import status_codes from ..exceptions import * + class TCPHandler(object): + def __init__(self, rfile, wfile=None): self.rfile = rfile self.wfile = wfile + class HTTP1Protocol(semantics.ProtocolMixin): def __init__(self, tcp_handler=None, rfile=None, wfile=None): self.tcp_handler = tcp_handler or TCPHandler(rfile, wfile) - - def read_request(self, include_body=True, body_size_limit=None, allow_empty=False): + def read_request( + self, + include_body=True, + body_size_limit=None, + allow_empty=False, + ): """ Parse an HTTP request from a file stream @@ -129,8 +132,12 @@ class HTTP1Protocol(semantics.ProtocolMixin): timestamp_end, ) - - def read_response(self, request_method, body_size_limit, include_body=True): + def read_response( + self, + request_method, + body_size_limit, + include_body=True, + ): """ Returns an http.Response @@ -175,7 +182,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): # read separately body = None - if hasattr(self.tcp_handler.rfile, "first_byte_timestamp"): # more accurate timestamp_start timestamp_start = self.tcp_handler.rfile.first_byte_timestamp @@ -195,7 +201,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): timestamp_end=timestamp_end, ) - def assemble_request(self, request): assert isinstance(request, semantics.Request) @@ -208,7 +213,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): headers = self._assemble_request_headers(request) return "%s\r\n%s\r\n%s" % (first_line, headers, request.body) - def assemble_response(self, response): assert isinstance(response, semantics.Response) @@ -221,7 +225,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): headers = self._assemble_response_headers(response) return "%s\r\n%s\r\n%s" % (first_line, headers, response.body) - def read_headers(self): """ Read a set of headers. @@ -266,7 +269,7 @@ class HTTP1Protocol(semantics.ProtocolMixin): response_code, is_request, max_chunk_size=None - ): + ): """ Read an HTTP message body: headers: An ODictCaseless object @@ -321,9 +324,14 @@ class HTTP1Protocol(semantics.ProtocolMixin): "HTTP Body too large. Limit is %s," % limit ) - @classmethod - def expected_http_body_size(self, headers, is_request, request_method, response_code): + def expected_http_body_size( + self, + headers, + is_request, + request_method, + response_code, + ): """ Returns the expected body length: - a positive integer, if the size is known in advance @@ -360,20 +368,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): @classmethod - def request_preamble(self, method, resource, http_major="1", http_minor="1"): - return '%s %s HTTP/%s.%s' % ( - method, resource, http_major, http_minor - ) - - - @classmethod - def response_preamble(self, code, message=None, http_major="1", http_minor="1"): - if message is None: - message = status_codes.RESPONSES.get(code) - return 'HTTP/%s.%s %s %s' % (http_major, http_minor, code, message) - - - @classmethod def has_chunked_encoding(self, headers): return "chunked" in [ i.lower() for i in utils.get_header_tokens(headers, "transfer-encoding") @@ -390,7 +384,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): line = self.tcp_handler.rfile.readline() return line - def _read_chunked(self, limit, is_request): """ Read a chunked HTTP body. @@ -427,7 +420,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): if length == 0: return - @classmethod def _parse_http_protocol(self, line): """ @@ -447,7 +439,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): return None return major, minor - @classmethod def _parse_init(self, line): try: @@ -461,7 +452,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): return None return method, url, httpversion - @classmethod def _parse_init_connect(self, line): """ @@ -489,7 +479,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): return None return host, port, httpversion - @classmethod def _parse_init_proxy(self, line): v = self._parse_init(line) @@ -503,7 +492,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): scheme, host, port, path = parts return method, scheme, host, port, path, httpversion - @classmethod def _parse_init_http(self, line): """ @@ -519,7 +507,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): return None return method, url, httpversion - @classmethod def connection_close(self, httpversion, headers): """ @@ -539,7 +526,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): # be persistent return httpversion != (1, 1) - @classmethod def parse_response_line(self, line): parts = line.strip().split(" ", 2) @@ -554,7 +540,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): return None return (proto, code, msg) - @classmethod def _assemble_request_first_line(self, request): return request.legacy_first_line() @@ -575,7 +560,6 @@ class HTTP1Protocol(semantics.ProtocolMixin): return headers.format() - def _assemble_response_first_line(self, response): return 'HTTP/%s.%s %s %s' % ( response.httpversion[0], @@ -584,7 +568,11 @@ class HTTP1Protocol(semantics.ProtocolMixin): response.msg, ) - def _assemble_response_headers(self, response, preserve_transfer_encoding=False): + def _assemble_response_headers( + self, + response, + preserve_transfer_encoding=False, + ): headers = response.headers.copy() for k in response._headers_to_strip_off: del headers[k] diff --git a/netlib/http/http2/frame.py b/netlib/http/http2/frame.py index f7e60471..aa1fbae4 100644 --- a/netlib/http/http2/frame.py +++ b/netlib/http/http2/frame.py @@ -117,7 +117,7 @@ class Frame(object): return "\n".join([ "%s: %s | length: %d | flags: %#x | stream_id: %d" % ( - direction, self.__class__.__name__, self.length, self.flags, self.stream_id), + direction, self.__class__.__name__, self.length, self.flags, self.stream_id), self.payload_human_readable(), "===============================================================", ]) diff --git a/netlib/http/http2/protocol.py b/netlib/http/http2/protocol.py index a1ca4a18..c2ad5edd 100644 --- a/netlib/http/http2/protocol.py +++ b/netlib/http/http2/protocol.py @@ -9,6 +9,7 @@ from . import frame class TCPHandler(object): + def __init__(self, rfile, wfile=None): self.rfile = rfile self.wfile = wfile @@ -39,7 +40,6 @@ class HTTP2Protocol(semantics.ProtocolMixin): ALPN_PROTO_H2 = 'h2' - def __init__( self, tcp_handler=None, @@ -60,7 +60,12 @@ class HTTP2Protocol(semantics.ProtocolMixin): self.current_stream_id = None self.connection_preface_performed = False - def read_request(self, include_body=True, body_size_limit=None, allow_empty=False): + def read_request( + self, + include_body=True, + body_size_limit=None, + allow_empty=False, + ): self.perform_connection_preface() timestamp_start = time.time() @@ -92,7 +97,12 @@ class HTTP2Protocol(semantics.ProtocolMixin): return request - def read_response(self, request_method='', body_size_limit=None, include_body=True): + def read_response( + self, + request_method='', + body_size_limit=None, + include_body=True, + ): self.perform_connection_preface() timestamp_start = time.time() @@ -123,7 +133,6 @@ class HTTP2Protocol(semantics.ProtocolMixin): return response - def assemble_request(self, request): assert isinstance(request, semantics.Request) @@ -133,13 +142,13 @@ class HTTP2Protocol(semantics.ProtocolMixin): headers = request.headers.copy() - if not ':authority' in headers.keys(): + if ':authority' not in headers.keys(): headers.add(':authority', bytes(authority), prepend=True) - if not ':scheme' in headers.keys(): + if ':scheme' not in headers.keys(): headers.add(':scheme', bytes(request.scheme), prepend=True) - if not ':path' in headers.keys(): + if ':path' not in headers.keys(): headers.add(':path', bytes(request.path), prepend=True) - if not ':method' in headers.keys(): + if ':method' not in headers.keys(): headers.add(':method', bytes(request.method), prepend=True) headers = headers.items() @@ -158,7 +167,7 @@ class HTTP2Protocol(semantics.ProtocolMixin): headers = response.headers.copy() - if not ':status' in headers.keys(): + if ':status' not in headers.keys(): headers.add(':status', bytes(str(response.status_code)), prepend=True) headers = headers.items() diff --git a/netlib/http/semantics.py b/netlib/http/semantics.py index e7ae2b5f..76213cd1 100644 --- a/netlib/http/semantics.py +++ b/netlib/http/semantics.py @@ -1,13 +1,9 @@ from __future__ import (absolute_import, print_function, division) -import binascii -import collections -import string -import sys import urllib import urlparse from .. import utils, odict -from . import cookies +from . import cookies, exceptions from netlib import utils, encoding HDR_FORM_URLENCODED = "application/x-www-form-urlencoded" @@ -18,11 +14,11 @@ CONTENT_MISSING = 0 class ProtocolMixin(object): - def read_request(self): - raise NotImplemented + def read_request(self, *args, **kwargs): # pragma: no cover + raise NotImplementedError - def read_response(self): - raise NotImplemented + def read_response(self, *args, **kwargs): # pragma: no cover + raise NotImplementedError def assemble(self, message): if isinstance(message, Request): @@ -32,14 +28,23 @@ class ProtocolMixin(object): else: raise ValueError("HTTP message not supported.") - def assemble_request(self, request): - raise NotImplemented + def assemble_request(self, *args, **kwargs): # pragma: no cover + raise NotImplementedError - def assemble_response(self, response): - raise NotImplemented + def assemble_response(self, *args, **kwargs): # pragma: no cover + raise NotImplementedError class Request(object): + # This list is adopted legacy code. + # We probably don't need to strip off keep-alive. + _headers_to_strip_off = [ + 'Proxy-Connection', + 'Keep-Alive', + 'Connection', + 'Transfer-Encoding', + 'Upgrade', + ] def __init__( self, @@ -71,7 +76,6 @@ class Request(object): self.timestamp_start = timestamp_start self.timestamp_end = timestamp_end - def __eq__(self, other): try: self_d = [self.__dict__[k] for k in self.__dict__ if k not in ('timestamp_start', 'timestamp_end')] @@ -114,7 +118,7 @@ class Request(object): self.httpversion[1], ) else: - raise http.HttpError(400, "Invalid request form") + raise exceptions.HttpError(400, "Invalid request form") def anticache(self): """ @@ -143,7 +147,7 @@ class Request(object): if self.headers["accept-encoding"]: self.headers["accept-encoding"] = [ ', '.join( - e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0])] + e for e in encoding.ENCODINGS if e in self.headers.get_first("accept-encoding"))] def update_host_header(self): """ @@ -317,17 +321,18 @@ class Request(object): self.scheme, self.host, self.port, self.path = parts @property - def content(self): + def content(self): # pragma: no cover # TODO: remove deprecated getter return self.body @content.setter - def content(self, content): + def content(self, content): # pragma: no cover # TODO: remove deprecated setter self.body = content class EmptyRequest(Request): + def __init__(self): super(EmptyRequest, self).__init__( form_in="", @@ -339,10 +344,15 @@ class EmptyRequest(Request): httpversion=(0, 0), headers=odict.ODictCaseless(), body="", - ) + ) class Response(object): + _headers_to_strip_off = [ + 'Proxy-Connection', + 'Alternate-Protocol', + 'Alt-Svc', + ] def __init__( self, @@ -368,7 +378,6 @@ class Response(object): self.timestamp_start = timestamp_start self.timestamp_end = timestamp_end - def __eq__(self, other): try: self_d = [self.__dict__[k] for k in self.__dict__ if k not in ('timestamp_start', 'timestamp_end')] @@ -388,11 +397,9 @@ class Response(object): status_code=self.status_code, msg=self.msg, contenttype=self.headers.get_first( - "content-type", "unknown content type" - ), - size=size - ) - + "content-type", + "unknown content type"), + size=size) def get_cookies(self): """ @@ -430,21 +437,21 @@ class Response(object): self.headers["Set-Cookie"] = values @property - def content(self): + def content(self): # pragma: no cover # TODO: remove deprecated getter return self.body @content.setter - def content(self, content): + def content(self, content): # pragma: no cover # TODO: remove deprecated setter self.body = content @property - def code(self): + def code(self): # pragma: no cover # TODO: remove deprecated getter return self.status_code @code.setter - def code(self, code): + def code(self, code): # pragma: no cover # TODO: remove deprecated setter self.status_code = code |