diff options
Diffstat (limited to 'libmproxy/protocol/http.py')
| -rw-r--r-- | libmproxy/protocol/http.py | 191 |
1 files changed, 78 insertions, 113 deletions
diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 230f2be9..a876df41 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -6,14 +6,14 @@ import traceback import six from netlib import tcp -from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers -from netlib.http.semantics import CONTENT_MISSING +from netlib.exceptions import HttpException, HttpReadDisconnect +from netlib.http import http1, Headers +from netlib.http import CONTENT_MISSING from netlib.tcp import NetLibError, Address -from netlib.http.http1 import HTTP1Protocol -from netlib.http.http2 import HTTP2Protocol +from netlib.http.http2.connections import HTTP2Protocol from netlib.http.http2.frame import GoAwayFrame, PriorityFrame, WindowUpdateFrame from .. import utils -from ..exceptions import InvalidCredentials, HttpException, ProtocolException +from ..exceptions import HttpProtocolException, ProtocolException from ..models import ( HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error ) @@ -45,14 +45,14 @@ class _StreamingHttpLayer(_HttpLayer): def read_response_headers(self): raise NotImplementedError - def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + def read_response_body(self, request, response): raise NotImplementedError() yield "this is a generator" # pragma: no cover - def read_response(self, request_method): + def read_response(self, request): response = self.read_response_headers() - response.body = "".join( - self.read_response_body(response.headers, request_method, response.code) + response.body = b"".join( + self.read_response_body(request, response) ) return response @@ -64,7 +64,7 @@ class _StreamingHttpLayer(_HttpLayer): def send_response(self, response): if response.body == CONTENT_MISSING: - raise HttpError(502, "Cannot assemble flow with CONTENT_MISSING") + raise HttpException("Cannot assemble flow with CONTENT_MISSING") self.send_response_headers(response) self.send_response_body(response, [response.body]) @@ -73,48 +73,31 @@ class Http1Layer(_StreamingHttpLayer): def __init__(self, ctx, mode): super(Http1Layer, self).__init__(ctx) self.mode = mode - self.client_protocol = HTTP1Protocol(self.client_conn) - self.server_protocol = HTTP1Protocol(self.server_conn) def read_request(self): - return HTTPRequest.from_protocol( - self.client_protocol, - body_size_limit=self.config.body_size_limit - ) + req = http1.read_request(self.client_conn.rfile, body_size_limit=self.config.body_size_limit) + return HTTPRequest.wrap(req) def send_request(self, request): - self.server_conn.send(self.server_protocol.assemble(request)) + self.server_conn.wfile.write(http1.assemble_request(request)) + self.server_conn.wfile.flush() def read_response_headers(self): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=None, # does not matter if we don't read the body. - body_size_limit=self.config.body_size_limit, - include_body=False - ) + resp = http1.read_response_head(self.server_conn.rfile) + return HTTPResponse.wrap(resp) - def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): - return self.server_protocol.read_http_body_chunked( - headers, - self.config.body_size_limit, - request_method, - response_code, - False, - max_chunk_size - ) + def read_response_body(self, request, response): + expected_size = http1.expected_http_body_size(request, response) + return http1.read_body(self.server_conn.rfile, expected_size, self.config.body_size_limit) def send_response_headers(self, response): - h = self.client_protocol._assemble_response_first_line(response) - self.client_conn.wfile.write(h + "\r\n") - h = self.client_protocol._assemble_response_headers( - response, - preserve_transfer_encoding=True - ) - self.client_conn.wfile.write(h + "\r\n") + raw = http1.assemble_response_head(response, preserve_transfer_encoding=True) + self.client_conn.wfile.write(raw) self.client_conn.wfile.flush() def send_response_body(self, response, chunks): - if self.client_protocol.has_chunked_encoding(response.headers): + if b"chunked" in response.headers.get(b"transfer-encoding", b"").lower(): + # TODO: Move this into netlib.http.http1 chunks = itertools.chain( ( "{:x}\r\n{}\r\n".format(len(chunk), chunk) @@ -127,36 +110,24 @@ class Http1Layer(_StreamingHttpLayer): self.client_conn.wfile.flush() def check_close_connection(self, flow): - close_connection = ( - http1.HTTP1Protocol.connection_close( - flow.request.httpversion, - flow.request.headers - ) or http1.HTTP1Protocol.connection_close( - flow.response.httpversion, - flow.response.headers - ) or http1.HTTP1Protocol.expected_http_body_size( - flow.response.headers, - False, - flow.request.method, - flow.response.code) == -1 + request_close = http1.connection_close( + flow.request.httpversion, + flow.request.headers ) + response_close = http1.connection_close( + flow.response.httpversion, + flow.response.headers + ) + read_until_eof = http1.expected_http_body_size(flow.request, flow.response) == -1 + close_connection = request_close or response_close or read_until_eof if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 + # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313: + # Charles Proxy sends a CONNECT response with HTTP/1.0 # and no Content-Length header return False return close_connection - def connect(self): - self.ctx.connect() - self.server_protocol = HTTP1Protocol(self.server_conn) - - def set_server(self, *args, **kwargs): - self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP1Protocol(self.server_conn) - def __call__(self): layer = HttpLayer(self, self.mode) layer() @@ -184,10 +155,10 @@ class Http2Layer(_HttpLayer): # TODO: implement flow control and WINDOW_UPDATE frames self.server_conn.send(self.server_protocol.assemble(message)) - def read_response(self, request_method): + def read_response(self, request): return HTTPResponse.from_protocol( self.server_protocol, - request_method=request_method, + request_method=request.method, body_size_limit=self.config.body_size_limit, include_body=True, stream_id=self._stream_id @@ -295,7 +266,7 @@ class UpstreamConnectLayer(Layer): def _send_connect_request(self): self.send_request(self.connect_request) - resp = self.read_response("CONNECT") + resp = self.read_response(self.connect_request) if resp.code != 200: raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") @@ -337,28 +308,31 @@ class HttpLayer(Layer): self.__original_server_conn = self.server_conn while True: try: - flow = HTTPFlow(self.client_conn, self.server_conn, live=self) + request = self.read_request() + self.log("request", "debug", [repr(request)]) - try: - request = self.read_request() - except tcp.NetLibError: - # don't throw an error for disconnects that happen - # before/between requests. + # Handle Proxy Authentication + if not self.authenticate(request): return - self.log("request", "debug", [repr(request)]) + # Make sure that the incoming request matches our expectations + self.validate_request(request) - # Handle Proxy Authentication - self.authenticate(request) + except HttpReadDisconnect: + # don't throw an error for disconnects that happen before/between requests. + return + except (HttpException, NetLibError) as e: + self.send_error_response(400, repr(e)) + six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) + + try: + flow = HTTPFlow(self.client_conn, self.server_conn, live=self) # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.form_in == "authority": self.handle_regular_mode_connect(request) return - # Make sure that the incoming request matches our expectations - self.validate_request(request) - flow.request = request self.process_request_hook(flow) @@ -384,30 +358,26 @@ class HttpLayer(Layer): self.handle_upstream_mode_connect(flow.request.copy()) return - except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: - error_propagated = False - if flow.request and not flow.response: + except (HttpException, NetLibError) as e: + self.send_error_response(502, repr(e)) + + if not flow.response: flow.error = Error(str(e)) self.channel.ask("error", flow) self.log(traceback.format_exc(), "debug") - error_propagated = True - - try: - self.send_response(make_error_response( - getattr(e, "code", 502), - repr(e) - )) - except NetLibError: - pass - - if not error_propagated: - if isinstance(e, ProtocolException): - six.reraise(ProtocolException, e, sys.exc_info()[2]) - else: - six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) + return + else: + six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) finally: flow.live = False + def send_error_response(self, code, message): + try: + response = make_error_response(code, message) + self.send_response(response) + except NetLibError: + pass + def change_upstream_proxy_server(self, address): # Make set_upstream_proxy_server always available, # even if there's no UpstreamConnectLayer @@ -435,10 +405,8 @@ class HttpLayer(Layer): # First send the headers and then transfer the response incrementally self.send_response_headers(flow.response) chunks = self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code, - max_chunk_size=4096 + flow.request, + flow.response ) if callable(flow.response.stream): chunks = flow.response.stream(chunks) @@ -451,11 +419,11 @@ class HttpLayer(Layer): if self.supports_streaming: flow.response = self.read_response_headers() else: - flow.response = self.read_response(flow.request.method) + flow.response = self.read_response(flow.request) try: get_response() - except (tcp.NetLibError, HttpErrorConnClosed) as v: + except (tcp.NetLibError, HttpException) as v: self.log( "server communication error: %s" % repr(v), level="debug" @@ -485,10 +453,9 @@ class HttpLayer(Layer): if flow.response.stream: flow.response.content = CONTENT_MISSING else: - flow.response.content = "".join(self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code + flow.response.content = b"".join(self.read_response_body( + flow.request, + flow.response )) flow.response.timestamp_end = utils.timestamp() @@ -543,7 +510,7 @@ class HttpLayer(Layer): if not self.server_conn: self.connect() if tls: - raise HttpException("Cannot change scheme in upstream proxy mode.") + raise HttpProtocolException("Cannot change scheme in upstream proxy mode.") """ # This is a very ugly (untested) workaround to solve a very ugly problem. if self.server_conn and self.server_conn.tls_established and not ssl: @@ -561,12 +528,10 @@ class HttpLayer(Layer): def validate_request(self, request): if request.form_in == "absolute" and request.scheme != "http": - self.send_response( - make_error_response(400, "Invalid request scheme: %s" % request.scheme)) raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { - "regular": ("absolute",), # an authority request would already be handled. + "regular": ("authority", "absolute",), "upstream": ("authority", "absolute"), "transparent": ("relative",) } @@ -576,10 +541,9 @@ class HttpLayer(Layer): err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( " or ".join(allowed_request_forms), request.form_in ) - self.send_response(make_error_response(400, err_message)) raise HttpException(err_message) - if self.mode == "regular": + if self.mode == "regular" and request.form_in == "absolute": request.form_out = "relative" def authenticate(self, request): @@ -592,4 +556,5 @@ class HttpLayer(Layer): "Proxy Authentication Required", Headers(**self.config.authenticator.auth_challenge_headers()) )) - raise InvalidCredentials("Proxy Authentication Required") + return False + return True |