From 96de7ad562da9b5110059988b851c66b51874510 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 18 Aug 2015 14:15:08 +0200 Subject: various fixes --- libmproxy/protocol2/http.py | 64 ++++++++++++++++++++++++-------- libmproxy/protocol2/http_proxy.py | 5 ++- libmproxy/protocol2/layer.py | 2 + libmproxy/protocol2/reverse_proxy.py | 2 + libmproxy/protocol2/root_context.py | 10 +++-- libmproxy/protocol2/socks_proxy.py | 2 + libmproxy/protocol2/transparent_proxy.py | 2 + 7 files changed, 67 insertions(+), 20 deletions(-) (limited to 'libmproxy/protocol2') diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index eadde3b3..53f40a72 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -10,13 +10,14 @@ from libmproxy.protocol import KILL from libmproxy.protocol.http import HTTPFlow from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from netlib import tcp -from netlib.http import status_codes, http1, HttpErrorConnClosed +from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError from netlib.http.semantics import CONTENT_MISSING from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol + # TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. @@ -31,6 +32,7 @@ class Http1Layer(Layer): layer = HttpLayer(self, self.mode) for message in layer(): yield message + self.server_protocol = HTTP1Protocol(self.server_conn) class Http2Layer(Layer): @@ -41,10 +43,10 @@ class Http2Layer(Layer): self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False) def __call__(self): - # FIXME: Handle Reconnect etc. layer = HttpLayer(self, self.mode) for message in layer(): yield message + self.server_protocol = HTTP1Protocol(self.server_conn) def make_error_response(status_code, message, headers=None): @@ -100,6 +102,7 @@ class ConnectServerConnection(object): """ "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. """ + def __init__(self, address, ctx): self.address = tcp.Address.wrap(address) self._ctx = ctx @@ -124,6 +127,8 @@ class HttpLayer(Layer): def __call__(self): while True: try: + flow = HTTPFlow(self.client_conn, self.server_conn, live=True) + try: request = HTTPRequest.from_protocol( self.client_protocol, @@ -148,7 +153,6 @@ class HttpLayer(Layer): # Make sure that the incoming request matches our expectations self.validate_request(request) - flow = HTTPFlow(self.client_conn, self.server_conn) flow.request = request for message in self.process_request_hook(flow): yield message @@ -164,15 +168,22 @@ class HttpLayer(Layer): if self.check_close_connection(flow): return + # TODO: Implement HTTP Upgrade + # Upstream Proxy Mode: Handle CONNECT if flow.request.form_in == "authority" and flow.response.code == 200: for message in self.handle_upstream_mode_connect(flow.request.copy()): yield message return - except (HttpErrorConnClosed, NetLibError) as e: - make_error_response(502, repr(e)) + except (HttpErrorConnClosed, NetLibError, HttpError) as e: + self.send_to_client(make_error_response( + getattr(e, "code", 502), + repr(e) + )) raise ProtocolException(repr(e), e) + finally: + flow.live = False def handle_regular_mode_connect(self, request): yield SetServer((request.host, request.port), False, None) @@ -267,21 +278,43 @@ class HttpLayer(Layer): for chunk in chunks: for part in chunk: + # TODO: That's going to fail. self.send_to_client(part) self.client_conn.wfile.flush() flow.response.timestamp_end = utils.timestamp() def get_response_from_server(self, flow): - # TODO: Add second attempt. - self.send_to_server(flow.request) - - flow.response = HTTPResponse.from_protocol( - self.server_protocol, - flow.request.method, - body_size_limit=self.config.body_size_limit, - include_body=False, - ) + def get_response(): + self.send_to_server(flow.request) + # Only get the headers at first... + flow.response = HTTPResponse.from_protocol( + self.server_protocol, + flow.request.method, + body_size_limit=self.config.body_size_limit, + include_body=False, + ) + + try: + get_response() + except (tcp.NetLibError, HttpErrorConnClosed) as v: + self.log( + "server communication error: %s" % repr(v), + level="debug" + ) + # In any case, we try to reconnect at least once. This is + # necessary because it might be possible that we already + # initiated an upstream connection after clientconnect that + # has already been expired, e.g consider the following event + # log: + # > clientconnect (transparent mode destination known) + # > serverconnect (required for client tls handshake) + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + yield Reconnect() + get_response() # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True @@ -293,7 +326,6 @@ class HttpLayer(Layer): flow.response.content = CONTENT_MISSING else: flow.response.content = self.server_protocol.read_http_body( - self.server_conn, flow.response.headers, self.config.body_size_limit, flow.request.method, @@ -405,7 +437,7 @@ class HttpLayer(Layer): self.send_to_client(make_error_response( 407, "Proxy Authentication Required", - self.config.authenticator.auth_challenge_headers() + odict.ODictCaseless([[k,v] for k, v in self.config.authenticator.auth_challenge_headers().items()]) )) raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py index 19b5f7ef..652aa473 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/protocol2/http_proxy.py @@ -10,7 +10,8 @@ class HttpProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message - + if self.server_conn: + self._disconnect() class HttpUpstreamProxy(Layer, ServerConnectionMixin): def __init__(self, ctx, server_address): @@ -21,3 +22,5 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + if self.server_conn: + self._disconnect() diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index 67f3d549..eb41bab7 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -109,7 +109,9 @@ class ServerConnectionMixin(object): def _handle_server_message(self, message): if message == Reconnect: + address = self.server_conn.address self._disconnect() + self.server_conn.address = address self._connect() return True elif message == Connect: diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py index 2ee3d9d8..767107ad 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/protocol2/reverse_proxy.py @@ -19,3 +19,5 @@ class ReverseProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py index 6ba6ca9a..f8a645b0 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/protocol2/root_context.py @@ -1,4 +1,5 @@ from __future__ import (absolute_import, print_function, division) +import string from .messages import Kill from .rawtcp import RawTcpLayer @@ -36,6 +37,8 @@ class RootContext(object): d[2] in ('\x00', '\x01', '\x02', '\x03') ) + is_ascii = all(x in string.ascii_uppercase for x in d) + # TODO: build is_http2_magic check here, maybe this is an easy way to detect h2c if not d: @@ -43,10 +46,11 @@ class RootContext(object): if is_tls_client_hello: return TlsLayer(top_layer, True, True) - elif isinstance(top_layer, TlsLayer) and top_layer.client_conn.get_alpn_proto_negotiated() == 'h2': + elif isinstance(top_layer, TlsLayer) and is_ascii: + if top_layer.client_conn.get_alpn_proto_negotiated() == 'h2': return Http2Layer(top_layer, 'transparent') - elif isinstance(top_layer, TlsLayer) and isinstance(top_layer.ctx, Http1Layer): - return Http1Layer(top_layer, "transparent") + else: + return Http1Layer(top_layer, "transparent") else: return RawTcpLayer(top_layer) diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py index c6126a42..5bb8e5f8 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/protocol2/socks_proxy.py @@ -20,3 +20,5 @@ class Socks5Proxy(ServerConnectionMixin, Layer): for message in layer(): if not self._handle_server_message(message): yield message + if self.server_conn: + self._disconnect() \ No newline at end of file diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py index 4ed4c14b..28ad3726 100644 --- a/libmproxy/protocol2/transparent_proxy.py +++ b/libmproxy/protocol2/transparent_proxy.py @@ -21,3 +21,5 @@ class TransparentProxy(Layer, ServerConnectionMixin): for message in layer(): if not self._handle_server_message(message): yield message + if self.server_conn: + self._disconnect() \ No newline at end of file -- cgit v1.2.3