From 96de7ad562da9b5110059988b851c66b51874510 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 18 Aug 2015 14:15:08 +0200 Subject: various fixes --- libmproxy/protocol/http.py | 1 + 1 file changed, 1 insertion(+) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4c15c80d..4472cb2a 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -729,6 +729,7 @@ class RequestReplayThread(threading.Thread): if not self.flow.response: # In all modes, we directly connect to the server displayed if self.config.mode == "upstream": + # FIXME server_address = self.config.mode.get_upstream_server( self.flow.client_conn )[2:] -- cgit v1.2.3 From 9bae97eb17ed66a33b5b988c6857ca6c9fae8e22 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 18 Aug 2015 13:43:26 +0200 Subject: http2: fix connection preface and wrappers --- libmproxy/protocol/http_wrappers.py | 36 ++++++++++-------------------------- 1 file changed, 10 insertions(+), 26 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index ed5759ea..e41d65d6 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -247,24 +247,11 @@ class HTTPRequest(MessageMixin, semantics.Request): include_body = include_body, body_size_limit = body_size_limit, ) - - return HTTPRequest( - req.form_in, - req.method, - req.scheme, - req.host, - req.port, - req.path, - req.httpversion, - req.headers, - req.body, - req.timestamp_start, - req.timestamp_end, - ) + return self.wrap(req) @classmethod def wrap(self, request): - return HTTPRequest( + req = HTTPRequest( form_in=request.form_in, method=request.method, scheme=request.scheme, @@ -278,6 +265,9 @@ class HTTPRequest(MessageMixin, semantics.Request): timestamp_end=request.timestamp_end, form_out=(request.form_out if hasattr(request, 'form_out') else None), ) + if hasattr(request, 'stream_id'): + req.stream_id = request.stream_id + return req def __hash__(self): return id(self) @@ -371,20 +361,11 @@ class HTTPResponse(MessageMixin, semantics.Response): body_size_limit, include_body=include_body ) - - return HTTPResponse( - resp.httpversion, - resp.status_code, - resp.msg, - resp.headers, - resp.body, - resp.timestamp_start, - resp.timestamp_end, - ) + return self.wrap(resp) @classmethod def wrap(self, response): - return HTTPResponse( + resp = HTTPResponse( httpversion=response.httpversion, status_code=response.status_code, msg=response.msg, @@ -393,6 +374,9 @@ class HTTPResponse(MessageMixin, semantics.Response): timestamp_start=response.timestamp_start, timestamp_end=response.timestamp_end, ) + if hasattr(response, 'stream_id'): + resp.stream_id = response.stream_id + return resp def _refresh_cookie(self, c, delta): """ -- cgit v1.2.3 From 05d26545e4c65e8fd2242142833a40a96ce5fb81 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 21 Aug 2015 10:26:28 +0200 Subject: adapt netlib changes --- libmproxy/protocol/http.py | 6 +++--- libmproxy/protocol/http_wrappers.py | 4 ++-- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4472cb2a..1b168569 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -41,7 +41,7 @@ def send_connect_request(conn, host, port, update_state=True): protocol = http1.HTTP1Protocol(conn) conn.send(protocol.assemble(upstream_request)) - resp = HTTPResponse.from_protocol(protocol, upstream_request.method) + resp = HTTPResponse.from_protocol(protocol, upstream_request) if resp.status_code != 200: raise proxy.ProxyError(resp.status_code, "Cannot establish SSL " + @@ -177,7 +177,7 @@ class HTTPHandler(ProtocolHandler): # Only get the headers at first... flow.response = HTTPResponse.from_protocol( self.c.server_conn.protocol, - flow.request.method, + flow.request, body_size_limit=self.c.config.body_size_limit, include_body=False, ) @@ -760,7 +760,7 @@ class RequestReplayThread(threading.Thread): self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) self.flow.response = HTTPResponse.from_protocol( self.flow.server_conn.protocol, - r.method, + r, body_size_limit=self.config.body_size_limit, ) if self.channel: diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index e41d65d6..f91b936c 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -352,12 +352,12 @@ class HTTPResponse(MessageMixin, semantics.Response): def from_protocol( self, protocol, - request_method, + request, include_body=True, body_size_limit=None ): resp = protocol.read_response( - request_method, + request, body_size_limit, include_body=include_body ) -- cgit v1.2.3 From 56a4bc381efc177c08ed7b9e7c845f74120050e4 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 24 Aug 2015 18:17:04 +0200 Subject: request -> request_method --- libmproxy/protocol/http.py | 6 +++--- libmproxy/protocol/http_wrappers.py | 20 ++++++-------------- 2 files changed, 9 insertions(+), 17 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 1b168569..4472cb2a 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -41,7 +41,7 @@ def send_connect_request(conn, host, port, update_state=True): protocol = http1.HTTP1Protocol(conn) conn.send(protocol.assemble(upstream_request)) - resp = HTTPResponse.from_protocol(protocol, upstream_request) + resp = HTTPResponse.from_protocol(protocol, upstream_request.method) if resp.status_code != 200: raise proxy.ProxyError(resp.status_code, "Cannot establish SSL " + @@ -177,7 +177,7 @@ class HTTPHandler(ProtocolHandler): # Only get the headers at first... flow.response = HTTPResponse.from_protocol( self.c.server_conn.protocol, - flow.request, + flow.request.method, body_size_limit=self.c.config.body_size_limit, include_body=False, ) @@ -760,7 +760,7 @@ class RequestReplayThread(threading.Thread): self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) self.flow.response = HTTPResponse.from_protocol( self.flow.server_conn.protocol, - r, + r.method, body_size_limit=self.config.body_size_limit, ) if self.channel: diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index f91b936c..b1000a79 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -240,13 +240,10 @@ class HTTPRequest(MessageMixin, semantics.Request): def from_protocol( self, protocol, - include_body=True, - body_size_limit=None, + *args, + **kwargs ): - req = protocol.read_request( - include_body = include_body, - body_size_limit = body_size_limit, - ) + req = protocol.read_request(*args, **kwargs) return self.wrap(req) @classmethod @@ -352,15 +349,10 @@ class HTTPResponse(MessageMixin, semantics.Response): def from_protocol( self, protocol, - request, - include_body=True, - body_size_limit=None + *args, + **kwargs ): - resp = protocol.read_response( - request, - body_size_limit, - include_body=include_body - ) + resp = protocol.read_response(*args, **kwargs) return self.wrap(resp) @classmethod -- cgit v1.2.3 From 1cc48345e13917aadc1e0fd93d6011139e78e3d9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 28 Aug 2015 01:51:13 +0200 Subject: clean up config/cmdline, fix bugs, remove cruft --- libmproxy/protocol/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4472cb2a..56d7d57f 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -10,6 +10,7 @@ from email.utils import parsedate_tz, formatdate, mktime_tz import netlib from netlib import http, tcp, odict, utils, encoding from netlib.http import cookies, http1, http2 +from netlib.http.http1 import HTTP1Protocol from netlib.http.semantics import CONTENT_MISSING from .tcp import TCPHandler @@ -757,7 +758,6 @@ class RequestReplayThread(threading.Thread): server.send(self.flow.server_conn.protocol.assemble(r)) self.flow.server_conn = server - self.flow.server_conn.protocol = http1.HTTP1Protocol(self.flow.server_conn) self.flow.response = HTTPResponse.from_protocol( self.flow.server_conn.protocol, r.method, -- cgit v1.2.3 From dd7f50d64bef38fa67b4cace91913d03691dde26 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 01:21:58 +0200 Subject: restructure code, remove cruft --- libmproxy/protocol/http.py | 83 +--------------------------------------------- 1 file changed, 1 insertion(+), 82 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 56d7d57f..a30437d1 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -695,85 +695,4 @@ class HTTPHandler(ProtocolHandler): else: raise http.HttpAuthenticationError( self.c.config.authenticator.auth_challenge_headers()) - return request.headers - - -class RequestReplayThread(threading.Thread): - name = "RequestReplayThread" - - def __init__(self, config, flow, masterq, should_exit): - """ - masterqueue can be a queue or None, if no scripthooks should be - processed. - """ - self.config, self.flow = config, flow - if masterq: - self.channel = controller.Channel(masterq, should_exit) - else: - self.channel = None - super(RequestReplayThread, self).__init__() - - def run(self): - r = self.flow.request - form_out_backup = r.form_out - try: - self.flow.response = None - - # If we have a channel, run script hooks. - if self.channel: - request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == KILL: - raise KillSignal() - elif isinstance(request_reply, HTTPResponse): - self.flow.response = request_reply - - if not self.flow.response: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - # FIXME - server_address = self.config.mode.get_upstream_server( - self.flow.client_conn - )[2:] - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - send_connect_request(server, r.host, r.port) - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - else: - r.form_out = "absolute" - else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - if r.scheme == "https": - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - - server.send(self.flow.server_conn.protocol.assemble(r)) - self.flow.server_conn = server - self.flow.response = HTTPResponse.from_protocol( - self.flow.server_conn.protocol, - r.method, - body_size_limit=self.config.body_size_limit, - ) - if self.channel: - response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == KILL: - raise KillSignal() - except (proxy.ProxyError, http.HttpError, tcp.NetLibError) as v: - self.flow.error = Error(repr(v)) - if self.channel: - self.channel.ask("error", self.flow) - except KillSignal: - # KillSignal should only be raised if there's a channel in the - # first place. - self.channel.tell("log", proxy.Log("Connection killed", "info")) - finally: - r.form_out = form_out_backup + return request.headers \ No newline at end of file -- cgit v1.2.3 From 3873e08339fd701738a1522af32e37363fcec14b Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 03:42:11 +0200 Subject: remove old code --- libmproxy/protocol/handle.py | 20 -- libmproxy/protocol/http.py | 608 +----------------------------------- libmproxy/protocol/http_wrappers.py | 40 +-- libmproxy/protocol/primitives.py | 130 +------- libmproxy/protocol/tcp.py | 97 ------ 5 files changed, 18 insertions(+), 877 deletions(-) delete mode 100644 libmproxy/protocol/handle.py delete mode 100644 libmproxy/protocol/tcp.py (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/handle.py b/libmproxy/protocol/handle.py deleted file mode 100644 index 49cb3c1b..00000000 --- a/libmproxy/protocol/handle.py +++ /dev/null @@ -1,20 +0,0 @@ -from __future__ import absolute_import -from . import http, tcp - -protocols = { - 'http': dict(handler=http.HTTPHandler, flow=http.HTTPFlow), - 'tcp': dict(handler=tcp.TCPHandler) -} - - -def protocol_handler(protocol): - """ - @type protocol: str - @returns: libmproxy.protocol.primitives.ProtocolHandler - """ - if protocol in protocols: - return protocols[protocol]["handler"] - - raise NotImplementedError( - "Unknown Protocol: %s" % - protocol) # pragma: nocover diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index a30437d1..bde7b088 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,62 +1,9 @@ from __future__ import absolute_import -import Cookie -import copy -import threading -import time -import urllib -import urlparse -from email.utils import parsedate_tz, formatdate, mktime_tz -import netlib -from netlib import http, tcp, odict, utils, encoding -from netlib.http import cookies, http1, http2 -from netlib.http.http1 import HTTP1Protocol -from netlib.http.semantics import CONTENT_MISSING - -from .tcp import TCPHandler -from .primitives import KILL, ProtocolHandler, Flow, Error -from ..proxy.connection import ServerConnection -from .. import utils, controller, stateobject, proxy +from .primitives import Flow from .http_wrappers import decoded, HTTPRequest, HTTPResponse - -class KillSignal(Exception): - pass - - -def send_connect_request(conn, host, port, update_state=True): - upstream_request = HTTPRequest( - "authority", - "CONNECT", - None, - host, - port, - None, - (1, 1), - odict.ODictCaseless(), - "" - ) - - # we currently only support HTTP/1 CONNECT requests - protocol = http1.HTTP1Protocol(conn) - - conn.send(protocol.assemble(upstream_request)) - resp = HTTPResponse.from_protocol(protocol, upstream_request.method) - if resp.status_code != 200: - raise proxy.ProxyError(resp.status_code, - "Cannot establish SSL " + - "connection with upstream proxy: \r\n" + - repr(resp)) - if update_state: - conn.state.append(("http", { - "state": "connect", - "host": host, - "port": port} - )) - return resp - - class HTTPFlow(Flow): """ A HTTPFlow is a collection of objects representing a single HTTP @@ -143,556 +90,3 @@ class HTTPFlow(Flow): if self.response: c += self.response.replace(pattern, repl, *args, **kwargs) return c - - -class HTTPHandler(ProtocolHandler): - """ - HTTPHandler implements mitmproxys understanding of the HTTP protocol. - - """ - - def __init__(self, c): - super(HTTPHandler, self).__init__(c) - self.expected_form_in = c.config.mode.http_form_in - self.expected_form_out = c.config.mode.http_form_out - self.skip_authentication = False - - def handle_messages(self): - while self.handle_flow(): - pass - - def get_response_from_server(self, flow): - self.c.establish_server_connection() - - for attempt in (0, 1): - try: - if not self.c.server_conn.protocol: - # instantiate new protocol if connection does not have one yet - # TODO: select correct protocol based on ALPN (?) - self.c.server_conn.protocol = http1.HTTP1Protocol(self.c.server_conn) - # self.c.server_conn.protocol = http2.HTTP2Protocol(self.c.server_conn) - # self.c.server_conn.protocol.perform_connection_preface() - - self.c.server_conn.send(self.c.server_conn.protocol.assemble(flow.request)) - - # Only get the headers at first... - flow.response = HTTPResponse.from_protocol( - self.c.server_conn.protocol, - flow.request.method, - body_size_limit=self.c.config.body_size_limit, - include_body=False, - ) - break - except (tcp.NetLibError, http.HttpErrorConnClosed) as v: - self.c.log( - "error in server communication: %s" % repr(v), - level="debug" - ) - if attempt == 0: - # In any case, we try to reconnect at least once. This is - # necessary because it might be possible that we already - # initiated an upstream connection after clientconnect that - # has already been expired, e.g consider the following event - # log: - # > clientconnect (transparent mode destination known) - # > serverconnect - # > read n% of large request - # > server detects timeout, disconnects - # > read (100-n)% of large request - # > send large request upstream - self.c.server_reconnect() - else: - raise - - # call the appropriate script hook - this is an opportunity for an - # inline script to set flow.stream = True - flow = self.c.channel.ask("responseheaders", flow) - if flow is None or flow == KILL: - raise KillSignal() - else: - # now get the rest of the request body, if body still needs to be - # read but not streaming this response - if flow.response.stream: - flow.response.content = CONTENT_MISSING - else: - if isinstance(self.c.server_conn.protocol, http1.HTTP1Protocol): - # streaming is only supported with HTTP/1 at the moment - flow.response.content = self.c.server_conn.protocol.read_http_body( - flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, - flow.response.code, - False - ) - flow.response.timestamp_end = utils.timestamp() - - def handle_flow(self): - flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live) - - try: - try: - if not flow.client_conn.protocol: - # instantiate new protocol if connection does not have one yet - # the first request might be a CONNECT - which is currently only supported with HTTP/1 - flow.client_conn.protocol = http1.HTTP1Protocol(self.c.client_conn) - - req = HTTPRequest.from_protocol( - flow.client_conn.protocol, - body_size_limit=self.c.config.body_size_limit - ) - except tcp.NetLibError: - # don't throw an error for disconnects that happen - # before/between requests. - return False - - self.c.log( - "request", - "debug", - [repr(req)] - ) - ret = self.process_request(flow, req) - if ret: - # instantiate new protocol if connection does not have one yet - # TODO: select correct protocol based on ALPN (?) - flow.client_conn.protocol = http1.HTTP1Protocol(self.c.client_conn) - # flow.client_conn.protocol = http2.HTTP2Protocol(self.c.client_conn, is_server=True) - if ret is not None: - return ret - - # Be careful NOT to assign the request to the flow before - # process_request completes. This is because the call can raise an - # exception. If the request object is already attached, this results - # in an Error object that has an attached request that has not been - # sent through to the Master. - flow.request = req - request_reply = self.c.channel.ask("request", flow) - if request_reply is None or request_reply == KILL: - raise KillSignal() - - # The inline script may have changed request.host - self.process_server_address(flow) - - if isinstance(request_reply, HTTPResponse): - flow.response = request_reply - else: - self.get_response_from_server(flow) - - # no further manipulation of self.c.server_conn beyond this point - # we can safely set it as the final attribute value here. - flow.server_conn = self.c.server_conn - - self.c.log( - "response", - "debug", - [repr(flow.response)] - ) - response_reply = self.c.channel.ask("response", flow) - if response_reply is None or response_reply == KILL: - raise KillSignal() - - self.send_response_to_client(flow) - - if self.check_close_connection(flow): - return False - - # We sent a CONNECT request to an upstream proxy. - if flow.request.form_in == "authority" and flow.response.code == 200: - # TODO: Possibly add headers (memory consumption/usefulness - # tradeoff) Make sure to add state info before the actual - # processing of the CONNECT request happens. During an SSL - # upgrade, we may receive an SNI indication from the client, - # which resets the upstream connection. If this is the case, we - # must already re-issue the CONNECT request at this point. - self.c.server_conn.state.append( - ( - "http", { - "state": "connect", - "host": flow.request.host, - "port": flow.request.port - } - ) - ) - if not self.process_connect_request( - (flow.request.host, flow.request.port)): - return False - - # If the user has changed the target server on this connection, - # restore the original target server - flow.live.restore_server() - - return True # Next flow please. - except ( - http.HttpAuthenticationError, - http.HttpError, - proxy.ProxyError, - tcp.NetLibError, - ) as e: - self.handle_error(e, flow) - except KillSignal: - self.c.log("Connection killed", "info") - finally: - flow.live = None # Connection is not live anymore. - return False - - def handle_server_reconnect(self, state): - if state["state"] == "connect": - send_connect_request( - self.c.server_conn, - state["host"], - state["port"], - update_state=False - ) - else: # pragma: nocover - raise RuntimeError("Unknown State: %s" % state["state"]) - - def handle_error(self, error, flow=None): - message = repr(error) - message_debug = None - - if isinstance(error, tcp.NetLibError): - message = None - message_debug = "TCP connection closed unexpectedly." - elif "tlsv1 alert unknown ca" in message: - message = "TLSv1 Alert Unknown CA: The client does not trust the proxy's certificate." - elif "handshake error" in message: - message_debug = message - message = "SSL handshake error: The client may not trust the proxy's certificate." - - if message: - self.c.log(message, level="info") - if message_debug: - self.c.log(message_debug, level="debug") - - if flow: - # TODO: no flows without request or with both request and response - # at the moment. - if flow.request and not flow.response: - flow.error = Error(message or message_debug) - self.c.channel.ask("error", flow) - try: - status_code = getattr(error, "code", 502) - headers = getattr(error, "headers", None) - - html_message = message or "" - if message_debug: - html_message += "
%s
" % message_debug - self.send_error(status_code, html_message, headers) - except: - pass - - def send_error(self, status_code, message, headers): - response = http.status_codes.RESPONSES.get(status_code, "Unknown") - body = """ - - - %d %s - - %s - - """ % (status_code, response, message) - - if not headers: - headers = odict.ODictCaseless() - assert isinstance(headers, odict.ODictCaseless) - - headers["Server"] = [self.c.config.server_version] - headers["Connection"] = ["close"] - headers["Content-Length"] = [len(body)] - headers["Content-Type"] = ["text/html"] - - resp = HTTPResponse( - (1, 1), # if HTTP/2 is used, this value is ignored anyway - status_code, - response, - headers, - body, - ) - - # if no protocol is assigned yet - just assume HTTP/1 - # TODO: maybe check ALPN and use HTTP/2 if required? - protocol = self.c.client_conn.protocol or http1.HTTP1Protocol(self.c.client_conn) - self.c.client_conn.send(protocol.assemble(resp)) - - def process_request(self, flow, request): - """ - @returns: - True, if the request should not be sent upstream - False, if the connection should be aborted - None, if the request should be sent upstream - (a status code != None should be returned directly by handle_flow) - """ - - if not self.skip_authentication: - self.authenticate(request) - - # Determine .scheme, .host and .port attributes - # For absolute-form requests, they are directly given in the request. - # For authority-form requests, we only need to determine the request scheme. - # For relative-form requests, we need to determine host and port as - # well. - if not request.scheme: - request.scheme = "https" if flow.server_conn and flow.server_conn.ssl_established else "http" - if not request.host: - # Host/Port Complication: In upstream mode, use the server we CONNECTed to, - # not the upstream proxy. - if flow.server_conn: - for s in flow.server_conn.state: - if s[0] == "http" and s[1]["state"] == "connect": - request.host, request.port = s[1]["host"], s[1]["port"] - if not request.host and flow.server_conn: - request.host, request.port = flow.server_conn.address.host, flow.server_conn.address.port - - - # Now we can process the request. - if request.form_in == "authority": - if self.c.client_conn.ssl_established: - raise http.HttpError( - 400, - "Must not CONNECT on already encrypted connection" - ) - - if self.c.config.mode == "regular": - self.c.set_server_address((request.host, request.port)) - # Update server_conn attribute on the flow - flow.server_conn = self.c.server_conn - - # since we currently only support HTTP/1 CONNECT requests - # the response must be HTTP/1 as well - self.c.client_conn.send( - ('HTTP/%s.%s 200 ' % (request.httpversion[0], request.httpversion[1])) + - 'Connection established\r\n' + - 'Content-Length: 0\r\n' + - ('Proxy-agent: %s\r\n' % self.c.config.server_version) + - '\r\n' - ) - return self.process_connect_request(self.c.server_conn.address) - elif self.c.config.mode == "upstream": - return None - else: - # CONNECT should never occur if we don't expect absolute-form - # requests - pass - - elif request.form_in == self.expected_form_in: - request.form_out = self.expected_form_out - if request.form_in == "absolute": - if request.scheme != "http": - raise http.HttpError( - 400, - "Invalid request scheme: %s" % request.scheme - ) - if self.c.config.mode == "regular": - # Update info so that an inline script sees the correct - # value at flow.server_conn - self.c.set_server_address((request.host, request.port)) - flow.server_conn = self.c.server_conn - - elif request.form_in == "relative": - if self.c.config.mode == "spoof": - # Host header - h = request.pretty_host(hostheader=True) - if h is None: - raise http.HttpError( - 400, - "Invalid request: No host information" - ) - p = netlib.utils.parse_url("http://" + h) - request.scheme = p[0] - request.host = p[1] - request.port = p[2] - self.c.set_server_address((request.host, request.port)) - flow.server_conn = self.c.server_conn - - if self.c.config.mode == "sslspoof": - # SNI is processed in server.py - if not (flow.server_conn and flow.server_conn.ssl_established): - raise http.HttpError( - 400, - "Invalid request: No host information" - ) - - return None - - raise http.HttpError( - 400, "Invalid HTTP request form (expected: %s, got: %s)" % ( - self.expected_form_in, request.form_in - ) - ) - - def process_server_address(self, flow): - # Depending on the proxy mode, server handling is entirely different - # We provide a mostly unified API to the user, which needs to be - # unfiddled here - # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 ) - address = tcp.Address((flow.request.host, flow.request.port)) - - ssl = (flow.request.scheme == "https") - - if self.c.config.mode == "upstream": - # The connection to the upstream proxy may have a state we may need - # to take into account. - connected_to = None - for s in flow.server_conn.state: - if s[0] == "http" and s[1]["state"] == "connect": - connected_to = tcp.Address((s[1]["host"], s[1]["port"])) - - # We need to reconnect if the current flow either requires a - # (possibly impossible) change to the connection state, e.g. the - # host has changed but we already CONNECTed somewhere else. - needs_server_change = ( - ssl != self.c.server_conn.ssl_established - or - # HTTP proxying is "stateless", CONNECT isn't. - (connected_to and address != connected_to) - ) - - if needs_server_change: - # force create new connection to the proxy server to reset - # state - self.live.change_server(self.c.server_conn.address, force=True) - if ssl: - send_connect_request( - self.c.server_conn, - address.host, - address.port - ) - self.c.establish_ssl(server=True) - else: - # If we're not in upstream mode, we just want to update the host - # and possibly establish TLS. This is a no op if the addresses - # match. - self.live.change_server(address, ssl=ssl) - - flow.server_conn = self.c.server_conn - - def send_response_to_client(self, flow): - if not flow.response.stream: - # no streaming: - # we already received the full response from the server and can - # send it to the client straight away. - self.c.client_conn.send(self.c.client_conn.protocol.assemble(flow.response)) - else: - if isinstance(self.c.client_conn.protocol, http2.HTTP2Protocol): - raise NotImplementedError("HTTP streaming with HTTP/2 is currently not supported.") - - - # streaming: - # First send the headers and then transfer the response - # incrementally: - h = self.c.client_conn.protocol._assemble_response_first_line(flow.response) - self.c.client_conn.send(h + "\r\n") - h = self.c.client_conn.protocol._assemble_response_headers(flow.response, preserve_transfer_encoding=True) - self.c.client_conn.send(h + "\r\n") - - chunks = self.c.server_conn.protocol.read_http_body_chunked( - flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, - flow.response.code, - False, - 4096 - ) - - if callable(flow.response.stream): - chunks = flow.response.stream(chunks) - - for chunk in chunks: - for part in chunk: - self.c.client_conn.wfile.write(part) - self.c.client_conn.wfile.flush() - - flow.response.timestamp_end = utils.timestamp() - - def check_close_connection(self, flow): - """ - Checks if the connection should be closed depending on the HTTP - semantics. Returns True, if so. - """ - - # TODO: add logic for HTTP/2 - - close_connection = ( - http1.HTTP1Protocol.connection_close( - flow.request.httpversion, - flow.request.headers - ) or http1.HTTP1Protocol.connection_close( - flow.response.httpversion, - flow.response.headers - ) or http1.HTTP1Protocol.expected_http_body_size( - flow.response.headers, - False, - flow.request.method, - flow.response.code) == -1 - ) - if close_connection: - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 - # and no Content-Length header - pass - else: - return True - return False - - def process_connect_request(self, address): - """ - Process a CONNECT request. - Returns True if the CONNECT request has been processed successfully. - Returns False, if the connection should be closed immediately. - """ - address = tcp.Address.wrap(address) - if self.c.config.check_ignore(address): - self.c.log("Ignore host: %s:%s" % address(), "info") - TCPHandler(self.c, log=False).handle_messages() - return False - else: - self.expected_form_in = "relative" - self.expected_form_out = "relative" - self.skip_authentication = True - - # In practice, nobody issues a CONNECT request to send unencrypted - # HTTP requests afterwards. If we don't delegate to TCP mode, we - # should always negotiate a SSL connection. - # - # FIXME: Turns out the previous statement isn't entirely true. - # Chrome on Windows CONNECTs to :80 if an explicit proxy is - # configured and a websocket connection should be established. We - # don't support websocket at the moment, so it fails anyway, but we - # should come up with a better solution to this if we start to - # support WebSockets. - should_establish_ssl = ( - address.port in self.c.config.ssl_ports - or - not self.c.config.check_tcp(address) - ) - - if should_establish_ssl: - self.c.log( - "Received CONNECT request to SSL port. " - "Upgrading to SSL...", "debug" - ) - server_ssl = not self.c.config.no_upstream_cert - if server_ssl: - self.c.establish_server_connection() - self.c.establish_ssl(server=server_ssl, client=True) - self.c.log("Upgrade to SSL completed.", "debug") - - if self.c.config.check_tcp(address): - self.c.log( - "Generic TCP mode for host: %s:%s" % address(), - "info" - ) - TCPHandler(self.c).handle_messages() - return False - - return True - - def authenticate(self, request): - if self.c.config.authenticator: - if self.c.config.authenticator.authenticate(request.headers): - self.c.config.authenticator.clean(request.headers) - else: - raise http.HttpAuthenticationError( - self.c.config.authenticator.auth_challenge_headers()) - return request.headers \ No newline at end of file diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py index b1000a79..a26ddbb4 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/protocol/http_wrappers.py @@ -1,20 +1,12 @@ from __future__ import absolute_import import Cookie import copy -import threading import time -import urllib -import urlparse from email.utils import parsedate_tz, formatdate, mktime_tz -import netlib -from netlib import http, tcp, odict, utils, encoding -from netlib.http import cookies, semantics, http1 - -from .tcp import TCPHandler -from .primitives import KILL, ProtocolHandler, Flow, Error -from ..proxy.connection import ServerConnection -from .. import utils, controller, stateobject, proxy +from netlib import odict, encoding +from netlib.http import semantics, CONTENT_MISSING +from .. import utils, stateobject class decoded(object): @@ -170,19 +162,19 @@ class HTTPRequest(MessageMixin, semantics.Request): """ def __init__( - self, - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - body, - timestamp_start=None, - timestamp_end=None, - form_out=None, + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + body, + timestamp_start=None, + timestamp_end=None, + form_out=None, ): semantics.Request.__init__( self, diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py index 92fc95e5..c663f0c5 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/protocol/primitives.py @@ -1,11 +1,10 @@ from __future__ import absolute_import import copy import uuid -import netlib.tcp + from .. import stateobject, utils, version from ..proxy.connection import ClientConnection, ServerConnection - KILL = 0 # const for killed requests @@ -165,130 +164,3 @@ class Flow(stateobject.StateObject): self.intercepted = False self.reply() master.handle_accept_intercept(self) - - - -class ProtocolHandler(object): - """ - A ProtocolHandler implements an application-layer protocol, e.g. HTTP. - See: libmproxy.protocol.http.HTTPHandler - """ - - def __init__(self, c): - self.c = c - """@type: libmproxy.proxy.server.ConnectionHandler""" - self.live = LiveConnection(c) - """@type: LiveConnection""" - - def handle_messages(self): - """ - This method gets called if a client connection has been made. Depending - on the proxy settings, a server connection might already exist as well. - """ - raise NotImplementedError # pragma: nocover - - def handle_server_reconnect(self, state): - """ - This method gets called if a server connection needs to reconnect and - there's a state associated with the server connection (e.g. a - previously-sent CONNECT request or a SOCKS proxy request). This method - gets called after the connection has been restablished but before SSL is - established. - """ - raise NotImplementedError # pragma: nocover - - def handle_error(self, error): - """ - This method gets called should there be an uncaught exception during the - connection. This might happen outside of handle_messages, e.g. if the - initial SSL handshake fails in transparent mode. - """ - raise error # pragma: nocover - - -class LiveConnection(object): - """ - This facade allows interested parties (FlowMaster, inline scripts) to - interface with a live connection, without exposing the internals - of the ConnectionHandler. - """ - - def __init__(self, c): - self.c = c - """@type: libmproxy.proxy.server.ConnectionHandler""" - self._backup_server_conn = None - """@type: libmproxy.proxy.connection.ServerConnection""" - - def change_server( - self, - address, - ssl=None, - sni=None, - force=False, - persistent_change=False): - """ - Change the server connection to the specified address. - @returns: - True, if a new connection has been established, - False, if an existing connection has been used - """ - address = netlib.tcp.Address.wrap(address) - - ssl_mismatch = ( - ssl is not None and - ( - (self.c.server_conn.connection and ssl != self.c.server_conn.ssl_established) - or - (sni is not None and sni != self.c.server_conn.sni) - ) - ) - address_mismatch = (address != self.c.server_conn.address) - - if persistent_change: - self._backup_server_conn = None - - if ssl_mismatch or address_mismatch or force: - - self.c.log( - "Change server connection: %s:%s -> %s:%s [persistent: %s]" % ( - self.c.server_conn.address.host, - self.c.server_conn.address.port, - address.host, - address.port, - persistent_change - ), - "debug" - ) - - if not self._backup_server_conn and not persistent_change: - self._backup_server_conn = self.c.server_conn - self.c.server_conn = None - else: - # This is at least the second temporary change. We can kill the - # current connection. - self.c.del_server_connection() - - self.c.set_server_address(address) - self.c.establish_server_connection(ask=False) - if ssl: - self.c.establish_ssl(server=True, sni=sni) - return True - return False - - def restore_server(self): - # TODO: Similar to _backup_server_conn, introduce _cache_server_conn, - # which keeps the changed connection open This may be beneficial if a - # user is rewriting all requests from http to https or similar. - if not self._backup_server_conn: - return - - self.c.log("Restore original server connection: %s:%s -> %s:%s" % ( - self.c.server_conn.address.host, - self.c.server_conn.address.port, - self._backup_server_conn.address.host, - self._backup_server_conn.address.port - ), "debug") - - self.c.del_server_connection() - self.c.server_conn = self._backup_server_conn - self._backup_server_conn = None diff --git a/libmproxy/protocol/tcp.py b/libmproxy/protocol/tcp.py deleted file mode 100644 index 0feb77c6..00000000 --- a/libmproxy/protocol/tcp.py +++ /dev/null @@ -1,97 +0,0 @@ -from __future__ import absolute_import -import select -import socket -from .primitives import ProtocolHandler -from netlib.utils import cleanBin -from netlib.tcp import NetLibError - - -class TCPHandler(ProtocolHandler): - """ - TCPHandler acts as a generic TCP forwarder. - Data will be .log()ed, but not stored any further. - """ - - chunk_size = 4096 - - def __init__(self, c, log=True): - super(TCPHandler, self).__init__(c) - self.log = log - - def handle_messages(self): - self.c.establish_server_connection() - - server = "%s:%s" % self.c.server_conn.address()[:2] - buf = memoryview(bytearray(self.chunk_size)) - conns = [self.c.client_conn.rfile, self.c.server_conn.rfile] - - try: - while True: - r, _, _ = select.select(conns, [], [], 10) - for rfile in r: - if self.c.client_conn.rfile == rfile: - src, dst = self.c.client_conn, self.c.server_conn - direction = "-> tcp ->" - src_str, dst_str = "client", server - else: - dst, src = self.c.client_conn, self.c.server_conn - direction = "<- tcp <-" - dst_str, src_str = "client", server - - closed = False - if src.ssl_established: - # Unfortunately, pyOpenSSL lacks a recv_into function. - # We need to read a single byte before .pending() - # becomes usable - contents = src.rfile.read(1) - contents += src.rfile.read(src.connection.pending()) - if not contents: - closed = True - else: - size = src.connection.recv_into(buf) - if not size: - closed = True - - if closed: - conns.remove(src.rfile) - # Shutdown connection to the other peer - if dst.ssl_established: - # We can't half-close a connection, so we just close everything here. - # Sockets will be cleaned up on a higher level. - return - else: - dst.connection.shutdown(socket.SHUT_WR) - - if len(conns) == 0: - return - continue - - if src.ssl_established or dst.ssl_established: - # if one of the peers is over SSL, we need to send - # bytes/strings - if not src.ssl_established: - # we revc'd into buf but need bytes/string now. - contents = buf[:size].tobytes() - if self.log: - self.c.log( - "%s %s\r\n%s" % ( - direction, dst_str, cleanBin(contents) - ), - "info" - ) - # Do not use dst.connection.send here, which may raise - # OpenSSL-specific errors. - dst.send(contents) - else: - # socket.socket.send supports raw bytearrays/memoryviews - if self.log: - self.c.log( - "%s %s\r\n%s" % ( - direction, dst_str, cleanBin(buf.tobytes()) - ), - "info" - ) - dst.connection.send(buf[:size]) - except (socket.error, NetLibError) as e: - self.c.log("TCP connection closed unexpectedly.", "debug") - return -- cgit v1.2.3 From a86ec56012136664688fa4a8efcd866b5e3e17a8 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 15:27:29 +0200 Subject: move files around --- libmproxy/protocol/__init__.py | 13 +- libmproxy/protocol/base.py | 152 +++++++++ libmproxy/protocol/http.py | 602 +++++++++++++++++++++++++++++++----- libmproxy/protocol/http_replay.py | 95 ++++++ libmproxy/protocol/http_wrappers.py | 413 ------------------------- libmproxy/protocol/primitives.py | 166 ---------- libmproxy/protocol/rawtcp.py | 66 ++++ libmproxy/protocol/tls.py | 288 +++++++++++++++++ 8 files changed, 1137 insertions(+), 658 deletions(-) create mode 100644 libmproxy/protocol/base.py create mode 100644 libmproxy/protocol/http_replay.py delete mode 100644 libmproxy/protocol/http_wrappers.py delete mode 100644 libmproxy/protocol/primitives.py create mode 100644 libmproxy/protocol/rawtcp.py create mode 100644 libmproxy/protocol/tls.py (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index bbc20dba..c582592b 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -1 +1,12 @@ -from .primitives import * +from __future__ import (absolute_import, print_function, division) +from .base import Layer, ServerConnectionMixin, Log, Kill +from .http import Http1Layer, Http2Layer +from .tls import TlsLayer, is_tls_record_magic +from .rawtcp import RawTCPLayer + +__all__ = [ + "Layer", "ServerConnectionMixin", "Log", "Kill", + "Http1Layer", "Http2Layer", + "TlsLayer", "is_tls_record_magic", + "RawTCPLayer" +] diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py new file mode 100644 index 00000000..d22a71c6 --- /dev/null +++ b/libmproxy/protocol/base.py @@ -0,0 +1,152 @@ +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible settings (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentProxy + Http1Layer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + HttpProxy + Http1Layer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentProxy + TLSLayer + Http2Layer + HttpLayer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can call functions provided by lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" +from __future__ import (absolute_import, print_function, division) +from netlib import tcp +from ..models import ServerConnection +from ..exceptions import ProtocolException + + +class _LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + + def __init__(self, *args, **kwargs): + super(_LayerCodeCompletion, self).__init__(*args, **kwargs) + if True: + return + self.config = None + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(_LayerCodeCompletion): + def __init__(self, ctx, *args, **kwargs): + """ + Args: + ctx: The (read-only) higher layer. + """ + super(Layer, self).__init__(*args, **kwargs) + self.ctx = ctx + + def __call__(self): + """ + Logic of the layer. + Raises: + ProtocolException in case of protocol exceptions. + """ + raise NotImplementedError + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "{}: {}".format(repr(self.client_conn.address), msg) + ] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + @property + def layers(self): + return [self] + self.ctx.layers + + def __repr__(self): + return type(self).__name__ + + +class ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + + def __init__(self, server_address=None): + super(ServerConnectionMixin, self).__init__() + self.server_conn = ServerConnection(server_address) + + def reconnect(self): + address = self.server_conn.address + self._disconnect() + self.server_conn.address = address + self.connect() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + if depth == 1: + if self.server_conn: + self._disconnect() + self.log("Set new server address: " + repr(address), "debug") + self.server_conn.address = address + else: + self.ctx.set_server(address, server_tls, sni, depth - 1) + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + self.server_conn.finish() + self.server_conn.close() + # self.channel.tell("serverdisconnect", self) + self.server_conn = ServerConnection(None) + + def connect(self): + if not self.server_conn.address: + raise ProtocolException("Cannot connect to server, no server address given.") + self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProtocolException( + "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) + + +class Log(object): + def __init__(self, msg, level="info"): + self.msg = msg + self.level = level + + +class Kill(Exception): + """ + Kill a connection. + """ diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index bde7b088..fc57f6df 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,92 +1,538 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) -from .primitives import Flow +from netlib import tcp +from netlib.http import http1, HttpErrorConnClosed, HttpError +from netlib.http.semantics import CONTENT_MISSING +from netlib import odict +from netlib.tcp import NetLibError, Address +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol -from .http_wrappers import decoded, HTTPRequest, HTTPResponse +from .. import utils +from ..exceptions import InvalidCredentials, HttpException, ProtocolException +from ..models import ( + HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error +) +from .base import Layer, Kill -class HTTPFlow(Flow): - """ - A HTTPFlow is a collection of objects representing a single HTTP - transaction. The main attributes are: - request: HTTPRequest object - response: HTTPResponse object - error: Error object - server_conn: ServerConnection object - client_conn: ClientConnection object +class _HttpLayer(Layer): + supports_streaming = False + + def read_request(self): + raise NotImplementedError() + + def send_request(self, request): + raise NotImplementedError() + + def read_response(self, request_method): + raise NotImplementedError() + + def send_response(self, response): + raise NotImplementedError() + + +class _StreamingHttpLayer(_HttpLayer): + supports_streaming = True + + def read_response_headers(self): + raise NotImplementedError + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + raise NotImplementedError() + yield "this is a generator" + + def send_response_headers(self, response): + raise NotImplementedError + + def send_response_body(self, response, chunks): + raise NotImplementedError() + + +class Http1Layer(_StreamingHttpLayer): + def __init__(self, ctx, mode): + super(Http1Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP1Protocol(self.client_conn) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def read_request(self): + return HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + + def send_request(self, request): + self.server_conn.send(self.server_protocol.assemble(request)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True + ) + + def send_response(self, response): + self.client_conn.send(self.client_protocol.assemble(response)) + + def read_response_headers(self): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=None, # does not matter if we don't read the body. + body_size_limit=self.config.body_size_limit, + include_body=False + ) + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + return self.server_protocol.read_http_body_chunked( + headers, + self.config.body_size_limit, + request_method, + response_code, + False, + max_chunk_size + ) + + def send_response_headers(self, response): + h = self.client_protocol._assemble_response_first_line(response) + self.client_conn.wfile.write(h + "\r\n") + h = self.client_protocol._assemble_response_headers( + response, + preserve_transfer_encoding=True + ) + self.client_conn.send(h + "\r\n") + + def send_response_body(self, response, chunks): + if self.client_protocol.has_chunked_encoding(response.headers): + chunks = ( + "%d\r\n%s\r\n" % (len(chunk), chunk) + for chunk in chunks + ) + for chunk in chunks: + self.client_conn.send(chunk) - Note that it's possible for a Flow to have both a response and an error - object. This might happen, for instance, when a response was received - from the server, but there was an error sending it back to the client. + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP1Protocol(self.server_conn) - The following additional attributes are exposed: + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP1Protocol(self.server_conn) - intercepted: Is this flow currently being intercepted? - live: Does this flow have a live client connection? + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def __call__(self): + layer = HttpLayer(self, self.mode) + layer() + + +# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. +class Http2Layer(_HttpLayer): + def __init__(self, ctx, mode): + super(Http2Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + + def read_request(self): + request = HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + self._stream_id = request.stream_id + return request + + def send_request(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.server_conn.send(self.server_protocol.assemble(message)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True, + stream_id=self._stream_id + ) + + def send_response(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.client_conn.send(self.client_protocol.assemble(message)) + + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def __call__(self): + self.server_protocol.perform_connection_preface() + layer = HttpLayer(self, self.mode) + layer() + + def handle_unexpected_frame(self, frm): + self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") + + +class ConnectServerConnection(object): + """ + "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. """ - def __init__(self, client_conn, server_conn, live=None): - super(HTTPFlow, self).__init__("http", client_conn, server_conn, live) - self.request = None - """@type: HTTPRequest""" - self.response = None - """@type: HTTPResponse""" - - _stateobject_attributes = Flow._stateobject_attributes.copy() - _stateobject_attributes.update( - request=HTTPRequest, - response=HTTPResponse - ) - - @classmethod - def from_state(cls, state): - f = cls(None, None) - f.load_state(state) - return f - - def __repr__(self): - s = " clientconnect (transparent mode destination known) + # > serverconnect (required for client tls handshake) + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + self.reconnect() + get_response() + + # call the appropriate script hook - this is an opportunity for an + # inline script to set flow.stream = True + flow = self.channel.ask("responseheaders", flow) + if flow is None or flow == Kill: + raise Kill() + + if self.supports_streaming: + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = "".join(self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code + )) + flow.response.timestamp_end = utils.timestamp() + + # no further manipulation of self.server_conn beyond this point + # we can safely set it as the final attribute value here. + flow.server_conn = self.server_conn + + self.log( + "response", + "debug", + [repr(flow.response)] + ) + response_reply = self.channel.ask("response", flow) + if response_reply is None or response_reply == Kill: + raise Kill() + + def process_request_hook(self, flow): + # Determine .scheme, .host and .port attributes for inline scripts. + # For absolute-form requests, they are directly given in the request. + # For authority-form requests, we only need to determine the request scheme. + # For relative-form requests, we need to determine host and port as + # well. + if self.mode == "regular": + pass # only absolute-form at this point, nothing to do here. + elif self.mode == "upstream": + if flow.request.form_in == "authority": + flow.request.scheme = "http" # pseudo value + else: + flow.request.host = self.__original_server_conn.address.host + flow.request.port = self.__original_server_conn.address.port + flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" + + request_reply = self.channel.ask("request", flow) + if request_reply is None or request_reply == Kill: + raise Kill() + if isinstance(request_reply, HTTPResponse): + flow.response = request_reply + return + + def establish_server_connection(self, flow): + address = tcp.Address((flow.request.host, flow.request.port)) + tls = (flow.request.scheme == "https") + + if self.mode == "regular" or self.mode == "transparent": + # If there's an existing connection that doesn't match our expectations, kill it. + if address != self.server_conn.address or tls != self.server_conn.ssl_established: + self.set_server(address, tls, address.host) + # Establish connection is neccessary. + if not self.server_conn: + self.connect() + + # SetServer is not guaranteed to work with TLS: + # If there's not TlsLayer below which could catch the exception, + # TLS will not be established. + if tls and not self.server_conn.tls_established: + raise ProtocolException( + "Cannot upgrade to SSL, no TLS layer on the protocol stack.") + else: + if not self.server_conn: + self.connect() + if tls: + raise HttpException("Cannot change scheme in upstream proxy mode.") + """ + # This is a very ugly (untested) workaround to solve a very ugly problem. + if self.server_conn and self.server_conn.tls_established and not ssl: + self.reconnect() + elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: + if self.server_conn.tls_established: + self.reconnect() + + self.send_request(make_connect_request(address)) + tls_layer = TlsLayer(self, False, True) + tls_layer._establish_tls_with_server() + """ + + def validate_request(self, request): + if request.form_in == "absolute" and request.scheme != "http": + self.send_response( + make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + raise HttpException("Invalid request scheme: %s" % request.scheme) + + expected_request_forms = { + "regular": ("absolute",), # an authority request would already be handled. + "upstream": ("authority", "absolute"), + "transparent": ("relative",) + } + + allowed_request_forms = expected_request_forms[self.mode] + if request.form_in not in allowed_request_forms: + err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( + " or ".join(allowed_request_forms), request.form_in + ) + self.send_response(make_error_response(400, err_message)) + raise HttpException(err_message) + + if self.mode == "regular": + request.form_out = "relative" + + def authenticate(self, request): + if self.config.authenticator: + if self.config.authenticator.authenticate(request.headers): + self.config.authenticator.clean(request.headers) + else: + self.send_response(make_error_response( + 407, + "Proxy Authentication Required", + odict.ODictCaseless( + [ + [k, v] for k, v in + self.config.authenticator.auth_challenge_headers().items() + ]) + )) + raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py new file mode 100644 index 00000000..e0144c93 --- /dev/null +++ b/libmproxy/protocol/http_replay.py @@ -0,0 +1,95 @@ +import threading + +from netlib.http import HttpError +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError +from ..controller import Channel +from ..models import Error, HTTPResponse, ServerConnection, make_connect_request +from .base import Log, Kill + + +# TODO: Doesn't really belong into libmproxy.protocol... + + +class RequestReplayThread(threading.Thread): + name = "RequestReplayThread" + + def __init__(self, config, flow, masterq, should_exit): + """ + masterqueue can be a queue or None, if no scripthooks should be + processed. + """ + self.config, self.flow = config, flow + if masterq: + self.channel = Channel(masterq, should_exit) + else: + self.channel = None + super(RequestReplayThread, self).__init__() + + def run(self): + r = self.flow.request + form_out_backup = r.form_out + try: + self.flow.response = None + + # If we have a channel, run script hooks. + if self.channel: + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == Kill: + raise Kill() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply + + if not self.flow.response: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + server_address = self.config.upstream_server.address + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + connect_request = make_connect_request((r.host, r.port)) + server.send(protocol.assemble(connect_request)) + resp = protocol.read_response("CONNECT") + if resp.code != 200: + raise HttpError(502, "Upstream server refuses CONNECT request") + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + + server.send(protocol.assemble(r)) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_protocol( + protocol, + r.method, + body_size_limit=self.config.body_size_limit, + ) + if self.channel: + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == Kill: + raise Kill() + except (HttpError, NetLibError) as v: + self.flow.error = Error(repr(v)) + if self.channel: + self.channel.ask("error", self.flow) + except Kill: + # KillSignal should only be raised if there's a channel in the + # first place. + self.channel.tell("log", Log("Connection killed", "info")) + finally: + r.form_out = form_out_backup diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py deleted file mode 100644 index a26ddbb4..00000000 --- a/libmproxy/protocol/http_wrappers.py +++ /dev/null @@ -1,413 +0,0 @@ -from __future__ import absolute_import -import Cookie -import copy -import time -from email.utils import parsedate_tz, formatdate, mktime_tz - -from netlib import odict, encoding -from netlib.http import semantics, CONTENT_MISSING -from .. import utils, stateobject - - -class decoded(object): - """ - A context manager that decodes a request or response, and then - re-encodes it with the same encoding after execution of the block. - - Example: - with decoded(request): - request.content = request.content.replace("foo", "bar") - """ - - def __init__(self, o): - self.o = o - ce = o.headers.get_first("content-encoding") - if ce in encoding.ENCODINGS: - self.ce = ce - else: - self.ce = None - - def __enter__(self): - if self.ce: - self.o.decode() - - def __exit__(self, type, value, tb): - if self.ce: - self.o.encode(self.ce) - - -class MessageMixin(stateobject.StateObject): - _stateobject_attributes = dict( - httpversion=tuple, - headers=odict.ODictCaseless, - body=str, - timestamp_start=float, - timestamp_end=float - ) - _stateobject_long_attributes = {"body"} - - def get_state(self, short=False): - ret = super(MessageMixin, self).get_state(short) - if short: - if self.body: - ret["contentLength"] = len(self.body) - elif self.body == CONTENT_MISSING: - ret["contentLength"] = None - else: - ret["contentLength"] = 0 - return ret - - def get_decoded_content(self): - """ - Returns the decoded content based on the current Content-Encoding - header. - Doesn't change the message iteself or its headers. - """ - ce = self.headers.get_first("content-encoding") - if not self.body or ce not in encoding.ENCODINGS: - return self.body - return encoding.decode(ce, self.body) - - def decode(self): - """ - Decodes body based on the current Content-Encoding header, then - removes the header. If there is no Content-Encoding header, no - action is taken. - - Returns True if decoding succeeded, False otherwise. - """ - ce = self.headers.get_first("content-encoding") - if not self.body or ce not in encoding.ENCODINGS: - return False - data = encoding.decode(ce, self.body) - if data is None: - return False - self.body = data - del self.headers["content-encoding"] - return True - - def encode(self, e): - """ - Encodes body with the encoding e, where e is "gzip", "deflate" - or "identity". - """ - # FIXME: Error if there's an existing encoding header? - self.body = encoding.encode(e, self.body) - self.headers["content-encoding"] = [e] - - def copy(self): - c = copy.copy(self) - c.headers = self.headers.copy() - return c - - def replace(self, pattern, repl, *args, **kwargs): - """ - Replaces a regular expression pattern with repl in both the headers - and the body of the message. Encoded body will be decoded - before replacement, and re-encoded afterwards. - - Returns the number of replacements made. - """ - with decoded(self): - self.body, c = utils.safe_subn( - pattern, repl, self.body, *args, **kwargs - ) - c += self.headers.replace(pattern, repl, *args, **kwargs) - return c - - -class HTTPRequest(MessageMixin, semantics.Request): - """ - An HTTP request. - - Exposes the following attributes: - - method: HTTP method - - scheme: URL scheme (http/https) - - host: Target hostname of the request. This is not neccessarily the - directy upstream server (which could be another proxy), but it's always - the target server we want to reach at the end. This attribute is either - inferred from the request itself (absolute-form, authority-form) or from - the connection metadata (e.g. the host in reverse proxy mode). - - port: Destination port - - path: Path portion of the URL (not present in authority-form) - - httpversion: HTTP version tuple, e.g. (1,1) - - headers: odict.ODictCaseless object - - content: Content of the request, None, or CONTENT_MISSING if there - is content associated, but not present. CONTENT_MISSING evaluates - to False to make checking for the presence of content natural. - - form_in: The request form which mitmproxy has received. The following - values are possible: - - - relative (GET /index.html, OPTIONS *) (covers origin form and - asterisk form) - - absolute (GET http://example.com:80/index.html) - - authority-form (CONNECT example.com:443) - Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 - - form_out: The request form which mitmproxy will send out to the - destination - - timestamp_start: Timestamp indicating when request transmission started - - timestamp_end: Timestamp indicating when request transmission ended - """ - - def __init__( - self, - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - body, - timestamp_start=None, - timestamp_end=None, - form_out=None, - ): - semantics.Request.__init__( - self, - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - body, - timestamp_start, - timestamp_end, - ) - self.form_out = form_out or form_in - - # Have this request's cookies been modified by sticky cookies or auth? - self.stickycookie = False - self.stickyauth = False - - # Is this request replayed? - self.is_replay = False - - _stateobject_attributes = MessageMixin._stateobject_attributes.copy() - _stateobject_attributes.update( - form_in=str, - method=str, - scheme=str, - host=str, - port=int, - path=str, - form_out=str, - is_replay=bool - ) - - @classmethod - def from_state(cls, state): - f = cls( - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None) - f.load_state(state) - return f - - @classmethod - def from_protocol( - self, - protocol, - *args, - **kwargs - ): - req = protocol.read_request(*args, **kwargs) - return self.wrap(req) - - @classmethod - def wrap(self, request): - req = HTTPRequest( - form_in=request.form_in, - method=request.method, - scheme=request.scheme, - host=request.host, - port=request.port, - path=request.path, - httpversion=request.httpversion, - headers=request.headers, - body=request.body, - timestamp_start=request.timestamp_start, - timestamp_end=request.timestamp_end, - form_out=(request.form_out if hasattr(request, 'form_out') else None), - ) - if hasattr(request, 'stream_id'): - req.stream_id = request.stream_id - return req - - def __hash__(self): - return id(self) - - def replace(self, pattern, repl, *args, **kwargs): - """ - Replaces a regular expression pattern with repl in the headers, the - request path and the body of the request. Encoded content will be - decoded before replacement, and re-encoded afterwards. - - Returns the number of replacements made. - """ - c = MessageMixin.replace(self, pattern, repl, *args, **kwargs) - self.path, pc = utils.safe_subn( - pattern, repl, self.path, *args, **kwargs - ) - c += pc - return c - - -class HTTPResponse(MessageMixin, semantics.Response): - """ - An HTTP response. - - Exposes the following attributes: - - httpversion: HTTP version tuple, e.g. (1, 0), (1, 1), or (2, 0) - - status_code: HTTP response status code - - msg: HTTP response message - - headers: ODict Caseless object - - content: Content of the request, None, or CONTENT_MISSING if there - is content associated, but not present. CONTENT_MISSING evaluates - to False to make checking for the presence of content natural. - - timestamp_start: Timestamp indicating when request transmission started - - timestamp_end: Timestamp indicating when request transmission ended - """ - - def __init__( - self, - httpversion, - status_code, - msg, - headers, - body, - timestamp_start=None, - timestamp_end=None, - ): - semantics.Response.__init__( - self, - httpversion, - status_code, - msg, - headers, - body, - timestamp_start=timestamp_start, - timestamp_end=timestamp_end, - ) - - # Is this request replayed? - self.is_replay = False - self.stream = False - - _stateobject_attributes = MessageMixin._stateobject_attributes.copy() - _stateobject_attributes.update( - status_code=int, - msg=str - ) - - @classmethod - def from_state(cls, state): - f = cls(None, None, None, None, None) - f.load_state(state) - return f - - @classmethod - def from_protocol( - self, - protocol, - *args, - **kwargs - ): - resp = protocol.read_response(*args, **kwargs) - return self.wrap(resp) - - @classmethod - def wrap(self, response): - resp = HTTPResponse( - httpversion=response.httpversion, - status_code=response.status_code, - msg=response.msg, - headers=response.headers, - body=response.body, - timestamp_start=response.timestamp_start, - timestamp_end=response.timestamp_end, - ) - if hasattr(response, 'stream_id'): - resp.stream_id = response.stream_id - return resp - - def _refresh_cookie(self, c, delta): - """ - Takes a cookie string c and a time delta in seconds, and returns - a refreshed cookie string. - """ - c = Cookie.SimpleCookie(str(c)) - for i in c.values(): - if "expires" in i: - d = parsedate_tz(i["expires"]) - if d: - d = mktime_tz(d) + delta - i["expires"] = formatdate(d) - else: - # This can happen when the expires tag is invalid. - # reddit.com sends a an expires tag like this: "Thu, 31 Dec - # 2037 23:59:59 GMT", which is valid RFC 1123, but not - # strictly correct according to the cookie spec. Browsers - # appear to parse this tolerantly - maybe we should too. - # For now, we just ignore this. - del i["expires"] - return c.output(header="").strip() - - def refresh(self, now=None): - """ - This fairly complex and heuristic function refreshes a server - response for replay. - - - It adjusts date, expires and last-modified headers. - - It adjusts cookie expiration. - """ - if not now: - now = time.time() - delta = now - self.timestamp_start - refresh_headers = [ - "date", - "expires", - "last-modified", - ] - for i in refresh_headers: - if i in self.headers: - d = parsedate_tz(self.headers[i][0]) - if d: - new = mktime_tz(d) + delta - self.headers[i] = [formatdate(new)] - c = [] - for i in self.headers["set-cookie"]: - c.append(self._refresh_cookie(i, delta)) - if c: - self.headers["set-cookie"] = c diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py deleted file mode 100644 index c663f0c5..00000000 --- a/libmproxy/protocol/primitives.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import absolute_import -import copy -import uuid - -from .. import stateobject, utils, version -from ..proxy.connection import ClientConnection, ServerConnection - -KILL = 0 # const for killed requests - - -class Error(stateobject.StateObject): - """ - An Error. - - This is distinct from an protocol error response (say, a HTTP code 500), - which is represented by a normal HTTPResponse object. This class is - responsible for indicating errors that fall outside of normal protocol - communications, like interrupted connections, timeouts, protocol errors. - - Exposes the following attributes: - - flow: Flow object - msg: Message describing the error - timestamp: Seconds since the epoch - """ - - def __init__(self, msg, timestamp=None): - """ - @type msg: str - @type timestamp: float - """ - self.flow = None # will usually be set by the flow backref mixin - self.msg = msg - self.timestamp = timestamp or utils.timestamp() - - _stateobject_attributes = dict( - msg=str, - timestamp=float - ) - - def __str__(self): - return self.msg - - @classmethod - def from_state(cls, state): - # the default implementation assumes an empty constructor. Override - # accordingly. - f = cls(None) - f.load_state(state) - return f - - def copy(self): - c = copy.copy(self) - return c - - -class Flow(stateobject.StateObject): - """ - A Flow is a collection of objects representing a single transaction. - This class is usually subclassed for each protocol, e.g. HTTPFlow. - """ - - def __init__(self, type, client_conn, server_conn, live=None): - self.type = type - self.id = str(uuid.uuid4()) - self.client_conn = client_conn - """@type: ClientConnection""" - self.server_conn = server_conn - """@type: ServerConnection""" - self.live = live - """@type: LiveConnection""" - - self.error = None - """@type: Error""" - self.intercepted = False - """@type: bool""" - self._backup = None - self.reply = None - - _stateobject_attributes = dict( - id=str, - error=Error, - client_conn=ClientConnection, - server_conn=ServerConnection, - type=str, - intercepted=bool - ) - - def get_state(self, short=False): - d = super(Flow, self).get_state(short) - d.update(version=version.IVERSION) - if self._backup and self._backup != d: - if short: - d.update(modified=True) - else: - d.update(backup=self._backup) - return d - - def __eq__(self, other): - return self is other - - def copy(self): - f = copy.copy(self) - - f.id = str(uuid.uuid4()) - f.live = False - f.client_conn = self.client_conn.copy() - f.server_conn = self.server_conn.copy() - - if self.error: - f.error = self.error.copy() - return f - - def modified(self): - """ - Has this Flow been modified? - """ - if self._backup: - return self._backup != self.get_state() - else: - return False - - def backup(self, force=False): - """ - Save a backup of this Flow, which can be reverted to using a - call to .revert(). - """ - if not self._backup: - self._backup = self.get_state() - - def revert(self): - """ - Revert to the last backed up state. - """ - if self._backup: - self.load_state(self._backup) - self._backup = None - - def kill(self, master): - """ - Kill this request. - """ - self.error = Error("Connection killed") - self.intercepted = False - self.reply(KILL) - master.handle_error(self) - - def intercept(self, master): - """ - Intercept this Flow. Processing will stop until accept_intercept is - called. - """ - if self.intercepted: - return - self.intercepted = True - master.handle_intercept(self) - - def accept_intercept(self, master): - """ - Continue with the flow - called after an intercept(). - """ - if not self.intercepted: - return - self.intercepted = False - self.reply() - master.handle_accept_intercept(self) diff --git a/libmproxy/protocol/rawtcp.py b/libmproxy/protocol/rawtcp.py new file mode 100644 index 00000000..86468773 --- /dev/null +++ b/libmproxy/protocol/rawtcp.py @@ -0,0 +1,66 @@ +from __future__ import (absolute_import, print_function, division) +import socket +import select + +from OpenSSL import SSL + +from netlib.tcp import NetLibError +from netlib.utils import cleanBin +from ..exceptions import ProtocolException +from .base import Layer + + +class RawTCPLayer(Layer): + chunk_size = 4096 + + def __init__(self, ctx, logging=True): + self.logging = logging + super(RawTCPLayer, self).__init__(ctx) + + def __call__(self): + self.connect() + + buf = memoryview(bytearray(self.chunk_size)) + + client = self.client_conn.connection + server = self.server_conn.connection + conns = [client, server] + + try: + while True: + r, _, _ = select.select(conns, [], [], 10) + for conn in r: + dst = server if conn == client else client + + size = conn.recv_into(buf, self.chunk_size) + if not size: + conns.remove(conn) + # Shutdown connection to the other peer + if isinstance(conn, SSL.Connection): + # We can't half-close a connection, so we just close everything here. + # Sockets will be cleaned up on a higher level. + return + else: + dst.shutdown(socket.SHUT_WR) + + if len(conns) == 0: + return + continue + + dst.sendall(buf[:size]) + + if self.logging: + # log messages are prepended with the client address, + # hence the "weird" direction string. + if dst == server: + direction = "-> tcp -> {}".format(repr(self.server_conn.address)) + else: + direction = "<- tcp <- {}".format(repr(self.server_conn.address)) + data = cleanBin(buf[:size].tobytes()) + self.log( + "{}\r\n{}".format(direction, data), + "info" + ) + + except (socket.error, NetLibError, SSL.Error) as e: + raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py new file mode 100644 index 00000000..b85a6595 --- /dev/null +++ b/libmproxy/protocol/tls.py @@ -0,0 +1,288 @@ +from __future__ import (absolute_import, print_function, division) + +import struct + +from construct import ConstructError + +from netlib.tcp import NetLibError, NetLibInvalidCertificateError +from netlib.http.http1 import HTTP1Protocol +from ..contrib.tls._constructs import ClientHello +from ..exceptions import ProtocolException +from .base import Layer + + +def is_tls_record_magic(d): + """ + Returns: + True, if the passed bytes start with the TLS record magic bytes. + False, otherwise. + """ + d = d[:3] + + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + return ( + len(d) == 3 and + d[0] == '\x16' and + d[1] == '\x03' and + d[2] in ('\x00', '\x01', '\x02', '\x03') + ) + + +class TlsLayer(Layer): + def __init__(self, ctx, client_tls, server_tls): + self.client_sni = None + self.client_alpn_protocols = None + + super(TlsLayer, self).__init__(ctx) + self._client_tls = client_tls + self._server_tls = server_tls + + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values. + + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + + client_tls_requires_server_cert = ( + self._client_tls and self._server_tls and not self.config.no_upstream_cert + ) + + if self._client_tls: + self._parse_client_hello() + + if client_tls_requires_server_cert: + self._establish_tls_with_client_and_server() + elif self._client_tls: + self._establish_tls_with_client() + + layer = self.ctx.next_layer(self) + layer() + + def __repr__(self): + if self._client_tls and self._server_tls: + return "TlsLayer(client and server)" + elif self._client_tls: + return "TlsLayer(client)" + elif self._server_tls: + return "TlsLayer(server)" + else: + return "TlsLayer(inactive)" + + def _get_client_hello(self): + """ + Peek into the socket and read all records that contain the initial client hello message. + + Returns: + The raw handshake packet bytes, without TLS record header(s). + """ + client_hello = "" + client_hello_size = 1 + offset = 0 + while len(client_hello) < client_hello_size: + record_header = self.client_conn.rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) != 5: + raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) + record_size = struct.unpack("!H", record_header[3:])[0] + 5 + record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) + client_hello += record_body + offset += record_size + client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 + return client_hello + + def _parse_client_hello(self): + """ + Peek into the connection, read the initial client hello and parse it to obtain ALPN values. + """ + try: + raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + except ProtocolException as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + return + + try: + client_hello = ClientHello.parse(raw_client_hello) + except ConstructError as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") + return + + for extension in client_hello.extensions: + if extension.type == 0x00: + if len(extension.server_names) != 1 or extension.server_names[0].type != 0: + self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") + self.client_sni = extension.server_names[0].name + elif extension.type == 0x10: + self.client_alpn_protocols = list(extension.alpn_protocols) + + self.log( + "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), + "debug" + ) + + def connect(self): + if not self.server_conn: + self.ctx.connect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def reconnect(self): + self.ctx.reconnect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + self.ctx.set_server(address, server_tls, sni, depth) + if depth == 1 and server_tls is not None: + self._sni_from_server_change = sni + self._server_tls = server_tls + + @property + def sni_for_server_connection(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self.client_sni + + @property + def alpn_for_client_connection(self): + return self.server_conn.get_alpn_proto_negotiated() + + def __alpn_select_callback(self, conn_, options): + """ + Once the client signals the alternate protocols it supports, + we reconnect upstream with the same list and pass the server's choice down to the client. + """ + + # This gets triggered if we haven't established an upstream connection yet. + default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 + # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 + + if self.alpn_for_client_connection in options: + choice = bytes(self.alpn_for_client_connection) + elif default_alpn in options: + choice = bytes(default_alpn) + else: + choice = options[0] + self.log("ALPN for client: %s" % choice, "debug") + return choice + + def _establish_tls_with_client_and_server(self): + self.ctx.connect() + + # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless + # to send an error message over TLS. + try: + self._establish_tls_with_server() + except Exception as e: + try: + self._establish_tls_with_client() + except: + pass + raise e + + self._establish_tls_with_client() + + def _establish_tls_with_client(self): + self.log("Establish TLS with client", "debug") + cert, key, chain_file = self._find_cert() + + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file, + alpn_select_callback=self.__alpn_select_callback, + ) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) + + def _establish_tls_with_server(self): + self.log("Establish TLS with server", "debug") + try: + # We only support http/1.1 and h2. + # If the server only supports spdy (next to http/1.1), it may select that + # and mitmproxy would enter TCP passthrough mode, which we want to avoid. + deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") + if self.client_alpn_protocols: + alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + else: + alpn = None + + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni_for_server_connection, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + alpn_protos=alpn, + ) + tls_cert_err = self.server_conn.ssl_verification_error + if tls_cert_err is not None: + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except NetLibInvalidCertificateError as e: + tls_cert_err = self.server_conn.ssl_verification_error + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + + self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") + + def _find_cert(self): + host = self.server_conn.address.host + sans = set() + # Incorporate upstream certificate + use_upstream_cert = ( + self.server_conn and + self.server_conn.tls_established and + (not self.config.no_upstream_cert) + ) + if use_upstream_cert: + upstream_cert = self.server_conn.cert + sans.update(upstream_cert.altnames) + if upstream_cert.cn: + sans.add(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self.client_sni: + sans.add(self.client_sni) + if self._sni_from_server_change: + sans.add(self._sni_from_server_change) + + sans.discard(host) + return self.config.certstore.get_cert(host, list(sans)) -- cgit v1.2.3 From 63ad4a4f5117d34ba6e9692eef1fc88f68b19c3d Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 15:59:50 +0200 Subject: coverage++ --- libmproxy/protocol/base.py | 4 ++-- libmproxy/protocol/http.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index d22a71c6..1c9b356c 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -43,7 +43,7 @@ class _LayerCodeCompletion(object): Dummy class that provides type hinting in PyCharm, which simplifies development a lot. """ - def __init__(self, *args, **kwargs): + def __init__(self, *args, **kwargs): # pragma: nocover super(_LayerCodeCompletion, self).__init__(*args, **kwargs) if True: return @@ -70,7 +70,7 @@ class Layer(_LayerCodeCompletion): Raises: ProtocolException in case of protocol exceptions. """ - raise NotImplementedError + raise NotImplementedError() def __getattr__(self, name): """ diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index fc57f6df..345b3aa8 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -40,7 +40,7 @@ class _StreamingHttpLayer(_HttpLayer): def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): raise NotImplementedError() - yield "this is a generator" + yield "this is a generator" # pragma: no cover def send_response_headers(self, response): raise NotImplementedError -- cgit v1.2.3 From 1e9aef5b1e3e1d60a2bb94d47be03b780c10a497 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 00:14:42 +0200 Subject: fix upstream proxy server change, update example --- libmproxy/protocol/base.py | 4 ++++ libmproxy/protocol/http.py | 25 ++++++++++++------------- libmproxy/protocol/tls.py | 4 +++- 3 files changed, 19 insertions(+), 14 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 1c9b356c..3440cb01 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -116,6 +116,10 @@ class ServerConnectionMixin(object): self._disconnect() self.log("Set new server address: " + repr(address), "debug") self.server_conn.address = address + if server_tls: + raise ProtocolException( + "Cannot upgrade to TLS, no TLS layer on the protocol stack." + ) else: self.ctx.set_server(address, server_tls, sni, depth - 1) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 345b3aa8..3b62c389 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -207,6 +207,9 @@ class ConnectServerConnection(object): def __getattr__(self, item): return getattr(self.via, item) + def __nonzero__(self): + return bool(self.via) + class UpstreamConnectLayer(Layer): def __init__(self, ctx, connect_request): @@ -221,19 +224,22 @@ class UpstreamConnectLayer(Layer): layer = self.ctx.next_layer(self) layer() + def _send_connect_request(self): + self.send_request(self.connect_request) + resp = self.read_response("CONNECT") + if resp.code != 200: + raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") + def connect(self): if not self.server_conn: self.ctx.connect() - self.send_request(self.connect_request) + self._send_connect_request() else: pass # swallow the message def reconnect(self): self.ctx.reconnect() - self.send_request(self.connect_request) - resp = self.read_response("CONNECT") - if resp.code != 200: - raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") + self._send_connect_request() def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: @@ -386,7 +392,7 @@ class HttpLayer(Layer): if self.supports_streaming: flow.response = self.read_response_headers() else: - flow.response = self.read_response() + flow.response = self.read_response(flow.request.method) try: get_response() @@ -473,13 +479,6 @@ class HttpLayer(Layer): # Establish connection is neccessary. if not self.server_conn: self.connect() - - # SetServer is not guaranteed to work with TLS: - # If there's not TlsLayer below which could catch the exception, - # TLS will not be established. - if tls and not self.server_conn.tls_established: - raise ProtocolException( - "Cannot upgrade to SSL, no TLS layer on the protocol stack.") else: if not self.server_conn: self.connect() diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index b85a6595..2646ec4f 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -152,10 +152,12 @@ class TlsLayer(Layer): self._establish_tls_with_server() def set_server(self, address, server_tls=None, sni=None, depth=1): - self.ctx.set_server(address, server_tls, sni, depth) if depth == 1 and server_tls is not None: + self.ctx.set_server(address, None, None, 1) self._sni_from_server_change = sni self._server_tls = server_tls + else: + self.ctx.set_server(address, server_tls, sni, depth) @property def sni_for_server_connection(self): -- cgit v1.2.3 From 7450bef615436d39bcd2a0d2a8892b8f42beea6f Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 13:43:30 +0200 Subject: fix dns_spoofing example, avoid connecting to itself --- libmproxy/protocol/base.py | 21 ++++++++++++++++++++- libmproxy/protocol/http_replay.py | 1 + 2 files changed, 21 insertions(+), 1 deletion(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 3440cb01..d1af547f 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -103,6 +103,7 @@ class ServerConnectionMixin(object): def __init__(self, server_address=None): super(ServerConnectionMixin, self).__init__() self.server_conn = ServerConnection(server_address) + self._check_self_connect() def reconnect(self): address = self.server_conn.address @@ -110,12 +111,30 @@ class ServerConnectionMixin(object): self.server_conn.address = address self.connect() + def _check_self_connect(self): + """ + We try to protect the proxy from _accidentally_ connecting to itself, + e.g. because of a failed transparent lookup or an invalid configuration. + """ + address = self.server_conn.address + if address: + self_connect = ( + address.port == self.config.port and + address.host in ("localhost", "127.0.0.1", "::1") + ) + if self_connect: + raise ProtocolException( + "Invalid server address: {}\r\n" + "The proxy shall not connect to itself.".format(repr(address)) + ) + def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: if self.server_conn: self._disconnect() self.log("Set new server address: " + repr(address), "debug") self.server_conn.address = address + self._check_self_connect() if server_tls: raise ProtocolException( "Cannot upgrade to TLS, no TLS layer on the protocol stack." @@ -141,7 +160,7 @@ class ServerConnectionMixin(object): self.server_conn.connect() except tcp.NetLibError as e: raise ProtocolException( - "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) + "Server connection to %s failed: %s" % (repr(self.server_conn.address), e), e) class Log(object): diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py index e0144c93..c37fd131 100644 --- a/libmproxy/protocol/http_replay.py +++ b/libmproxy/protocol/http_replay.py @@ -1,3 +1,4 @@ +from __future__ import (absolute_import, print_function, division) import threading from netlib.http import HttpError -- cgit v1.2.3 From 41e6e538dfa758b7d9f867f85f62e881ae408684 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 13:49:47 +0200 Subject: fix layer initialization --- libmproxy/protocol/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index d1af547f..4eb843e4 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -61,8 +61,8 @@ class Layer(_LayerCodeCompletion): Args: ctx: The (read-only) higher layer. """ - super(Layer, self).__init__(*args, **kwargs) self.ctx = ctx + super(Layer, self).__init__(*args, **kwargs) def __call__(self): """ -- cgit v1.2.3 From b04e6e56ab1e69853abebfb950539e3a3aefbdf2 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 31 Aug 2015 17:05:52 +0200 Subject: update inline script hooks --- libmproxy/protocol/base.py | 10 +++++++--- libmproxy/protocol/http.py | 6 +++--- libmproxy/protocol/http_replay.py | 4 ++-- 3 files changed, 12 insertions(+), 8 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 4eb843e4..40ec0536 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -48,9 +48,11 @@ class _LayerCodeCompletion(object): if True: return self.config = None - """@type: libmproxy.proxy.config.ProxyConfig""" + """@type: libmproxy.proxy.ProxyConfig""" self.client_conn = None - """@type: libmproxy.proxy.connection.ClientConnection""" + """@type: libmproxy.models.ClientConnection""" + self.server_conn = None + """@type: libmproxy.models.ServerConnection""" self.channel = None """@type: libmproxy.controller.Channel""" @@ -62,6 +64,7 @@ class Layer(_LayerCodeCompletion): ctx: The (read-only) higher layer. """ self.ctx = ctx + """@type: libmproxy.protocol.Layer""" super(Layer, self).__init__(*args, **kwargs) def __call__(self): @@ -149,13 +152,14 @@ class ServerConnectionMixin(object): self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) self.server_conn.finish() self.server_conn.close() - # self.channel.tell("serverdisconnect", self) + self.channel.tell("serverdisconnect", self.server_conn) self.server_conn = ServerConnection(None) def connect(self): if not self.server_conn.address: raise ProtocolException("Cannot connect to server, no server address given.") self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + self.channel.ask("serverconnect", self.server_conn) try: self.server_conn.connect() except tcp.NetLibError as e: diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 3b62c389..f0f4ac24 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -418,7 +418,7 @@ class HttpLayer(Layer): # call the appropriate script hook - this is an opportunity for an # inline script to set flow.stream = True flow = self.channel.ask("responseheaders", flow) - if flow is None or flow == Kill: + if flow == Kill: raise Kill() if self.supports_streaming: @@ -442,7 +442,7 @@ class HttpLayer(Layer): [repr(flow.response)] ) response_reply = self.channel.ask("response", flow) - if response_reply is None or response_reply == Kill: + if response_reply == Kill: raise Kill() def process_request_hook(self, flow): @@ -462,7 +462,7 @@ class HttpLayer(Layer): flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" request_reply = self.channel.ask("request", flow) - if request_reply is None or request_reply == Kill: + if request_reply == Kill: raise Kill() if isinstance(request_reply, HTTPResponse): flow.response = request_reply diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py index c37fd131..2759a019 100644 --- a/libmproxy/protocol/http_replay.py +++ b/libmproxy/protocol/http_replay.py @@ -36,7 +36,7 @@ class RequestReplayThread(threading.Thread): # If we have a channel, run script hooks. if self.channel: request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == Kill: + if request_reply == Kill: raise Kill() elif isinstance(request_reply, HTTPResponse): self.flow.response = request_reply @@ -82,7 +82,7 @@ class RequestReplayThread(threading.Thread): ) if self.channel: response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == Kill: + if response_reply == Kill: raise Kill() except (HttpError, NetLibError) as v: self.flow.error = Error(repr(v)) -- cgit v1.2.3 From c4d6b357262a5964a8d10ea20b92d22efc9c68a4 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 31 Aug 2015 22:20:12 +0200 Subject: do not log WindowUpdateFrame frames --- libmproxy/protocol/http.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index f0f4ac24..7f57d17c 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -7,6 +7,7 @@ from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol +from netlib.http.http2.frame import WindowUpdateFrame from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException @@ -187,8 +188,15 @@ class Http2Layer(_HttpLayer): layer = HttpLayer(self, self.mode) layer() - def handle_unexpected_frame(self, frm): - self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") + def handle_unexpected_frame(self, frame): + if isinstance(frame, WindowUpdateFrame): + # Clients are sending WindowUpdate frames depending on their flow control algorithm. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should keep track of our own flow control window and + # stall transmission if the outgoing flow control buffer is full. + return + self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") class ConnectServerConnection(object): -- cgit v1.2.3 From f1c8b47b1eb153d448061c0ddce21030c31af2b7 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Tue, 1 Sep 2015 19:24:36 +0200 Subject: better tls error messages, fix #672 --- libmproxy/protocol/tls.py | 12 ++++++++++-- 1 file changed, 10 insertions(+), 2 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index 2646ec4f..a8dc8bb2 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -259,9 +259,17 @@ class TlsLayer(Layer): (tls_cert_err['depth'], tls_cert_err['errno']), "error") self.log("Aborting connection attempt", "error") - raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + address=repr(self.server_conn.address), + sni=self.sni_for_server_connection, + e=repr(e), + ), e) except NetLibError as e: - raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + address=repr(self.server_conn.address), + sni=self.sni_for_server_connection, + e=repr(e), + ), e) self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") -- cgit v1.2.3