From 0047ac4cdc4b717777140b761a2337e86cdceef9 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Tue, 1 Sep 2015 23:27:36 +0200 Subject: pass-through ciphers from client to server --- libmproxy/protocol/tls.py | 204 +++++++++++++++++++++++++++++++++++++++++++++- 1 file changed, 202 insertions(+), 2 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index a8dc8bb2..2b37c5f4 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -6,11 +6,201 @@ from construct import ConstructError from netlib.tcp import NetLibError, NetLibInvalidCertificateError from netlib.http.http1 import HTTP1Protocol -from ..contrib.tls._constructs import ClientHello +from ..contrib.tls._constructs import ClientHello, CipherSuites from ..exceptions import ProtocolException from .base import Layer +# taken from https://testssl.sh/openssl-rfc.mappping.html +CIPHER_ID_NAME_MAP = { + 0x00: 'NULL-MD5', + 0x01: 'NULL-MD5', + 0x02: 'NULL-SHA', + 0x03: 'EXP-RC4-MD5', + 0x04: 'RC4-MD5', + 0x05: 'RC4-SHA', + 0x06: 'EXP-RC2-CBC-MD5', + 0x07: 'IDEA-CBC-SHA', + 0x08: 'EXP-DES-CBC-SHA', + 0x09: 'DES-CBC-SHA', + 0x0a: 'DES-CBC3-SHA', + 0x0b: 'EXP-DH-DSS-DES-CBC-SHA', + 0x0c: 'DH-DSS-DES-CBC-SHA', + 0x0d: 'DH-DSS-DES-CBC3-SHA', + 0x0e: 'EXP-DH-RSA-DES-CBC-SHA', + 0x0f: 'DH-RSA-DES-CBC-SHA', + 0x10: 'DH-RSA-DES-CBC3-SHA', + 0x11: 'EXP-EDH-DSS-DES-CBC-SHA', + 0x12: 'EDH-DSS-DES-CBC-SHA', + 0x13: 'EDH-DSS-DES-CBC3-SHA', + 0x14: 'EXP-EDH-RSA-DES-CBC-SHA', + 0x15: 'EDH-RSA-DES-CBC-SHA', + 0x16: 'EDH-RSA-DES-CBC3-SHA', + 0x17: 'EXP-ADH-RC4-MD5', + 0x18: 'ADH-RC4-MD5', + 0x19: 'EXP-ADH-DES-CBC-SHA', + 0x1a: 'ADH-DES-CBC-SHA', + 0x1b: 'ADH-DES-CBC3-SHA', + # 0x1c: , + # 0x1d: , + 0x1e: 'KRB5-DES-CBC-SHA', + 0x1f: 'KRB5-DES-CBC3-SHA', + 0x20: 'KRB5-RC4-SHA', + 0x21: 'KRB5-IDEA-CBC-SHA', + 0x22: 'KRB5-DES-CBC-MD5', + 0x23: 'KRB5-DES-CBC3-MD5', + 0x24: 'KRB5-RC4-MD5', + 0x25: 'KRB5-IDEA-CBC-MD5', + 0x26: 'EXP-KRB5-DES-CBC-SHA', + 0x27: 'EXP-KRB5-RC2-CBC-SHA', + 0x28: 'EXP-KRB5-RC4-SHA', + 0x29: 'EXP-KRB5-DES-CBC-MD5', + 0x2a: 'EXP-KRB5-RC2-CBC-MD5', + 0x2b: 'EXP-KRB5-RC4-MD5', + 0x2f: 'AES128-SHA', + 0x30: 'DH-DSS-AES128-SHA', + 0x31: 'DH-RSA-AES128-SHA', + 0x32: 'DHE-DSS-AES128-SHA', + 0x33: 'DHE-RSA-AES128-SHA', + 0x34: 'ADH-AES128-SHA', + 0x35: 'AES256-SHA', + 0x36: 'DH-DSS-AES256-SHA', + 0x37: 'DH-RSA-AES256-SHA', + 0x38: 'DHE-DSS-AES256-SHA', + 0x39: 'DHE-RSA-AES256-SHA', + 0x3a: 'ADH-AES256-SHA', + 0x3b: 'NULL-SHA256', + 0x3c: 'AES128-SHA256', + 0x3d: 'AES256-SHA256', + 0x3e: 'DH-DSS-AES128-SHA256', + 0x3f: 'DH-RSA-AES128-SHA256', + 0x40: 'DHE-DSS-AES128-SHA256', + 0x41: 'CAMELLIA128-SHA', + 0x42: 'DH-DSS-CAMELLIA128-SHA', + 0x43: 'DH-RSA-CAMELLIA128-SHA', + 0x44: 'DHE-DSS-CAMELLIA128-SHA', + 0x45: 'DHE-RSA-CAMELLIA128-SHA', + 0x46: 'ADH-CAMELLIA128-SHA', + 0x62: 'EXP1024-DES-CBC-SHA', + 0x63: 'EXP1024-DHE-DSS-DES-CBC-SHA', + 0x64: 'EXP1024-RC4-SHA', + 0x65: 'EXP1024-DHE-DSS-RC4-SHA', + 0x66: 'DHE-DSS-RC4-SHA', + 0x67: 'DHE-RSA-AES128-SHA256', + 0x68: 'DH-DSS-AES256-SHA256', + 0x69: 'DH-RSA-AES256-SHA256', + 0x6a: 'DHE-DSS-AES256-SHA256', + 0x6b: 'DHE-RSA-AES256-SHA256', + 0x6c: 'ADH-AES128-SHA256', + 0x6d: 'ADH-AES256-SHA256', + 0x80: 'GOST94-GOST89-GOST89', + 0x81: 'GOST2001-GOST89-GOST89', + 0x82: 'GOST94-NULL-GOST94', + 0x83: 'GOST2001-GOST89-GOST89', + 0x84: 'CAMELLIA256-SHA', + 0x85: 'DH-DSS-CAMELLIA256-SHA', + 0x86: 'DH-RSA-CAMELLIA256-SHA', + 0x87: 'DHE-DSS-CAMELLIA256-SHA', + 0x88: 'DHE-RSA-CAMELLIA256-SHA', + 0x89: 'ADH-CAMELLIA256-SHA', + 0x8a: 'PSK-RC4-SHA', + 0x8b: 'PSK-3DES-EDE-CBC-SHA', + 0x8c: 'PSK-AES128-CBC-SHA', + 0x8d: 'PSK-AES256-CBC-SHA', + # 0x8e: , + # 0x8f: , + # 0x90: , + # 0x91: , + # 0x92: , + # 0x93: , + # 0x94: , + # 0x95: , + 0x96: 'SEED-SHA', + 0x97: 'DH-DSS-SEED-SHA', + 0x98: 'DH-RSA-SEED-SHA', + 0x99: 'DHE-DSS-SEED-SHA', + 0x9a: 'DHE-RSA-SEED-SHA', + 0x9b: 'ADH-SEED-SHA', + 0x9c: 'AES128-GCM-SHA256', + 0x9d: 'AES256-GCM-SHA384', + 0x9e: 'DHE-RSA-AES128-GCM-SHA256', + 0x9f: 'DHE-RSA-AES256-GCM-SHA384', + 0xa0: 'DH-RSA-AES128-GCM-SHA256', + 0xa1: 'DH-RSA-AES256-GCM-SHA384', + 0xa2: 'DHE-DSS-AES128-GCM-SHA256', + 0xa3: 'DHE-DSS-AES256-GCM-SHA384', + 0xa4: 'DH-DSS-AES128-GCM-SHA256', + 0xa5: 'DH-DSS-AES256-GCM-SHA384', + 0xa6: 'ADH-AES128-GCM-SHA256', + 0xa7: 'ADH-AES256-GCM-SHA384', + 0x5600: 'TLS_FALLBACK_SCSV', + 0xc001: 'ECDH-ECDSA-NULL-SHA', + 0xc002: 'ECDH-ECDSA-RC4-SHA', + 0xc003: 'ECDH-ECDSA-DES-CBC3-SHA', + 0xc004: 'ECDH-ECDSA-AES128-SHA', + 0xc005: 'ECDH-ECDSA-AES256-SHA', + 0xc006: 'ECDHE-ECDSA-NULL-SHA', + 0xc007: 'ECDHE-ECDSA-RC4-SHA', + 0xc008: 'ECDHE-ECDSA-DES-CBC3-SHA', + 0xc009: 'ECDHE-ECDSA-AES128-SHA', + 0xc00a: 'ECDHE-ECDSA-AES256-SHA', + 0xc00b: 'ECDH-RSA-NULL-SHA', + 0xc00c: 'ECDH-RSA-RC4-SHA', + 0xc00d: 'ECDH-RSA-DES-CBC3-SHA', + 0xc00e: 'ECDH-RSA-AES128-SHA', + 0xc00f: 'ECDH-RSA-AES256-SHA', + 0xc010: 'ECDHE-RSA-NULL-SHA', + 0xc011: 'ECDHE-RSA-RC4-SHA', + 0xc012: 'ECDHE-RSA-DES-CBC3-SHA', + 0xc013: 'ECDHE-RSA-AES128-SHA', + 0xc014: 'ECDHE-RSA-AES256-SHA', + 0xc015: 'AECDH-NULL-SHA', + 0xc016: 'AECDH-RC4-SHA', + 0xc017: 'AECDH-DES-CBC3-SHA', + 0xc018: 'AECDH-AES128-SHA', + 0xc019: 'AECDH-AES256-SHA', + 0xc01a: 'SRP-3DES-EDE-CBC-SHA', + 0xc01b: 'SRP-RSA-3DES-EDE-CBC-SHA', + 0xc01c: 'SRP-DSS-3DES-EDE-CBC-SHA', + 0xc01d: 'SRP-AES-128-CBC-SHA', + 0xc01e: 'SRP-RSA-AES-128-CBC-SHA', + 0xc01f: 'SRP-DSS-AES-128-CBC-SHA', + 0xc020: 'SRP-AES-256-CBC-SHA', + 0xc021: 'SRP-RSA-AES-256-CBC-SHA', + 0xc022: 'SRP-DSS-AES-256-CBC-SHA', + 0xc023: 'ECDHE-ECDSA-AES128-SHA256', + 0xc024: 'ECDHE-ECDSA-AES256-SHA384', + 0xc025: 'ECDH-ECDSA-AES128-SHA256', + 0xc026: 'ECDH-ECDSA-AES256-SHA384', + 0xc027: 'ECDHE-RSA-AES128-SHA256', + 0xc028: 'ECDHE-RSA-AES256-SHA384', + 0xc029: 'ECDH-RSA-AES128-SHA256', + 0xc02a: 'ECDH-RSA-AES256-SHA384', + 0xc02b: 'ECDHE-ECDSA-AES128-GCM-SHA256', + 0xc02c: 'ECDHE-ECDSA-AES256-GCM-SHA384', + 0xc02d: 'ECDH-ECDSA-AES128-GCM-SHA256', + 0xc02e: 'ECDH-ECDSA-AES256-GCM-SHA384', + 0xc02f: 'ECDHE-RSA-AES128-GCM-SHA256', + 0xc030: 'ECDHE-RSA-AES256-GCM-SHA384', + 0xc031: 'ECDH-RSA-AES128-GCM-SHA256', + 0xc032: 'ECDH-RSA-AES256-GCM-SHA384', + 0xcc13: 'ECDHE-RSA-CHACHA20-POLY1305', + 0xcc14: 'ECDHE-ECDSA-CHACHA20-POLY1305', + 0xcc15: 'DHE-RSA-CHACHA20-POLY1305', + 0xff00: 'GOST-MD5', + 0xff01: 'GOST-GOST94', + 0xff02: 'GOST-GOST89MAC', + 0xff03: 'GOST-GOST89STREAM', + 0x010080: 'RC4-MD5', + 0x020080: 'EXP-RC4-MD5', + 0x030080: 'RC2-CBC-MD5', + 0x040080: 'EXP-RC2-CBC-MD5', + 0x050080: 'IDEA-CBC-MD5', + 0x060040: 'DES-CBC-MD5', + 0x0700c0: 'DES-CBC3-MD5', + 0x080080: 'RC4-64-MD5', +} + def is_tls_record_magic(d): """ Returns: @@ -127,6 +317,8 @@ class TlsLayer(Layer): self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") return + self.client_ciphers = client_hello.cipher_suites.cipher_suites + for extension in client_hello.extensions: if extension.type == 0x00: if len(extension.server_names) != 1 or extension.server_names[0].type != 0: @@ -234,6 +426,14 @@ class TlsLayer(Layer): else: alpn = None + ciphers_server = self.config.ciphers_server + if not ciphers_server: + ciphers_server = [] + for id in self.client_ciphers: + if id in CIPHER_ID_NAME_MAP.keys(): + ciphers_server.append(CIPHER_ID_NAME_MAP[id]) + ciphers_server = ':'.join(ciphers_server) + self.server_conn.establish_ssl( self.config.clientcerts, self.sni_for_server_connection, @@ -242,7 +442,7 @@ class TlsLayer(Layer): verify_options=self.config.openssl_verification_mode_server, ca_path=self.config.openssl_trusted_cadir_server, ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, + cipher_list=ciphers_server, alpn_protos=alpn, ) tls_cert_err = self.server_conn.ssl_verification_error -- cgit v1.2.3 From c79af6276340decd34730069614d6cac9283a822 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 2 Sep 2015 20:50:50 +0200 Subject: ignore http2 priority frames --- libmproxy/protocol/http.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 7f57d17c..4be1f762 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -7,7 +7,7 @@ from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol -from netlib.http.http2.frame import WindowUpdateFrame +from netlib.http.http2.frame import PriorityFrame, WindowUpdateFrame from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException @@ -196,6 +196,13 @@ class Http2Layer(_HttpLayer): # Ideally we should keep track of our own flow control window and # stall transmission if the outgoing flow control buffer is full. return + if isinstance(frame, PriorityFrame): + # Clients are sending Priority frames depending on their implementation. + # The RFC does not clearly state when or which priority preferences should be set. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should forward them to the server. + return self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") -- cgit v1.2.3 From 37e6b3c401c21abfc705ed9f3173f9d4b6184169 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 11:09:59 +0200 Subject: http2: improve unexpected frame handling and shutdown --- libmproxy/protocol/http.py | 48 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4be1f762..bf9abc90 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -7,7 +7,7 @@ from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol -from netlib.http.http2.frame import PriorityFrame, WindowUpdateFrame +from netlib.http.http2.frame import Frame, GoAwayFrame, PriorityFrame, WindowUpdateFrame from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException @@ -136,9 +136,9 @@ class Http2Layer(_HttpLayer): super(Http2Layer, self).__init__(ctx) self.mode = mode self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_client) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_server) def read_request(self): request = HTTPRequest.from_protocol( @@ -162,25 +162,26 @@ class Http2Layer(_HttpLayer): ) def send_response(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames + # TODO: implement flow control to prevent client buffer filling up + # maintain a send buffer size, and read WindowUpdateFrames from client to increase the send buffer self.client_conn.send(self.client_protocol.assemble(message)) def connect(self): self.ctx.connect() self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_server) self.server_protocol.perform_connection_preface() def reconnect(self): self.ctx.reconnect() self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_server) self.server_protocol.perform_connection_preface() def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_server) self.server_protocol.perform_connection_preface() def __call__(self): @@ -188,7 +189,24 @@ class Http2Layer(_HttpLayer): layer = HttpLayer(self, self.mode) layer() - def handle_unexpected_frame(self, frame): + # terminate the connection + self.client_conn.send(GoAwayFrame().to_bytes()) + + def handle_unexpected_frame_from_client(self, frame): + if isinstance(frame, PriorityFrame): + # Clients are sending Priority frames depending on their implementation. + # The RFC does not clearly state when or which priority preferences should be set. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should forward them to the server. + return + if isinstance(frame, PingFrame): + # respond with pong + self.server_conn.send(PingFrame(flags=frame.Frame.FLAG_ACK, payload=frame.payload).to_bytes()) + return + self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") + + def handle_unexpected_frame_from_server(self, frame): if isinstance(frame, WindowUpdateFrame): # Clients are sending WindowUpdate frames depending on their flow control algorithm. # Since we cannot predict these frames, and we do not need to respond to them, @@ -196,12 +214,14 @@ class Http2Layer(_HttpLayer): # Ideally we should keep track of our own flow control window and # stall transmission if the outgoing flow control buffer is full. return - if isinstance(frame, PriorityFrame): - # Clients are sending Priority frames depending on their implementation. - # The RFC does not clearly state when or which priority preferences should be set. - # Since we cannot predict these frames, and we do not need to respond to them, - # simply accept them, and hide them from the log. - # Ideally we should forward them to the server. + if isinstance(frame, GoAwayFrame): + # Server wants to terminate the connection, + # relay it to the client. + self.client_conn.send(frame.to_bytes()) + return + if isinstance(frame, PingFrame): + # respond with pong + self.client_conn.send(PingFrame(flags=frame.Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") -- cgit v1.2.3 From bde4bdd1d2182ef81db8cea1fd6baab014f96bb8 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 13:40:35 +0200 Subject: http2: fix unhandled frames --- libmproxy/protocol/http.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index bf9abc90..5e4656b1 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -7,7 +7,7 @@ from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol -from netlib.http.http2.frame import Frame, GoAwayFrame, PriorityFrame, WindowUpdateFrame +from netlib.http.http2.frame import Frame, PingFrame, GoAwayFrame, PriorityFrame, WindowUpdateFrame from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException @@ -193,6 +193,13 @@ class Http2Layer(_HttpLayer): self.client_conn.send(GoAwayFrame().to_bytes()) def handle_unexpected_frame_from_client(self, frame): + if isinstance(frame, WindowUpdateFrame): + # Clients are sending WindowUpdate frames depending on their flow control algorithm. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should keep track of our own flow control window and + # stall transmission if the outgoing flow control buffer is full. + return if isinstance(frame, PriorityFrame): # Clients are sending Priority frames depending on their implementation. # The RFC does not clearly state when or which priority preferences should be set. @@ -207,13 +214,6 @@ class Http2Layer(_HttpLayer): self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") def handle_unexpected_frame_from_server(self, frame): - if isinstance(frame, WindowUpdateFrame): - # Clients are sending WindowUpdate frames depending on their flow control algorithm. - # Since we cannot predict these frames, and we do not need to respond to them, - # simply accept them, and hide them from the log. - # Ideally we should keep track of our own flow control window and - # stall transmission if the outgoing flow control buffer is full. - return if isinstance(frame, GoAwayFrame): # Server wants to terminate the connection, # relay it to the client. -- cgit v1.2.3 From 29ae2bbf911db16c695eccbef682320f6b15f769 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 13:49:27 +0200 Subject: http2: fix multiple stream per connection fixes #746 --- libmproxy/protocol/http.py | 60 +++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 5e4656b1..b37ff7cf 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -32,6 +32,9 @@ class _HttpLayer(Layer): def send_response(self, response): raise NotImplementedError() + def check_close_connection(self, flow): + raise NotImplementedError() + class _StreamingHttpLayer(_HttpLayer): supports_streaming = True @@ -113,6 +116,29 @@ class Http1Layer(_StreamingHttpLayer): for chunk in chunks: self.client_conn.send(chunk) + def check_close_connection(self, flow): + close_connection = ( + http1.HTTP1Protocol.connection_close( + flow.request.httpversion, + flow.request.headers + ) or http1.HTTP1Protocol.connection_close( + flow.response.httpversion, + flow.response.headers + ) or http1.HTTP1Protocol.expected_http_body_size( + flow.response.headers, + False, + flow.request.method, + flow.response.code) == -1 + ) + if flow.request.form_in == "authority" and flow.response.code == 200: + # Workaround for + # https://github.com/mitmproxy/mitmproxy/issues/313: Some + # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 + # and no Content-Length header + + return False + return close_connection + def connect(self): self.ctx.connect() self.server_protocol = HTTP1Protocol(self.server_conn) @@ -166,6 +192,10 @@ class Http2Layer(_HttpLayer): # maintain a send buffer size, and read WindowUpdateFrames from client to increase the send buffer self.client_conn.send(self.client_protocol.assemble(message)) + def check_close_connection(self, flow): + # TODO: add a timer to disconnect after a 10 second timeout + return False + def connect(self): self.ctx.connect() self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, @@ -370,36 +400,6 @@ class HttpLayer(Layer): layer = UpstreamConnectLayer(self, connect_request) layer() - def check_close_connection(self, flow): - """ - Checks if the connection should be closed depending on the HTTP - semantics. Returns True, if so. - """ - - # TODO: add logic for HTTP/2 - - close_connection = ( - http1.HTTP1Protocol.connection_close( - flow.request.httpversion, - flow.request.headers - ) or http1.HTTP1Protocol.connection_close( - flow.response.httpversion, - flow.response.headers - ) or http1.HTTP1Protocol.expected_http_body_size( - flow.response.headers, - False, - flow.request.method, - flow.response.code) == -1 - ) - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 - # and no Content-Length header - - return False - return close_connection - def send_response_to_client(self, flow): if not (self.supports_streaming and flow.response.stream): # no streaming: -- cgit v1.2.3 From b4d6f2e12b031ed1fa95b6a029d11dfa9f52d4e9 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 13:52:40 +0200 Subject: http2: fix PingFrame again --- libmproxy/protocol/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index b37ff7cf..b345ee06 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -239,7 +239,7 @@ class Http2Layer(_HttpLayer): return if isinstance(frame, PingFrame): # respond with pong - self.server_conn.send(PingFrame(flags=frame.Frame.FLAG_ACK, payload=frame.payload).to_bytes()) + self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") @@ -251,7 +251,7 @@ class Http2Layer(_HttpLayer): return if isinstance(frame, PingFrame): # respond with pong - self.client_conn.send(PingFrame(flags=frame.Frame.FLAG_ACK, payload=frame.payload).to_bytes()) + self.client_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") -- cgit v1.2.3 From bc93600a66b50d06a7a3a17ee689c5899b61b975 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 13:53:45 +0200 Subject: http2: add GoAway support for client --- libmproxy/protocol/http.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index b345ee06..222af45f 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -237,6 +237,11 @@ class Http2Layer(_HttpLayer): # simply accept them, and hide them from the log. # Ideally we should forward them to the server. return + if isinstance(frame, GoAwayFrame): + # Client wants to terminate the connection, + # relay it to the server. + self.server_conn.send(frame.to_bytes()) + return if isinstance(frame, PingFrame): # respond with pong self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) -- cgit v1.2.3 From 1f6d05f89fada5fe360aa79abfa80a3c91ce54da Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 14:09:59 +0200 Subject: http2: server can send WindowUpdate frames as well --- libmproxy/protocol/http.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 222af45f..dbf46973 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -246,9 +246,16 @@ class Http2Layer(_HttpLayer): # respond with pong self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return - self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") + self.log("Unexpected HTTP2 frame from client: %s" % frame.human_readable(), "info") def handle_unexpected_frame_from_server(self, frame): + if isinstance(frame, WindowUpdateFrame): + # Servers are sending WindowUpdate frames depending on their flow control algorithm. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should keep track of our own flow control window and + # stall transmission if the outgoing flow control buffer is full. + return if isinstance(frame, GoAwayFrame): # Server wants to terminate the connection, # relay it to the client. @@ -258,7 +265,7 @@ class Http2Layer(_HttpLayer): # respond with pong self.client_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return - self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") + self.log("Unexpected HTTP2 frame from server: %s" % frame.human_readable(), "info") class ConnectServerConnection(object): -- cgit v1.2.3 From 3a229f60e35a05b0358ef6bf6c81a3a42461c4a2 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 14:26:36 +0200 Subject: http2: fix ping response --- libmproxy/protocol/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index dbf46973..cf8d86c3 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -244,7 +244,7 @@ class Http2Layer(_HttpLayer): return if isinstance(frame, PingFrame): # respond with pong - self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) + self.client_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return self.log("Unexpected HTTP2 frame from client: %s" % frame.human_readable(), "info") @@ -263,7 +263,7 @@ class Http2Layer(_HttpLayer): return if isinstance(frame, PingFrame): # respond with pong - self.client_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) + self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return self.log("Unexpected HTTP2 frame from server: %s" % frame.human_readable(), "info") -- cgit v1.2.3 From f4272de5ec77fb57723e2274e4ddc50d73489e1e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 3 Sep 2015 17:01:25 +0200 Subject: remove ServerConnectionMixin.reconnect --- libmproxy/protocol/__init__.py | 4 ++-- libmproxy/protocol/base.py | 34 +++++++--------------------------- libmproxy/protocol/http.py | 25 +++++++------------------ libmproxy/protocol/http_replay.py | 5 +++-- libmproxy/protocol/tls.py | 5 ----- 5 files changed, 19 insertions(+), 54 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index c582592b..b0e66dbd 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -1,11 +1,11 @@ from __future__ import (absolute_import, print_function, division) -from .base import Layer, ServerConnectionMixin, Log, Kill +from .base import Layer, ServerConnectionMixin, Kill from .http import Http1Layer, Http2Layer from .tls import TlsLayer, is_tls_record_magic from .rawtcp import RawTCPLayer __all__ = [ - "Layer", "ServerConnectionMixin", "Log", "Kill", + "Layer", "ServerConnectionMixin", "Kill", "Http1Layer", "Http2Layer", "TlsLayer", "is_tls_record_magic", "RawTCPLayer" diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 40ec0536..f1718065 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -81,15 +81,6 @@ class Layer(_LayerCodeCompletion): """ return getattr(self.ctx, name) - def log(self, msg, level, subs=()): - full_msg = [ - "{}: {}".format(repr(self.client_conn.address), msg) - ] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - @property def layers(self): return [self] + self.ctx.layers @@ -106,15 +97,9 @@ class ServerConnectionMixin(object): def __init__(self, server_address=None): super(ServerConnectionMixin, self).__init__() self.server_conn = ServerConnection(server_address) - self._check_self_connect() - - def reconnect(self): - address = self.server_conn.address - self._disconnect() - self.server_conn.address = address - self.connect() + self.__check_self_connect() - def _check_self_connect(self): + def __check_self_connect(self): """ We try to protect the proxy from _accidentally_ connecting to itself, e.g. because of a failed transparent lookup or an invalid configuration. @@ -134,10 +119,10 @@ class ServerConnectionMixin(object): def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: if self.server_conn: - self._disconnect() + self.disconnect() self.log("Set new server address: " + repr(address), "debug") self.server_conn.address = address - self._check_self_connect() + self.__check_self_connect() if server_tls: raise ProtocolException( "Cannot upgrade to TLS, no TLS layer on the protocol stack." @@ -145,15 +130,16 @@ class ServerConnectionMixin(object): else: self.ctx.set_server(address, server_tls, sni, depth - 1) - def _disconnect(self): + def disconnect(self): """ Deletes (and closes) an existing server connection. """ self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + address = self.server_conn.address self.server_conn.finish() self.server_conn.close() self.channel.tell("serverdisconnect", self.server_conn) - self.server_conn = ServerConnection(None) + self.server_conn = ServerConnection(address) def connect(self): if not self.server_conn.address: @@ -167,12 +153,6 @@ class ServerConnectionMixin(object): "Server connection to %s failed: %s" % (repr(self.server_conn.address), e), e) -class Log(object): - def __init__(self, msg, level="info"): - self.msg = msg - self.level = level - - class Kill(Exception): """ Kill a connection. diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index dbf46973..7768b9ad 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -143,10 +143,6 @@ class Http1Layer(_StreamingHttpLayer): self.ctx.connect() self.server_protocol = HTTP1Protocol(self.server_conn) - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP1Protocol(self.server_conn) - def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) self.server_protocol = HTTP1Protocol(self.server_conn) @@ -202,12 +198,6 @@ class Http2Layer(_HttpLayer): unhandled_frame_cb=self.handle_unexpected_frame_from_server) self.server_protocol.perform_connection_preface() - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame_from_server) - self.server_protocol.perform_connection_preface() - def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, @@ -314,14 +304,10 @@ class UpstreamConnectLayer(Layer): else: pass # swallow the message - def reconnect(self): - self.ctx.reconnect() - self._send_connect_request() - def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: if self.ctx.server_conn: - self.ctx.reconnect() + self.ctx.disconnect() address = Address.wrap(address) self.connect_request.host = address.host self.connect_request.port = address.port @@ -459,7 +445,8 @@ class HttpLayer(Layer): # > server detects timeout, disconnects # > read (100-n)% of large request # > send large request upstream - self.reconnect() + self.disconnect() + self.connect() get_response() # call the appropriate script hook - this is an opportunity for an @@ -534,10 +521,12 @@ class HttpLayer(Layer): """ # This is a very ugly (untested) workaround to solve a very ugly problem. if self.server_conn and self.server_conn.tls_established and not ssl: - self.reconnect() + self.disconnect() + self.connect() elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: if self.server_conn.tls_established: - self.reconnect() + self.disconnect() + self.connect() self.send_request(make_connect_request(address)) tls_layer = TlsLayer(self, False, True) diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py index 2759a019..a9ee5506 100644 --- a/libmproxy/protocol/http_replay.py +++ b/libmproxy/protocol/http_replay.py @@ -6,7 +6,7 @@ from netlib.http.http1 import HTTP1Protocol from netlib.tcp import NetLibError from ..controller import Channel from ..models import Error, HTTPResponse, ServerConnection, make_connect_request -from .base import Log, Kill +from .base import Kill # TODO: Doesn't really belong into libmproxy.protocol... @@ -89,8 +89,9 @@ class RequestReplayThread(threading.Thread): if self.channel: self.channel.ask("error", self.flow) except Kill: - # KillSignal should only be raised if there's a channel in the + # Kill should only be raised if there's a channel in the # first place. + from ..proxy.root_context import Log self.channel.tell("log", Log("Connection killed", "info")) finally: r.form_out = form_out_backup diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index 2b37c5f4..00e016ea 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -338,11 +338,6 @@ class TlsLayer(Layer): if self._server_tls and not self.server_conn.tls_established: self._establish_tls_with_server() - def reconnect(self): - self.ctx.reconnect() - if self._server_tls and not self.server_conn.tls_established: - self._establish_tls_with_server() - def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1 and server_tls is not None: self.ctx.set_server(address, None, None, 1) -- cgit v1.2.3 From 99126f62ed947847eba4cfa687cb0b0f012092bb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 3 Sep 2015 18:25:36 +0200 Subject: remove depth attribute from set_server --- libmproxy/protocol/base.py | 23 ++++++++++------------- libmproxy/protocol/http.py | 32 ++++++++++++++++++++++---------- libmproxy/protocol/tls.py | 8 +++----- 3 files changed, 35 insertions(+), 28 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index f1718065..f27cb04b 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -116,19 +116,16 @@ class ServerConnectionMixin(object): "The proxy shall not connect to itself.".format(repr(address)) ) - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1: - if self.server_conn: - self.disconnect() - self.log("Set new server address: " + repr(address), "debug") - self.server_conn.address = address - self.__check_self_connect() - if server_tls: - raise ProtocolException( - "Cannot upgrade to TLS, no TLS layer on the protocol stack." - ) - else: - self.ctx.set_server(address, server_tls, sni, depth - 1) + def set_server(self, address, server_tls=None, sni=None): + if self.server_conn: + self.disconnect() + self.log("Set new server address: " + repr(address), "debug") + self.server_conn.address = address + self.__check_self_connect() + if server_tls: + raise ProtocolException( + "Cannot upgrade to TLS, no TLS layer on the protocol stack." + ) def disconnect(self): """ diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 3c934393..f2265c34 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -304,16 +304,22 @@ class UpstreamConnectLayer(Layer): else: pass # swallow the message - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1: - if self.ctx.server_conn: - self.ctx.disconnect() - address = Address.wrap(address) - self.connect_request.host = address.host - self.connect_request.port = address.port - self.server_conn.address = address - else: - self.ctx.set_server(address, server_tls, sni, depth - 1) + def change_upstream_proxy_server(self, address): + if address != self.server_conn.via.address: + self.ctx.set_server(address) + + def set_server(self, address, server_tls=None, sni=None): + if self.ctx.server_conn: + self.ctx.disconnect() + address = Address.wrap(address) + self.connect_request.host = address.host + self.connect_request.port = address.port + self.server_conn.address = address + + if server_tls: + raise ProtocolException( + "Cannot upgrade to TLS, no TLS layer on the protocol stack." + ) class HttpLayer(Layer): @@ -388,6 +394,12 @@ class HttpLayer(Layer): finally: flow.live = False + def change_upstream_proxy_server(self, address): + # Make set_upstream_proxy_server always available, + # even if there's no UpstreamConnectLayer + if address != self.server_conn.address: + return self.set_server(address) + def handle_regular_mode_connect(self, request): self.set_server((request.host, request.port)) self.send_response(make_connect_response(request.httpversion)) diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index 00e016ea..a62b1a22 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -338,13 +338,11 @@ class TlsLayer(Layer): if self._server_tls and not self.server_conn.tls_established: self._establish_tls_with_server() - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1 and server_tls is not None: - self.ctx.set_server(address, None, None, 1) + def set_server(self, address, server_tls=None, sni=None): + if server_tls is not None: self._sni_from_server_change = sni self._server_tls = server_tls - else: - self.ctx.set_server(address, server_tls, sni, depth) + self.ctx.set_server(address, None, None) @property def sni_for_server_connection(self): -- cgit v1.2.3 From 14457f29b3d89e234d0791c4980e5cf9514185dd Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 3 Sep 2015 18:55:38 +0200 Subject: docs++ --- libmproxy/protocol/__init__.py | 27 ++++++++++ libmproxy/protocol/base.py | 116 +++++++++++++++++++++++++---------------- libmproxy/protocol/tls.py | 4 +- 3 files changed, 100 insertions(+), 47 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index b0e66dbd..35d59f28 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -1,3 +1,30 @@ +""" +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +The first layer is usually the proxy mode, e.g. transparent proxy or normal HTTP proxy. Next, +various protocol layers are stacked on top of each other - imagine WebSockets on top of an HTTP +Upgrade request. An actual mitmproxy connection may look as follows (outermost layer first): + + Transparent HTTP proxy, no TLS: + - TransparentProxy + - Http1Layer + - HttpLayer + + Regular proxy, CONNECT request with WebSockets over SSL: + - ReverseProxy + - Http1Layer + - HttpLayer + - TLSLayer + - WebsocketLayer (or TCPLayer) + +Every layer acts as a read-only context for its inner layers (see :py:class:`Layer`). To communicate +with an outer layer, a layer can use functions provided in the context. The next layer is always +determined by a call to :py:meth:`.next_layer() `, +which is provided by the root context. + +Another subtle design goal of this architecture is that upstream connections should be established +as late as possible; this makes server replay without any outgoing connections possible. +""" + from __future__ import (absolute_import, print_function, division) from .base import Layer, ServerConnectionMixin, Kill from .http import Http1Layer, Http2Layer diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index f27cb04b..9d8c8bfe 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -1,37 +1,3 @@ -""" -mitmproxy protocol architecture - -In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. -For example, the following scenarios depict possible settings (lowest layer first): - -Transparent HTTP proxy, no SSL: - TransparentProxy - Http1Layer - HttpLayer - -Regular proxy, CONNECT request with WebSockets over SSL: - HttpProxy - Http1Layer - HttpLayer - SslLayer - WebsocketLayer (or TcpLayer) - -Automated protocol detection by peeking into the buffer: - TransparentProxy - TLSLayer - Http2Layer - HttpLayer - -Communication between layers is done as follows: - - lower layers provide context information to higher layers - - higher layers can call functions provided by lower layers, - which are propagated until they reach a suitable layer. - -Further goals: - - Connections should always be peekable to make automatic protocol detection work. - - Upstream connections should be established as late as possible; - inline scripts shall have a chance to handle everything locally. -""" from __future__ import (absolute_import, print_function, division) from netlib import tcp from ..models import ServerConnection @@ -43,8 +9,8 @@ class _LayerCodeCompletion(object): Dummy class that provides type hinting in PyCharm, which simplifies development a lot. """ - def __init__(self, *args, **kwargs): # pragma: nocover - super(_LayerCodeCompletion, self).__init__(*args, **kwargs) + def __init__(self, **mixin_args): # pragma: nocover + super(_LayerCodeCompletion, self).__init__(**mixin_args) if True: return self.config = None @@ -55,34 +21,64 @@ class _LayerCodeCompletion(object): """@type: libmproxy.models.ServerConnection""" self.channel = None """@type: libmproxy.controller.Channel""" + self.ctx = None + """@type: libmproxy.protocol.Layer""" class Layer(_LayerCodeCompletion): - def __init__(self, ctx, *args, **kwargs): + """ + Base class for all layers. All other protocol layers should inherit from this class. + """ + + def __init__(self, ctx, **mixin_args): """ + Each layer usually passes itself to its child layers as a context. Properties of the + context are transparently mapped to the layer, so that the following works: + + .. code-block:: python + + root_layer = Layer(None) + root_layer.client_conn = 42 + sub_layer = Layer(root_layer) + print(sub_layer.client_conn) # 42 + + The root layer is passed a :py:class:`libmproxy.proxy.RootContext` object, + which provides access to :py:attr:`.client_conn `, + :py:attr:`.next_layer ` and other basic attributes. + Args: - ctx: The (read-only) higher layer. + ctx: The (read-only) parent layer / context. """ self.ctx = ctx - """@type: libmproxy.protocol.Layer""" - super(Layer, self).__init__(*args, **kwargs) + """ + The parent layer. - def __call__(self): + :type: :py:class:`Layer` """ - Logic of the layer. + super(Layer, self).__init__(**mixin_args) + + def __call__(self): + """Logic of the layer. + + Returns: + Once the protocol has finished without exceptions. + Raises: - ProtocolException in case of protocol exceptions. + ~libmproxy.exceptions.ProtocolException: if an exception occurs. No other exceptions must be raised. """ raise NotImplementedError() def __getattr__(self, name): """ - Attributes not present on the current layer may exist on a higher layer. + Attributes not present on the current layer are looked up on the context. """ return getattr(self.ctx, name) @property def layers(self): + """ + List of all layers, including the current layer (``[self, self.ctx, self.ctx.ctx, ...]``) + """ return [self] + self.ctx.layers def __repr__(self): @@ -92,6 +88,20 @@ class Layer(_LayerCodeCompletion): class ServerConnectionMixin(object): """ Mixin that provides a layer with the capabilities to manage a server connection. + The server address can be passed in the constructor or set by calling :py:meth:`set_server`. + Subclasses are responsible for calling :py:meth:`disconnect` before returning. + + Recommended Usage: + + .. code-block:: python + + class MyLayer(Layer, ServerConnectionMixin): + def __call__(self): + try: + # Do something. + finally: + if self.server_conn: + self.disconnect() """ def __init__(self, server_address=None): @@ -117,6 +127,14 @@ class ServerConnectionMixin(object): ) def set_server(self, address, server_tls=None, sni=None): + """ + Sets a new server address. If there is an existing connection, it will be closed. + + Raises: + ~libmproxy.exceptions.ProtocolException: + if ``server_tls`` is ``True``, but there was no TLS layer on the + protocol stack which could have processed this. + """ if self.server_conn: self.disconnect() self.log("Set new server address: " + repr(address), "debug") @@ -130,6 +148,7 @@ class ServerConnectionMixin(object): def disconnect(self): """ Deletes (and closes) an existing server connection. + Must not be called if there is no existing connection. """ self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) address = self.server_conn.address @@ -139,6 +158,13 @@ class ServerConnectionMixin(object): self.server_conn = ServerConnection(address) def connect(self): + """ + Establishes a server connection. + Must not be called if there is an existing connection. + + Raises: + ~libmproxy.exceptions.ProtocolException: if the connection could not be established. + """ if not self.server_conn.address: raise ProtocolException("Cannot connect to server, no server address given.") self.log("serverconnect", "debug", [repr(self.server_conn.address)]) @@ -152,5 +178,5 @@ class ServerConnectionMixin(object): class Kill(Exception): """ - Kill a connection. + Signal that both client and server connection(s) should be killed immediately. """ diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index a62b1a22..88a8398f 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -237,8 +237,8 @@ class TlsLayer(Layer): If so, we first connect to the server and then to the client. If not, we only connect to the client and do the server_ssl lazily on a Connect message. - An additional complexity is that establish ssl with the server may require a SNI value from the client. - In an ideal world, we'd do the following: + An additional complexity is that establish ssl with the server may require a SNI value from + the client. In an ideal world, we'd do the following: 1. Start the SSL handshake with the client 2. Check if the client sends a SNI. 3. Pause the client handshake, establish SSL with the server. -- cgit v1.2.3 From 47ab7f04eaa85dda7be524b946c22222b2a6de91 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 21:23:19 +0200 Subject: http2: Ping frames are handled in netlib --- libmproxy/protocol/http.py | 8 -------- 1 file changed, 8 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index f2265c34..74c93e16 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -232,10 +232,6 @@ class Http2Layer(_HttpLayer): # relay it to the server. self.server_conn.send(frame.to_bytes()) return - if isinstance(frame, PingFrame): - # respond with pong - self.client_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) - return self.log("Unexpected HTTP2 frame from client: %s" % frame.human_readable(), "info") def handle_unexpected_frame_from_server(self, frame): @@ -251,10 +247,6 @@ class Http2Layer(_HttpLayer): # relay it to the client. self.client_conn.send(frame.to_bytes()) return - if isinstance(frame, PingFrame): - # respond with pong - self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) - return self.log("Unexpected HTTP2 frame from server: %s" % frame.human_readable(), "info") -- cgit v1.2.3 From 00561d280ccd4aac06b13b434e0aef4492148cb5 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 4 Sep 2015 02:11:09 +0200 Subject: speed up filters --- libmproxy/protocol/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 74c93e16..f51fea95 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -7,7 +7,7 @@ from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol -from netlib.http.http2.frame import Frame, PingFrame, GoAwayFrame, PriorityFrame, WindowUpdateFrame +from netlib.http.http2.frame import GoAwayFrame, PriorityFrame, WindowUpdateFrame from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException -- cgit v1.2.3 From 5125c669ccd2db5de5f90c66db61e64f63f3ba4c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 5 Sep 2015 20:45:58 +0200 Subject: adjust to new netlib Headers class --- libmproxy/protocol/http.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index f51fea95..fbf4ac9b 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,7 +1,7 @@ from __future__ import (absolute_import, print_function, division) from netlib import tcp -from netlib.http import http1, HttpErrorConnClosed, HttpError +from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers from netlib.http.semantics import CONTENT_MISSING from netlib import odict from netlib.tcp import NetLibError, Address @@ -568,10 +568,6 @@ class HttpLayer(Layer): self.send_response(make_error_response( 407, "Proxy Authentication Required", - odict.ODictCaseless( - [ - [k, v] for k, v in - self.config.authenticator.auth_challenge_headers().items() - ]) + Headers(**self.config.authenticator.auth_challenge_headers()) )) raise InvalidCredentials("Proxy Authentication Required") -- cgit v1.2.3 From d002371d30e4b0ab7d1d23023236a9446d4c2396 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 7 Sep 2015 13:51:46 +0200 Subject: expose `next_layer` to inline scripts --- libmproxy/protocol/http.py | 1 - libmproxy/protocol/tls.py | 16 ++++++++-------- 2 files changed, 8 insertions(+), 9 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index fbf4ac9b..93972111 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -3,7 +3,6 @@ from __future__ import (absolute_import, print_function, division) from netlib import tcp from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers from netlib.http.semantics import CONTENT_MISSING -from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index 88a8398f..6e8535ae 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -6,8 +6,8 @@ from construct import ConstructError from netlib.tcp import NetLibError, NetLibInvalidCertificateError from netlib.http.http1 import HTTP1Protocol -from ..contrib.tls._constructs import ClientHello, CipherSuites -from ..exceptions import ProtocolException +from ..contrib.tls._constructs import ClientHello +from ..exceptions import ProtocolException, TlsException from .base import Layer @@ -201,6 +201,7 @@ CIPHER_ID_NAME_MAP = { 0x080080: 'RC4-64-MD5', } + def is_tls_record_magic(d): """ Returns: @@ -290,11 +291,11 @@ class TlsLayer(Layer): while len(client_hello) < client_hello_size: record_header = self.client_conn.rfile.peek(offset + 5)[offset:] if not is_tls_record_magic(record_header) or len(record_header) != 5: - raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) + raise TlsException('Expected TLS record, got "%s" instead.' % record_header) record_size = struct.unpack("!H", record_header[3:])[0] + 5 record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] if len(record_body) != record_size - 5: - raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) + raise TlsException("Unexpected EOF in TLS handshake: %s" % record_body) client_hello += record_body offset += record_size client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 @@ -405,7 +406,7 @@ class TlsLayer(Layer): alpn_select_callback=self.__alpn_select_callback, ) except NetLibError as e: - raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) + raise TlsException("Cannot establish TLS with client: %s" % repr(e), e) def _establish_tls_with_server(self): self.log("Establish TLS with server", "debug") @@ -452,13 +453,13 @@ class TlsLayer(Layer): (tls_cert_err['depth'], tls_cert_err['errno']), "error") self.log("Aborting connection attempt", "error") - raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + raise TlsException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( address=repr(self.server_conn.address), sni=self.sni_for_server_connection, e=repr(e), ), e) except NetLibError as e: - raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + raise TlsException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( address=repr(self.server_conn.address), sni=self.sni_for_server_connection, e=repr(e), @@ -487,5 +488,4 @@ class TlsLayer(Layer): if self._sni_from_server_change: sans.add(self._sni_from_server_change) - sans.discard(host) return self.config.certstore.get_cert(host, list(sans)) -- cgit v1.2.3 From 61f4319491ccc9a6c2dff84eaebe628014987148 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 9 Sep 2015 18:49:32 +0200 Subject: http protocol: use new tls attribute --- libmproxy/protocol/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 93972111..dbe91e3c 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -511,7 +511,7 @@ class HttpLayer(Layer): if self.mode == "regular" or self.mode == "transparent": # If there's an existing connection that doesn't match our expectations, kill it. - if address != self.server_conn.address or tls != self.server_conn.ssl_established: + if address != self.server_conn.address or tls != self.server_conn.tls_established: self.set_server(address, tls, address.host) # Establish connection is neccessary. if not self.server_conn: -- cgit v1.2.3 From cf2b2e0cc71b34d851503852544eed3cd5442ca0 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 10 Sep 2015 10:20:11 +0200 Subject: simplify streaming http layer --- libmproxy/protocol/http.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index dbe91e3c..a05f3597 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -45,12 +45,23 @@ class _StreamingHttpLayer(_HttpLayer): raise NotImplementedError() yield "this is a generator" # pragma: no cover + def read_response(self, request_method): + response = self.read_response_headers() + response.body = "".join( + self.read_response_body(response.headers, request_method, response.code) + ) + return response + def send_response_headers(self, response): raise NotImplementedError def send_response_body(self, response, chunks): raise NotImplementedError() + def send_response(self, response): + self.send_response_headers(response) + self.send_response_body(response, response.body) + class Http1Layer(_StreamingHttpLayer): def __init__(self, ctx, mode): @@ -68,17 +79,6 @@ class Http1Layer(_StreamingHttpLayer): def send_request(self, request): self.server_conn.send(self.server_protocol.assemble(request)) - def read_response(self, request_method): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=request_method, - body_size_limit=self.config.body_size_limit, - include_body=True - ) - - def send_response(self, response): - self.client_conn.send(self.client_protocol.assemble(response)) - def read_response_headers(self): return HTTPResponse.from_protocol( self.server_protocol, -- cgit v1.2.3 From 3b6140dfffe5abe5bd3ce48a9371620cbd7ef78a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 10 Sep 2015 10:32:08 +0200 Subject: fix send_response if content is missing --- libmproxy/protocol/http.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index a05f3597..212bec96 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -59,6 +59,8 @@ class _StreamingHttpLayer(_HttpLayer): raise NotImplementedError() def send_response(self, response): + if response.body == CONTENT_MISSING: + raise HttpError(502, "Cannot assemble flow with CONTENT_MISSING") self.send_response_headers(response) self.send_response_body(response, response.body) -- cgit v1.2.3 From ebba79b6558b23cec13faa3562ff05ada5912ca4 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 10 Sep 2015 11:33:03 +0200 Subject: raw tcp mode: use ssl_read_select --- libmproxy/protocol/rawtcp.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/rawtcp.py b/libmproxy/protocol/rawtcp.py index 86468773..6ab0196a 100644 --- a/libmproxy/protocol/rawtcp.py +++ b/libmproxy/protocol/rawtcp.py @@ -4,7 +4,7 @@ import select from OpenSSL import SSL -from netlib.tcp import NetLibError +from netlib.tcp import NetLibError, ssl_read_select from netlib.utils import cleanBin from ..exceptions import ProtocolException from .base import Layer @@ -28,7 +28,7 @@ class RawTCPLayer(Layer): try: while True: - r, _, _ = select.select(conns, [], [], 10) + r = ssl_read_select(conns, 10) for conn in r: dst = server if conn == client else client -- cgit v1.2.3 From 35a99d2faf867dba1285a81a9baba6d1feeb71f9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 10 Sep 2015 16:24:22 +0200 Subject: start reraising exceptions properly --- libmproxy/protocol/http.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 212bec96..70c5095d 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,4 +1,6 @@ from __future__ import (absolute_import, print_function, division) +import six +import sys from netlib import tcp from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers @@ -62,7 +64,7 @@ class _StreamingHttpLayer(_HttpLayer): if response.body == CONTENT_MISSING: raise HttpError(502, "Cannot assemble flow with CONTENT_MISSING") self.send_response_headers(response) - self.send_response_body(response, response.body) + self.send_response_body(response, [response.body]) class Http1Layer(_StreamingHttpLayer): @@ -381,9 +383,9 @@ class HttpLayer(Layer): except NetLibError: pass if isinstance(e, ProtocolException): - raise e + six.reraise(*sys.exc_info()) else: - raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) + six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e), e), sys.exc_info()[2]) finally: flow.live = False -- cgit v1.2.3 From 33c0d3653077c9e6834034ec03a42beeba3ca7d7 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 10 Sep 2015 18:36:50 +0200 Subject: fix exception re-raise --- libmproxy/protocol/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 70c5095d..52164241 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -383,7 +383,7 @@ class HttpLayer(Layer): except NetLibError: pass if isinstance(e, ProtocolException): - six.reraise(*sys.exc_info()) + six.reraise(ProtocolException, e, sys.exc_info()[2]) else: six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e), e), sys.exc_info()[2]) finally: -- cgit v1.2.3 From d1bc966e5b7e2ef822443f3ad28a5f3d40965e75 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 11 Sep 2015 00:00:00 +0200 Subject: polish for release: introduce http2 and rawtcp as command line switches --- libmproxy/protocol/http.py | 10 ++++++++-- libmproxy/protocol/tls.py | 8 ++++++-- 2 files changed, 14 insertions(+), 4 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 52164241..308fa0a0 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -16,7 +16,7 @@ from ..models import ( HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error ) from .base import Layer, Kill - +from .rawtcp import RawTCPLayer class _HttpLayer(Layer): supports_streaming = False @@ -364,7 +364,13 @@ class HttpLayer(Layer): if self.check_close_connection(flow): return - # TODO: Implement HTTP Upgrade + # Handle 101 Switching Protocols + # It may be useful to pass additional args (such as the upgrade header) + # to next_layer in the future + if flow.response.status_code == 101: + layer = self.ctx.next_layer(self) + layer() + return # Upstream Proxy Mode: Handle CONNECT if flow.request.form_in == "authority" and flow.response.code == 200: diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index 6e8535ae..2cddb1dd 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -3,6 +3,8 @@ from __future__ import (absolute_import, print_function, division) import struct from construct import ConstructError +import six +import sys from netlib.tcp import NetLibError, NetLibInvalidCertificateError from netlib.http.http1 import HTTP1Protocol @@ -387,7 +389,7 @@ class TlsLayer(Layer): self._establish_tls_with_client() except: pass - raise e + six.reraise(*sys.exc_info()) self._establish_tls_with_client() @@ -416,9 +418,11 @@ class TlsLayer(Layer): # and mitmproxy would enter TCP passthrough mode, which we want to avoid. deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") if self.client_alpn_protocols: - alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + alpn = [x for x in self.client_alpn_protocols if not deprecated_http2_variant(x)] else: alpn = None + if alpn and "h2" in alpn and not self.config.http2 : + alpn.remove("h2") ciphers_server = self.config.ciphers_server if not ciphers_server: -- cgit v1.2.3 From c159c8ca13afa6a909f456e41c1a3f57b98baf8a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 11 Sep 2015 01:18:17 +0200 Subject: fix chunked encoding --- libmproxy/protocol/http.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 308fa0a0..636b72f4 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,7 +1,9 @@ from __future__ import (absolute_import, print_function, division) -import six +import itertools import sys +import six + from netlib import tcp from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers from netlib.http.semantics import CONTENT_MISSING @@ -9,14 +11,13 @@ from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol from netlib.http.http2.frame import GoAwayFrame, PriorityFrame, WindowUpdateFrame - from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException from ..models import ( HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error ) from .base import Layer, Kill -from .rawtcp import RawTCPLayer + class _HttpLayer(Layer): supports_streaming = False @@ -108,16 +109,21 @@ class Http1Layer(_StreamingHttpLayer): response, preserve_transfer_encoding=True ) - self.client_conn.send(h + "\r\n") + self.client_conn.wfile.write(h + "\r\n") + self.client_conn.wfile.flush() def send_response_body(self, response, chunks): if self.client_protocol.has_chunked_encoding(response.headers): - chunks = ( - "%d\r\n%s\r\n" % (len(chunk), chunk) - for chunk in chunks + chunks = itertools.chain( + ( + "{:x}\r\n{}\r\n".format(len(chunk), chunk) + for chunk in chunks if chunk + ), + ("0\r\n\r\n",) ) for chunk in chunks: - self.client_conn.send(chunk) + self.client_conn.wfile.write(chunk) + self.client_conn.wfile.flush() def check_close_connection(self, flow): close_connection = ( -- cgit v1.2.3 From ffdf143be42490f05cb2b69cdb83e74264d6070a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 11 Sep 2015 01:39:33 +0200 Subject: better exception handling --- libmproxy/protocol/base.py | 10 ++++++++-- libmproxy/protocol/rawtcp.py | 8 +++++++- libmproxy/protocol/tls.py | 44 +++++++++++++++++++++++++++++++------------- 3 files changed, 46 insertions(+), 16 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 9d8c8bfe..6793d3df 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -1,4 +1,6 @@ from __future__ import (absolute_import, print_function, division) +import six +import sys from netlib import tcp from ..models import ServerConnection from ..exceptions import ProtocolException @@ -172,8 +174,12 @@ class ServerConnectionMixin(object): try: self.server_conn.connect() except tcp.NetLibError as e: - raise ProtocolException( - "Server connection to %s failed: %s" % (repr(self.server_conn.address), e), e) + six.reraise( + ProtocolException, + ProtocolException("Server connection to %s failed: %s" % + (repr(self.server_conn.address), e), e), + sys.exc_info()[2] + ) class Kill(Exception): diff --git a/libmproxy/protocol/rawtcp.py b/libmproxy/protocol/rawtcp.py index 6ab0196a..8a597a15 100644 --- a/libmproxy/protocol/rawtcp.py +++ b/libmproxy/protocol/rawtcp.py @@ -1,6 +1,8 @@ from __future__ import (absolute_import, print_function, division) import socket import select +import six +import sys from OpenSSL import SSL @@ -63,4 +65,8 @@ class RawTCPLayer(Layer): ) except (socket.error, NetLibError, SSL.Error) as e: - raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) + six.reraise( + ProtocolException, + ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e), + sys.exc_info()[2] + ) diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index 2cddb1dd..6f6c2c78 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -1,18 +1,19 @@ from __future__ import (absolute_import, print_function, division) import struct +import sys from construct import ConstructError import six -import sys from netlib.tcp import NetLibError, NetLibInvalidCertificateError from netlib.http.http1 import HTTP1Protocol from ..contrib.tls._constructs import ClientHello -from ..exceptions import ProtocolException, TlsException +from ..exceptions import ProtocolException, TlsException, ClientHandshakeException from .base import Layer + # taken from https://testssl.sh/openssl-rfc.mappping.html CIPHER_ID_NAME_MAP = { 0x00: 'NULL-MD5', @@ -407,8 +408,17 @@ class TlsLayer(Layer): chain_file=chain_file, alpn_select_callback=self.__alpn_select_callback, ) + # Some TLS clients will not fail the handshake, + # but will immediately throw an "unexpected eof" error on the first read. + # The reason for this might be difficult to find, so we try to peek here to see if it + # raises ann error. + self.client_conn.rfile.peek(0) except NetLibError as e: - raise TlsException("Cannot establish TLS with client: %s" % repr(e), e) + six.reraise( + ClientHandshakeException, + ClientHandshakeException("Cannot establish TLS with client: %s" % repr(e), e), + sys.exc_info()[2] + ) def _establish_tls_with_server(self): self.log("Establish TLS with server", "debug") @@ -457,17 +467,25 @@ class TlsLayer(Layer): (tls_cert_err['depth'], tls_cert_err['errno']), "error") self.log("Aborting connection attempt", "error") - raise TlsException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( - address=repr(self.server_conn.address), - sni=self.sni_for_server_connection, - e=repr(e), - ), e) + six.reraise( + TlsException, + TlsException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + address=repr(self.server_conn.address), + sni=self.sni_for_server_connection, + e=repr(e), + ), e), + sys.exc_info()[2] + ) except NetLibError as e: - raise TlsException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( - address=repr(self.server_conn.address), - sni=self.sni_for_server_connection, - e=repr(e), - ), e) + six.reraise( + TlsException, + TlsException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + address=repr(self.server_conn.address), + sni=self.sni_for_server_connection, + e=repr(e), + ), e), + sys.exc_info()[2] + ) self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") -- cgit v1.2.3 From 40ce06e780aeb90a239c7af8b5231214d5d9f9d9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 11 Sep 2015 01:47:13 +0200 Subject: tls layer: fix early peek --- libmproxy/protocol/tls.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index 6f6c2c78..1164681c 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -412,7 +412,7 @@ class TlsLayer(Layer): # but will immediately throw an "unexpected eof" error on the first read. # The reason for this might be difficult to find, so we try to peek here to see if it # raises ann error. - self.client_conn.rfile.peek(0) + self.client_conn.rfile.peek(1) except NetLibError as e: six.reraise( ClientHandshakeException, -- cgit v1.2.3 From dd414e485212e3cab612a66d5d858c1a766ace04 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 11 Sep 2015 02:17:04 +0200 Subject: better error messages, remove error cause --- libmproxy/protocol/base.py | 11 ++++++++--- libmproxy/protocol/http.py | 2 +- libmproxy/protocol/rawtcp.py | 2 +- libmproxy/protocol/tls.py | 11 ++++++++--- 4 files changed, 18 insertions(+), 8 deletions(-) (limited to 'libmproxy/protocol') diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 6793d3df..b92aeea1 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -1,6 +1,8 @@ from __future__ import (absolute_import, print_function, division) -import six import sys + +import six + from netlib import tcp from ..models import ServerConnection from ..exceptions import ProtocolException @@ -176,8 +178,11 @@ class ServerConnectionMixin(object): except tcp.NetLibError as e: six.reraise( ProtocolException, - ProtocolException("Server connection to %s failed: %s" % - (repr(self.server_conn.address), e), e), + ProtocolException( + "Server connection to {} failed: {}".format( + repr(self.server_conn.address), str(e) + ) + ), sys.exc_info()[2] ) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 636b72f4..3a415320 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -397,7 +397,7 @@ class HttpLayer(Layer): if isinstance(e, ProtocolException): six.reraise(ProtocolException, e, sys.exc_info()[2]) else: - six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e), e), sys.exc_info()[2]) + six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) finally: flow.live = False diff --git a/libmproxy/protocol/rawtcp.py b/libmproxy/protocol/rawtcp.py index 8a597a15..9b155412 100644 --- a/libmproxy/protocol/rawtcp.py +++ b/libmproxy/protocol/rawtcp.py @@ -67,6 +67,6 @@ class RawTCPLayer(Layer): except (socket.error, NetLibError, SSL.Error) as e: six.reraise( ProtocolException, - ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e), + ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e))), sys.exc_info()[2] ) diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index 1164681c..4f7c9300 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -416,7 +416,12 @@ class TlsLayer(Layer): except NetLibError as e: six.reraise( ClientHandshakeException, - ClientHandshakeException("Cannot establish TLS with client: %s" % repr(e), e), + ClientHandshakeException( + "Cannot establish TLS with client (sni: {sni}): {e}".format( + sni=self.client_sni, e=repr(e) + ), + self.client_sni or repr(self.server_conn.address) + ), sys.exc_info()[2] ) @@ -473,7 +478,7 @@ class TlsLayer(Layer): address=repr(self.server_conn.address), sni=self.sni_for_server_connection, e=repr(e), - ), e), + )), sys.exc_info()[2] ) except NetLibError as e: @@ -483,7 +488,7 @@ class TlsLayer(Layer): address=repr(self.server_conn.address), sni=self.sni_for_server_connection, e=repr(e), - ), e), + )), sys.exc_info()[2] ) -- cgit v1.2.3