From c79af6276340decd34730069614d6cac9283a822 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 2 Sep 2015 20:50:50 +0200 Subject: ignore http2 priority frames --- libmproxy/protocol/http.py | 9 ++++++++- 1 file changed, 8 insertions(+), 1 deletion(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 7f57d17c..4be1f762 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -7,7 +7,7 @@ from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol -from netlib.http.http2.frame import WindowUpdateFrame +from netlib.http.http2.frame import PriorityFrame, WindowUpdateFrame from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException @@ -196,6 +196,13 @@ class Http2Layer(_HttpLayer): # Ideally we should keep track of our own flow control window and # stall transmission if the outgoing flow control buffer is full. return + if isinstance(frame, PriorityFrame): + # Clients are sending Priority frames depending on their implementation. + # The RFC does not clearly state when or which priority preferences should be set. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should forward them to the server. + return self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") -- cgit v1.2.3 From 37e6b3c401c21abfc705ed9f3173f9d4b6184169 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 11:09:59 +0200 Subject: http2: improve unexpected frame handling and shutdown --- libmproxy/protocol/http.py | 48 ++++++++++++++++++++++++++++++++-------------- 1 file changed, 34 insertions(+), 14 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 4be1f762..bf9abc90 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -7,7 +7,7 @@ from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol -from netlib.http.http2.frame import PriorityFrame, WindowUpdateFrame +from netlib.http.http2.frame import Frame, GoAwayFrame, PriorityFrame, WindowUpdateFrame from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException @@ -136,9 +136,9 @@ class Http2Layer(_HttpLayer): super(Http2Layer, self).__init__(ctx) self.mode = mode self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_client) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_server) def read_request(self): request = HTTPRequest.from_protocol( @@ -162,25 +162,26 @@ class Http2Layer(_HttpLayer): ) def send_response(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames + # TODO: implement flow control to prevent client buffer filling up + # maintain a send buffer size, and read WindowUpdateFrames from client to increase the send buffer self.client_conn.send(self.client_protocol.assemble(message)) def connect(self): self.ctx.connect() self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_server) self.server_protocol.perform_connection_preface() def reconnect(self): self.ctx.reconnect() self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_server) self.server_protocol.perform_connection_preface() def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_server) self.server_protocol.perform_connection_preface() def __call__(self): @@ -188,7 +189,24 @@ class Http2Layer(_HttpLayer): layer = HttpLayer(self, self.mode) layer() - def handle_unexpected_frame(self, frame): + # terminate the connection + self.client_conn.send(GoAwayFrame().to_bytes()) + + def handle_unexpected_frame_from_client(self, frame): + if isinstance(frame, PriorityFrame): + # Clients are sending Priority frames depending on their implementation. + # The RFC does not clearly state when or which priority preferences should be set. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should forward them to the server. + return + if isinstance(frame, PingFrame): + # respond with pong + self.server_conn.send(PingFrame(flags=frame.Frame.FLAG_ACK, payload=frame.payload).to_bytes()) + return + self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") + + def handle_unexpected_frame_from_server(self, frame): if isinstance(frame, WindowUpdateFrame): # Clients are sending WindowUpdate frames depending on their flow control algorithm. # Since we cannot predict these frames, and we do not need to respond to them, @@ -196,12 +214,14 @@ class Http2Layer(_HttpLayer): # Ideally we should keep track of our own flow control window and # stall transmission if the outgoing flow control buffer is full. return - if isinstance(frame, PriorityFrame): - # Clients are sending Priority frames depending on their implementation. - # The RFC does not clearly state when or which priority preferences should be set. - # Since we cannot predict these frames, and we do not need to respond to them, - # simply accept them, and hide them from the log. - # Ideally we should forward them to the server. + if isinstance(frame, GoAwayFrame): + # Server wants to terminate the connection, + # relay it to the client. + self.client_conn.send(frame.to_bytes()) + return + if isinstance(frame, PingFrame): + # respond with pong + self.client_conn.send(PingFrame(flags=frame.Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") -- cgit v1.2.3 From bde4bdd1d2182ef81db8cea1fd6baab014f96bb8 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 13:40:35 +0200 Subject: http2: fix unhandled frames --- libmproxy/protocol/http.py | 16 ++++++++-------- 1 file changed, 8 insertions(+), 8 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index bf9abc90..5e4656b1 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -7,7 +7,7 @@ from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol -from netlib.http.http2.frame import Frame, GoAwayFrame, PriorityFrame, WindowUpdateFrame +from netlib.http.http2.frame import Frame, PingFrame, GoAwayFrame, PriorityFrame, WindowUpdateFrame from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException @@ -193,6 +193,13 @@ class Http2Layer(_HttpLayer): self.client_conn.send(GoAwayFrame().to_bytes()) def handle_unexpected_frame_from_client(self, frame): + if isinstance(frame, WindowUpdateFrame): + # Clients are sending WindowUpdate frames depending on their flow control algorithm. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should keep track of our own flow control window and + # stall transmission if the outgoing flow control buffer is full. + return if isinstance(frame, PriorityFrame): # Clients are sending Priority frames depending on their implementation. # The RFC does not clearly state when or which priority preferences should be set. @@ -207,13 +214,6 @@ class Http2Layer(_HttpLayer): self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") def handle_unexpected_frame_from_server(self, frame): - if isinstance(frame, WindowUpdateFrame): - # Clients are sending WindowUpdate frames depending on their flow control algorithm. - # Since we cannot predict these frames, and we do not need to respond to them, - # simply accept them, and hide them from the log. - # Ideally we should keep track of our own flow control window and - # stall transmission if the outgoing flow control buffer is full. - return if isinstance(frame, GoAwayFrame): # Server wants to terminate the connection, # relay it to the client. -- cgit v1.2.3 From 29ae2bbf911db16c695eccbef682320f6b15f769 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 13:49:27 +0200 Subject: http2: fix multiple stream per connection fixes #746 --- libmproxy/protocol/http.py | 60 +++++++++++++++++++++++----------------------- 1 file changed, 30 insertions(+), 30 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 5e4656b1..b37ff7cf 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -32,6 +32,9 @@ class _HttpLayer(Layer): def send_response(self, response): raise NotImplementedError() + def check_close_connection(self, flow): + raise NotImplementedError() + class _StreamingHttpLayer(_HttpLayer): supports_streaming = True @@ -113,6 +116,29 @@ class Http1Layer(_StreamingHttpLayer): for chunk in chunks: self.client_conn.send(chunk) + def check_close_connection(self, flow): + close_connection = ( + http1.HTTP1Protocol.connection_close( + flow.request.httpversion, + flow.request.headers + ) or http1.HTTP1Protocol.connection_close( + flow.response.httpversion, + flow.response.headers + ) or http1.HTTP1Protocol.expected_http_body_size( + flow.response.headers, + False, + flow.request.method, + flow.response.code) == -1 + ) + if flow.request.form_in == "authority" and flow.response.code == 200: + # Workaround for + # https://github.com/mitmproxy/mitmproxy/issues/313: Some + # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 + # and no Content-Length header + + return False + return close_connection + def connect(self): self.ctx.connect() self.server_protocol = HTTP1Protocol(self.server_conn) @@ -166,6 +192,10 @@ class Http2Layer(_HttpLayer): # maintain a send buffer size, and read WindowUpdateFrames from client to increase the send buffer self.client_conn.send(self.client_protocol.assemble(message)) + def check_close_connection(self, flow): + # TODO: add a timer to disconnect after a 10 second timeout + return False + def connect(self): self.ctx.connect() self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, @@ -370,36 +400,6 @@ class HttpLayer(Layer): layer = UpstreamConnectLayer(self, connect_request) layer() - def check_close_connection(self, flow): - """ - Checks if the connection should be closed depending on the HTTP - semantics. Returns True, if so. - """ - - # TODO: add logic for HTTP/2 - - close_connection = ( - http1.HTTP1Protocol.connection_close( - flow.request.httpversion, - flow.request.headers - ) or http1.HTTP1Protocol.connection_close( - flow.response.httpversion, - flow.response.headers - ) or http1.HTTP1Protocol.expected_http_body_size( - flow.response.headers, - False, - flow.request.method, - flow.response.code) == -1 - ) - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 - # and no Content-Length header - - return False - return close_connection - def send_response_to_client(self, flow): if not (self.supports_streaming and flow.response.stream): # no streaming: -- cgit v1.2.3 From b4d6f2e12b031ed1fa95b6a029d11dfa9f52d4e9 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 13:52:40 +0200 Subject: http2: fix PingFrame again --- libmproxy/protocol/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index b37ff7cf..b345ee06 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -239,7 +239,7 @@ class Http2Layer(_HttpLayer): return if isinstance(frame, PingFrame): # respond with pong - self.server_conn.send(PingFrame(flags=frame.Frame.FLAG_ACK, payload=frame.payload).to_bytes()) + self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") @@ -251,7 +251,7 @@ class Http2Layer(_HttpLayer): return if isinstance(frame, PingFrame): # respond with pong - self.client_conn.send(PingFrame(flags=frame.Frame.FLAG_ACK, payload=frame.payload).to_bytes()) + self.client_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") -- cgit v1.2.3 From bc93600a66b50d06a7a3a17ee689c5899b61b975 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 13:53:45 +0200 Subject: http2: add GoAway support for client --- libmproxy/protocol/http.py | 5 +++++ 1 file changed, 5 insertions(+) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index b345ee06..222af45f 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -237,6 +237,11 @@ class Http2Layer(_HttpLayer): # simply accept them, and hide them from the log. # Ideally we should forward them to the server. return + if isinstance(frame, GoAwayFrame): + # Client wants to terminate the connection, + # relay it to the server. + self.server_conn.send(frame.to_bytes()) + return if isinstance(frame, PingFrame): # respond with pong self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) -- cgit v1.2.3 From 1f6d05f89fada5fe360aa79abfa80a3c91ce54da Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 14:09:59 +0200 Subject: http2: server can send WindowUpdate frames as well --- libmproxy/protocol/http.py | 11 +++++++++-- 1 file changed, 9 insertions(+), 2 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 222af45f..dbf46973 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -246,9 +246,16 @@ class Http2Layer(_HttpLayer): # respond with pong self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return - self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") + self.log("Unexpected HTTP2 frame from client: %s" % frame.human_readable(), "info") def handle_unexpected_frame_from_server(self, frame): + if isinstance(frame, WindowUpdateFrame): + # Servers are sending WindowUpdate frames depending on their flow control algorithm. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should keep track of our own flow control window and + # stall transmission if the outgoing flow control buffer is full. + return if isinstance(frame, GoAwayFrame): # Server wants to terminate the connection, # relay it to the client. @@ -258,7 +265,7 @@ class Http2Layer(_HttpLayer): # respond with pong self.client_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return - self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") + self.log("Unexpected HTTP2 frame from server: %s" % frame.human_readable(), "info") class ConnectServerConnection(object): -- cgit v1.2.3 From 3a229f60e35a05b0358ef6bf6c81a3a42461c4a2 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 14:26:36 +0200 Subject: http2: fix ping response --- libmproxy/protocol/http.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index dbf46973..cf8d86c3 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -244,7 +244,7 @@ class Http2Layer(_HttpLayer): return if isinstance(frame, PingFrame): # respond with pong - self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) + self.client_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return self.log("Unexpected HTTP2 frame from client: %s" % frame.human_readable(), "info") @@ -263,7 +263,7 @@ class Http2Layer(_HttpLayer): return if isinstance(frame, PingFrame): # respond with pong - self.client_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) + self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) return self.log("Unexpected HTTP2 frame from server: %s" % frame.human_readable(), "info") -- cgit v1.2.3 From f4272de5ec77fb57723e2274e4ddc50d73489e1e Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 3 Sep 2015 17:01:25 +0200 Subject: remove ServerConnectionMixin.reconnect --- libmproxy/protocol/http.py | 25 +++++++------------------ 1 file changed, 7 insertions(+), 18 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index dbf46973..7768b9ad 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -143,10 +143,6 @@ class Http1Layer(_StreamingHttpLayer): self.ctx.connect() self.server_protocol = HTTP1Protocol(self.server_conn) - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP1Protocol(self.server_conn) - def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) self.server_protocol = HTTP1Protocol(self.server_conn) @@ -202,12 +198,6 @@ class Http2Layer(_HttpLayer): unhandled_frame_cb=self.handle_unexpected_frame_from_server) self.server_protocol.perform_connection_preface() - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame_from_server) - self.server_protocol.perform_connection_preface() - def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, @@ -314,14 +304,10 @@ class UpstreamConnectLayer(Layer): else: pass # swallow the message - def reconnect(self): - self.ctx.reconnect() - self._send_connect_request() - def set_server(self, address, server_tls=None, sni=None, depth=1): if depth == 1: if self.ctx.server_conn: - self.ctx.reconnect() + self.ctx.disconnect() address = Address.wrap(address) self.connect_request.host = address.host self.connect_request.port = address.port @@ -459,7 +445,8 @@ class HttpLayer(Layer): # > server detects timeout, disconnects # > read (100-n)% of large request # > send large request upstream - self.reconnect() + self.disconnect() + self.connect() get_response() # call the appropriate script hook - this is an opportunity for an @@ -534,10 +521,12 @@ class HttpLayer(Layer): """ # This is a very ugly (untested) workaround to solve a very ugly problem. if self.server_conn and self.server_conn.tls_established and not ssl: - self.reconnect() + self.disconnect() + self.connect() elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: if self.server_conn.tls_established: - self.reconnect() + self.disconnect() + self.connect() self.send_request(make_connect_request(address)) tls_layer = TlsLayer(self, False, True) -- cgit v1.2.3 From 99126f62ed947847eba4cfa687cb0b0f012092bb Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 3 Sep 2015 18:25:36 +0200 Subject: remove depth attribute from set_server --- libmproxy/protocol/http.py | 32 ++++++++++++++++++++++---------- 1 file changed, 22 insertions(+), 10 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 3c934393..f2265c34 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -304,16 +304,22 @@ class UpstreamConnectLayer(Layer): else: pass # swallow the message - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1: - if self.ctx.server_conn: - self.ctx.disconnect() - address = Address.wrap(address) - self.connect_request.host = address.host - self.connect_request.port = address.port - self.server_conn.address = address - else: - self.ctx.set_server(address, server_tls, sni, depth - 1) + def change_upstream_proxy_server(self, address): + if address != self.server_conn.via.address: + self.ctx.set_server(address) + + def set_server(self, address, server_tls=None, sni=None): + if self.ctx.server_conn: + self.ctx.disconnect() + address = Address.wrap(address) + self.connect_request.host = address.host + self.connect_request.port = address.port + self.server_conn.address = address + + if server_tls: + raise ProtocolException( + "Cannot upgrade to TLS, no TLS layer on the protocol stack." + ) class HttpLayer(Layer): @@ -388,6 +394,12 @@ class HttpLayer(Layer): finally: flow.live = False + def change_upstream_proxy_server(self, address): + # Make set_upstream_proxy_server always available, + # even if there's no UpstreamConnectLayer + if address != self.server_conn.address: + return self.set_server(address) + def handle_regular_mode_connect(self, request): self.set_server((request.host, request.port)) self.send_response(make_connect_response(request.httpversion)) -- cgit v1.2.3 From 47ab7f04eaa85dda7be524b946c22222b2a6de91 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 3 Sep 2015 21:23:19 +0200 Subject: http2: Ping frames are handled in netlib --- libmproxy/protocol/http.py | 8 -------- 1 file changed, 8 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index f2265c34..74c93e16 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -232,10 +232,6 @@ class Http2Layer(_HttpLayer): # relay it to the server. self.server_conn.send(frame.to_bytes()) return - if isinstance(frame, PingFrame): - # respond with pong - self.client_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) - return self.log("Unexpected HTTP2 frame from client: %s" % frame.human_readable(), "info") def handle_unexpected_frame_from_server(self, frame): @@ -251,10 +247,6 @@ class Http2Layer(_HttpLayer): # relay it to the client. self.client_conn.send(frame.to_bytes()) return - if isinstance(frame, PingFrame): - # respond with pong - self.server_conn.send(PingFrame(flags=Frame.FLAG_ACK, payload=frame.payload).to_bytes()) - return self.log("Unexpected HTTP2 frame from server: %s" % frame.human_readable(), "info") -- cgit v1.2.3 From 00561d280ccd4aac06b13b434e0aef4492148cb5 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 4 Sep 2015 02:11:09 +0200 Subject: speed up filters --- libmproxy/protocol/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 74c93e16..f51fea95 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -7,7 +7,7 @@ from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol -from netlib.http.http2.frame import Frame, PingFrame, GoAwayFrame, PriorityFrame, WindowUpdateFrame +from netlib.http.http2.frame import GoAwayFrame, PriorityFrame, WindowUpdateFrame from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException -- cgit v1.2.3 From 5125c669ccd2db5de5f90c66db61e64f63f3ba4c Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sat, 5 Sep 2015 20:45:58 +0200 Subject: adjust to new netlib Headers class --- libmproxy/protocol/http.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index f51fea95..fbf4ac9b 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,7 +1,7 @@ from __future__ import (absolute_import, print_function, division) from netlib import tcp -from netlib.http import http1, HttpErrorConnClosed, HttpError +from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers from netlib.http.semantics import CONTENT_MISSING from netlib import odict from netlib.tcp import NetLibError, Address @@ -568,10 +568,6 @@ class HttpLayer(Layer): self.send_response(make_error_response( 407, "Proxy Authentication Required", - odict.ODictCaseless( - [ - [k, v] for k, v in - self.config.authenticator.auth_challenge_headers().items() - ]) + Headers(**self.config.authenticator.auth_challenge_headers()) )) raise InvalidCredentials("Proxy Authentication Required") -- cgit v1.2.3 From d002371d30e4b0ab7d1d23023236a9446d4c2396 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Mon, 7 Sep 2015 13:51:46 +0200 Subject: expose `next_layer` to inline scripts --- libmproxy/protocol/http.py | 1 - 1 file changed, 1 deletion(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index fbf4ac9b..93972111 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -3,7 +3,6 @@ from __future__ import (absolute_import, print_function, division) from netlib import tcp from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers from netlib.http.semantics import CONTENT_MISSING -from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol -- cgit v1.2.3 From 61f4319491ccc9a6c2dff84eaebe628014987148 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 9 Sep 2015 18:49:32 +0200 Subject: http protocol: use new tls attribute --- libmproxy/protocol/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 93972111..dbe91e3c 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -511,7 +511,7 @@ class HttpLayer(Layer): if self.mode == "regular" or self.mode == "transparent": # If there's an existing connection that doesn't match our expectations, kill it. - if address != self.server_conn.address or tls != self.server_conn.ssl_established: + if address != self.server_conn.address or tls != self.server_conn.tls_established: self.set_server(address, tls, address.host) # Establish connection is neccessary. if not self.server_conn: -- cgit v1.2.3 From cf2b2e0cc71b34d851503852544eed3cd5442ca0 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 10 Sep 2015 10:20:11 +0200 Subject: simplify streaming http layer --- libmproxy/protocol/http.py | 22 +++++++++++----------- 1 file changed, 11 insertions(+), 11 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index dbe91e3c..a05f3597 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -45,12 +45,23 @@ class _StreamingHttpLayer(_HttpLayer): raise NotImplementedError() yield "this is a generator" # pragma: no cover + def read_response(self, request_method): + response = self.read_response_headers() + response.body = "".join( + self.read_response_body(response.headers, request_method, response.code) + ) + return response + def send_response_headers(self, response): raise NotImplementedError def send_response_body(self, response, chunks): raise NotImplementedError() + def send_response(self, response): + self.send_response_headers(response) + self.send_response_body(response, response.body) + class Http1Layer(_StreamingHttpLayer): def __init__(self, ctx, mode): @@ -68,17 +79,6 @@ class Http1Layer(_StreamingHttpLayer): def send_request(self, request): self.server_conn.send(self.server_protocol.assemble(request)) - def read_response(self, request_method): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=request_method, - body_size_limit=self.config.body_size_limit, - include_body=True - ) - - def send_response(self, response): - self.client_conn.send(self.client_protocol.assemble(response)) - def read_response_headers(self): return HTTPResponse.from_protocol( self.server_protocol, -- cgit v1.2.3 From 3b6140dfffe5abe5bd3ce48a9371620cbd7ef78a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 10 Sep 2015 10:32:08 +0200 Subject: fix send_response if content is missing --- libmproxy/protocol/http.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index a05f3597..212bec96 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -59,6 +59,8 @@ class _StreamingHttpLayer(_HttpLayer): raise NotImplementedError() def send_response(self, response): + if response.body == CONTENT_MISSING: + raise HttpError(502, "Cannot assemble flow with CONTENT_MISSING") self.send_response_headers(response) self.send_response_body(response, response.body) -- cgit v1.2.3 From 35a99d2faf867dba1285a81a9baba6d1feeb71f9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 10 Sep 2015 16:24:22 +0200 Subject: start reraising exceptions properly --- libmproxy/protocol/http.py | 8 +++++--- 1 file changed, 5 insertions(+), 3 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 212bec96..70c5095d 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,4 +1,6 @@ from __future__ import (absolute_import, print_function, division) +import six +import sys from netlib import tcp from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers @@ -62,7 +64,7 @@ class _StreamingHttpLayer(_HttpLayer): if response.body == CONTENT_MISSING: raise HttpError(502, "Cannot assemble flow with CONTENT_MISSING") self.send_response_headers(response) - self.send_response_body(response, response.body) + self.send_response_body(response, [response.body]) class Http1Layer(_StreamingHttpLayer): @@ -381,9 +383,9 @@ class HttpLayer(Layer): except NetLibError: pass if isinstance(e, ProtocolException): - raise e + six.reraise(*sys.exc_info()) else: - raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) + six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e), e), sys.exc_info()[2]) finally: flow.live = False -- cgit v1.2.3 From 33c0d3653077c9e6834034ec03a42beeba3ca7d7 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 10 Sep 2015 18:36:50 +0200 Subject: fix exception re-raise --- libmproxy/protocol/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 70c5095d..52164241 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -383,7 +383,7 @@ class HttpLayer(Layer): except NetLibError: pass if isinstance(e, ProtocolException): - six.reraise(*sys.exc_info()) + six.reraise(ProtocolException, e, sys.exc_info()[2]) else: six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e), e), sys.exc_info()[2]) finally: -- cgit v1.2.3 From d1bc966e5b7e2ef822443f3ad28a5f3d40965e75 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 11 Sep 2015 00:00:00 +0200 Subject: polish for release: introduce http2 and rawtcp as command line switches --- libmproxy/protocol/http.py | 10 ++++++++-- 1 file changed, 8 insertions(+), 2 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 52164241..308fa0a0 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -16,7 +16,7 @@ from ..models import ( HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error ) from .base import Layer, Kill - +from .rawtcp import RawTCPLayer class _HttpLayer(Layer): supports_streaming = False @@ -364,7 +364,13 @@ class HttpLayer(Layer): if self.check_close_connection(flow): return - # TODO: Implement HTTP Upgrade + # Handle 101 Switching Protocols + # It may be useful to pass additional args (such as the upgrade header) + # to next_layer in the future + if flow.response.status_code == 101: + layer = self.ctx.next_layer(self) + layer() + return # Upstream Proxy Mode: Handle CONNECT if flow.request.form_in == "authority" and flow.response.code == 200: -- cgit v1.2.3 From c159c8ca13afa6a909f456e41c1a3f57b98baf8a Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 11 Sep 2015 01:18:17 +0200 Subject: fix chunked encoding --- libmproxy/protocol/http.py | 22 ++++++++++++++-------- 1 file changed, 14 insertions(+), 8 deletions(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 308fa0a0..636b72f4 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,7 +1,9 @@ from __future__ import (absolute_import, print_function, division) -import six +import itertools import sys +import six + from netlib import tcp from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers from netlib.http.semantics import CONTENT_MISSING @@ -9,14 +11,13 @@ from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol from netlib.http.http2.frame import GoAwayFrame, PriorityFrame, WindowUpdateFrame - from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException from ..models import ( HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error ) from .base import Layer, Kill -from .rawtcp import RawTCPLayer + class _HttpLayer(Layer): supports_streaming = False @@ -108,16 +109,21 @@ class Http1Layer(_StreamingHttpLayer): response, preserve_transfer_encoding=True ) - self.client_conn.send(h + "\r\n") + self.client_conn.wfile.write(h + "\r\n") + self.client_conn.wfile.flush() def send_response_body(self, response, chunks): if self.client_protocol.has_chunked_encoding(response.headers): - chunks = ( - "%d\r\n%s\r\n" % (len(chunk), chunk) - for chunk in chunks + chunks = itertools.chain( + ( + "{:x}\r\n{}\r\n".format(len(chunk), chunk) + for chunk in chunks if chunk + ), + ("0\r\n\r\n",) ) for chunk in chunks: - self.client_conn.send(chunk) + self.client_conn.wfile.write(chunk) + self.client_conn.wfile.flush() def check_close_connection(self, flow): close_connection = ( -- cgit v1.2.3 From dd414e485212e3cab612a66d5d858c1a766ace04 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Fri, 11 Sep 2015 02:17:04 +0200 Subject: better error messages, remove error cause --- libmproxy/protocol/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libmproxy/protocol/http.py') diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 636b72f4..3a415320 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -397,7 +397,7 @@ class HttpLayer(Layer): if isinstance(e, ProtocolException): six.reraise(ProtocolException, e, sys.exc_info()[2]) else: - six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e), e), sys.exc_info()[2]) + six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) finally: flow.live = False -- cgit v1.2.3