diff options
| -rw-r--r-- | libmproxy/protocol2/__init__.py | 3 | ||||
| -rw-r--r-- | libmproxy/protocol2/http.py | 144 | ||||
| -rw-r--r-- | libmproxy/protocol2/http_protocol_mock.py | 40 | ||||
| -rw-r--r-- | libmproxy/protocol2/layer.py | 5 | ||||
| -rw-r--r-- | libmproxy/protocol2/messages.py | 6 | ||||
| -rw-r--r-- | libmproxy/proxy/server.py | 4 |
6 files changed, 187 insertions, 15 deletions
diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py index d5dafaae..cf6032da 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/protocol2/__init__.py @@ -4,7 +4,8 @@ from .socks_proxy import Socks5Proxy from .reverse_proxy import ReverseProxy from .http_proxy import HttpProxy, HttpUpstreamProxy from .rawtcp import RawTcpLayer +from . import messages __all__ = [ - "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy" + "Socks5Proxy", "RawTcpLayer", "RootContext", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "messages" ] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py index 44ebf6a8..1e774648 100644 --- a/libmproxy/protocol2/http.py +++ b/libmproxy/protocol2/http.py @@ -3,7 +3,8 @@ from __future__ import (absolute_import, print_function, division) from .. import version from ..exceptions import InvalidCredentials, HttpException, ProtocolException from .layer import Layer, ServerConnectionMixin -from .messages import ChangeServer, Connect, Reconnect +from libmproxy import utils +from .messages import ChangeServer, Connect, Reconnect, Kill from .http_proxy import HttpProxy, HttpUpstreamProxy from libmproxy.protocol import KILL @@ -12,7 +13,8 @@ from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest from libmproxy.protocol2.http_protocol_mock import HTTP1 from libmproxy.protocol2.tls import TlsLayer from netlib import tcp -from netlib.http import status_codes +from netlib.http import status_codes, http1 +from netlib.http.semantics import CONTENT_MISSING from netlib import odict @@ -42,12 +44,14 @@ def make_error_response(status_code, message, headers=None): body, ) + def make_connect_request(address): return HTTPRequest( - "authority", "CONNECT", None, address.host, address.port, None, (1,1), + "authority", "CONNECT", None, address.host, address.port, None, (1, 1), odict.ODictCaseless(), "" ) + def make_connect_response(httpversion): headers = odict.ODictCaseless([ ["Content-Length", "0"], @@ -82,14 +86,14 @@ class HttpLayer(Layer, ServerConnectionMixin): try: request = HTTP1.read_request( self.client_conn, - body_size_limit=self.c.config.body_size_limit + body_size_limit=self.config.body_size_limit ) except tcp.NetLibError: # don't throw an error for disconnects that happen # before/between requests. return - self.c.log("request", "debug", [repr(request)]) + self.log("request", "debug", [repr(request)]) # Handle Proxy Authentication self.authenticate(request) @@ -109,12 +113,128 @@ class HttpLayer(Layer, ServerConnectionMixin): flow = HTTPFlow(self.client_conn, self.server_conn) flow.request = request - if not self.process_request_hook(flow): - self.log("Connection killed", "info") - return + for message in self.process_request_hook(flow): + yield message if not flow.response: - self.establish_server_connection(flow) + for message in self.establish_server_connection(flow): + yield message + for message in self.get_response_from_server(flow): + yield message + + self.send_response_to_client(flow) + + if self.check_close_connection(flow): + return + + if flow.request.form_in == "authority" and flow.response.code == 200: + raise NotImplementedError("Upstream mode CONNECT not implemented") + + def check_close_connection(self, flow): + """ + Checks if the connection should be closed depending on the HTTP + semantics. Returns True, if so. + """ + + # TODO: add logic for HTTP/2 + + close_connection = ( + http1.HTTP1Protocol.connection_close( + flow.request.httpversion, + flow.request.headers + ) or http1.HTTP1Protocol.connection_close( + flow.response.httpversion, + flow.response.headers + ) or http1.HTTP1Protocol.expected_http_body_size( + flow.response.headers, + False, + flow.request.method, + flow.response.code) == -1 + ) + if flow.request.form_in == "authority" and flow.response.code == 200: + # Workaround for + # https://github.com/mitmproxy/mitmproxy/issues/313: Some + # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 + # and no Content-Length header + + return False + return close_connection + + def send_response_to_client(self, flow): + if not flow.response.stream: + # no streaming: + # we already received the full response from the server and can + # send it to the client straight away. + self.send_to_client(flow.response) + else: + # streaming: + # First send the headers and then transfer the response + # incrementally: + h = HTTP1._assemble_response_first_line(flow.response) + self.send_to_client(h + "\r\n") + h = HTTP1._assemble_response_headers(flow.response, preserve_transfer_encoding=True) + self.send_to_client(h + "\r\n") + + chunks = HTTP1.read_http_body_chunked( + flow.response.headers, + self.config.body_size_limit, + flow.request.method, + flow.response.code, + False, + 4096 + ) + + if callable(flow.response.stream): + chunks = flow.response.stream(chunks) + + for chunk in chunks: + for part in chunk: + self.send_to_client(part) + self.client_conn.wfile.flush() + + flow.response.timestamp_end = utils.timestamp() + + def get_response_from_server(self, flow): + + self.send_to_server(flow.request) + + flow.response = HTTP1.read_response( + self.server_conn.protocol, + flow.request.method, + body_size_limit=self.config.body_size_limit, + include_body=False, + ) + + # call the appropriate script hook - this is an opportunity for an + # inline script to set flow.stream = True + flow = self.channel.ask("responseheaders", flow) + if flow is None or flow == KILL: + yield Kill() + + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = HTTP1.read_http_body( + flow.response.headers, + self.config.body_size_limit, + flow.request.method, + flow.response.code, + False + ) + flow.response.timestamp_end = utils.timestamp() + + # no further manipulation of self.server_conn beyond this point + # we can safely set it as the final attribute value here. + flow.server_conn = self.server_conn + + self.log( + "response", + "debug", + [repr(flow.response)] + ) + response_reply = self.channel.ask("response", flow) + if response_reply is None or response_reply == KILL: + yield Kill() def process_request_hook(self, flow): # Determine .scheme, .host and .port attributes for inline scripts. @@ -133,9 +253,9 @@ class HttpLayer(Layer, ServerConnectionMixin): flow.request.scheme = self.server_conn.tls_established # TODO: Expose ChangeServer functionality to inline scripts somehow? (yield_from_callback?) - request_reply = self.c.channel.ask("request", flow) + request_reply = self.channel.ask("request", flow) if request_reply is None or request_reply == KILL: - return False + yield Kill() if isinstance(request_reply, HTTPResponse): flow.response = request_reply return @@ -181,7 +301,7 @@ class HttpLayer(Layer, ServerConnectionMixin): raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { - "regular": ("absolute",), # an authority request would already be handled. + "regular": ("absolute",), # an authority request would already be handled. "upstream": ("authority", "absolute"), "transparent": ("regular",) } diff --git a/libmproxy/protocol2/http_protocol_mock.py b/libmproxy/protocol2/http_protocol_mock.py index 962a76d6..5fdb9f2b 100644 --- a/libmproxy/protocol2/http_protocol_mock.py +++ b/libmproxy/protocol2/http_protocol_mock.py @@ -10,4 +10,42 @@ class HTTP1(object): """ :type connection: object """ - return HTTP1Protocol(connection).read_request(*args, **kwargs)
\ No newline at end of file + return HTTP1Protocol(connection).read_request(*args, **kwargs) + + @staticmethod + def read_response(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection).read_response(*args, **kwargs) + + @staticmethod + def read_http_body(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection).read_http_body(*args, **kwargs) + + + @staticmethod + def _assemble_response_first_line(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection)._assemble_response_first_line(*args, **kwargs) + + + @staticmethod + def _assemble_response_headers(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection)._assemble_response_headers(*args, **kwargs) + + + @staticmethod + def read_http_body_chunked(connection, *args, **kwargs): + """ + :type connection: object + """ + return HTTP1Protocol(connection).read_http_body_chunked(*args, **kwargs)
\ No newline at end of file diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py index e9f5c667..2775845e 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol2/layer.py @@ -35,7 +35,7 @@ import threading from netlib import tcp from ..proxy import Log from ..proxy.connection import ServerConnection -from .messages import Connect, Reconnect, ChangeServer +from .messages import Connect, Reconnect, ChangeServer, Kill from ..exceptions import ProtocolException @@ -116,6 +116,9 @@ class ServerConnectionMixin(object): return True elif message == ChangeServer: raise NotImplementedError + elif message == Kill: + self._disconnect() + return False @property diff --git a/libmproxy/protocol2/messages.py b/libmproxy/protocol2/messages.py index 3f53fbd4..f6b584a1 100644 --- a/libmproxy/protocol2/messages.py +++ b/libmproxy/protocol2/messages.py @@ -41,3 +41,9 @@ class ChangeServer(_Message): # We can express this neatly as the "nth-server-providing-layer" # ServerConnection could get a `via` attribute. self.depth = depth + + +class Kill(_Message): + """ + Kill a connection. + """
\ No newline at end of file diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 6a7048e0..defcd464 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -84,6 +84,10 @@ class ConnectionHandler2: try: for message in root_layer(): + if message == protocol2.messages.Kill: + self.log("Connection killed", "info") + break + print("Root layer receveived: %s" % message) except ProtocolException as e: self.log(e, "info") |