From a86ec56012136664688fa4a8efcd866b5e3e17a8 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Sun, 30 Aug 2015 15:27:29 +0200 Subject: move files around --- libmproxy/console/common.py | 2 +- libmproxy/console/flowview.py | 2 +- libmproxy/filt.py | 2 +- libmproxy/flow.py | 23 +- libmproxy/models/__init__.py | 16 + libmproxy/models/connections.py | 194 ++++++++++ libmproxy/models/flow.py | 166 ++++++++ libmproxy/models/http.py | 554 ++++++++++++++++++++++++++ libmproxy/protocol/__init__.py | 13 +- libmproxy/protocol/base.py | 152 ++++++++ libmproxy/protocol/http.py | 602 +++++++++++++++++++++++++---- libmproxy/protocol/http_replay.py | 95 +++++ libmproxy/protocol/http_wrappers.py | 413 -------------------- libmproxy/protocol/primitives.py | 166 -------- libmproxy/protocol/rawtcp.py | 66 ++++ libmproxy/protocol/tls.py | 288 ++++++++++++++ libmproxy/protocol2/__init__.py | 13 - libmproxy/protocol2/http.py | 588 ---------------------------- libmproxy/protocol2/http_proxy.py | 26 -- libmproxy/protocol2/http_replay.py | 95 ----- libmproxy/protocol2/layer.py | 138 ------- libmproxy/protocol2/rawtcp.py | 66 ---- libmproxy/protocol2/reverse_proxy.py | 17 - libmproxy/protocol2/root_context.py | 95 ----- libmproxy/protocol2/socks_proxy.py | 59 --- libmproxy/protocol2/tls.py | 288 -------------- libmproxy/protocol2/transparent_proxy.py | 24 -- libmproxy/proxy/__init__.py | 8 +- libmproxy/proxy/config.py | 2 +- libmproxy/proxy/connection.py | 193 --------- libmproxy/proxy/modes/__init__.py | 12 + libmproxy/proxy/modes/http_proxy.py | 26 ++ libmproxy/proxy/modes/reverse_proxy.py | 17 + libmproxy/proxy/modes/socks_proxy.py | 60 +++ libmproxy/proxy/modes/transparent_proxy.py | 24 ++ libmproxy/proxy/primitives.py | 15 - libmproxy/proxy/root_context.py | 93 +++++ libmproxy/proxy/server.py | 23 +- 38 files changed, 2329 insertions(+), 2307 deletions(-) create mode 100644 libmproxy/models/__init__.py create mode 100644 libmproxy/models/connections.py create mode 100644 libmproxy/models/flow.py create mode 100644 libmproxy/models/http.py create mode 100644 libmproxy/protocol/base.py create mode 100644 libmproxy/protocol/http_replay.py delete mode 100644 libmproxy/protocol/http_wrappers.py delete mode 100644 libmproxy/protocol/primitives.py create mode 100644 libmproxy/protocol/rawtcp.py create mode 100644 libmproxy/protocol/tls.py delete mode 100644 libmproxy/protocol2/__init__.py delete mode 100644 libmproxy/protocol2/http.py delete mode 100644 libmproxy/protocol2/http_proxy.py delete mode 100644 libmproxy/protocol2/http_replay.py delete mode 100644 libmproxy/protocol2/layer.py delete mode 100644 libmproxy/protocol2/rawtcp.py delete mode 100644 libmproxy/protocol2/reverse_proxy.py delete mode 100644 libmproxy/protocol2/root_context.py delete mode 100644 libmproxy/protocol2/socks_proxy.py delete mode 100644 libmproxy/protocol2/tls.py delete mode 100644 libmproxy/protocol2/transparent_proxy.py delete mode 100644 libmproxy/proxy/connection.py create mode 100644 libmproxy/proxy/modes/__init__.py create mode 100644 libmproxy/proxy/modes/http_proxy.py create mode 100644 libmproxy/proxy/modes/reverse_proxy.py create mode 100644 libmproxy/proxy/modes/socks_proxy.py create mode 100644 libmproxy/proxy/modes/transparent_proxy.py delete mode 100644 libmproxy/proxy/primitives.py create mode 100644 libmproxy/proxy/root_context.py (limited to 'libmproxy') diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 1940e390..c25f7267 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -8,7 +8,7 @@ from netlib.http.semantics import CONTENT_MISSING import netlib.utils from .. import utils -from ..protocol.http import decoded +from ..models import decoded from . import signals diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 1e0f0c17..8b828653 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -9,7 +9,7 @@ from netlib.http.semantics import CONTENT_MISSING from . import common, grideditor, contentview, signals, searchable, tabs from . import flowdetailview from .. import utils, controller -from ..protocol.http import HTTPRequest, HTTPResponse, decoded +from ..models import HTTPRequest, HTTPResponse, decoded class SearchError(Exception): diff --git a/libmproxy/filt.py b/libmproxy/filt.py index 25747bc6..cfd3a1bc 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -35,7 +35,7 @@ from __future__ import absolute_import import re import sys import pyparsing as pp -from .protocol.http import decoded +from .models import decoded class _Token: diff --git a/libmproxy/flow.py b/libmproxy/flow.py index a2f57512..00ec83d2 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,19 +8,18 @@ import Cookie import cookielib import os import re -from libmproxy.protocol.http import HTTPFlow -from libmproxy.protocol2.http_replay import RequestReplayThread +import urlparse + from netlib import odict, wsgi from netlib.http.semantics import CONTENT_MISSING import netlib.http - -from . import controller, protocol, tnetstring, filt, script, version +from . import controller, tnetstring, filt, script, version from .onboarding import app -from .protocol import http from .proxy.config import HostMatcher -from .proxy.connection import ClientConnection, ServerConnection -import urlparse +from .protocol.http_replay import RequestReplayThread +from .protocol import Kill +from .models import ClientConnection, ServerConnection, HTTPResponse, HTTPFlow, HTTPRequest class AppRegistry: @@ -790,7 +789,7 @@ class FlowMaster(controller.Master): rflow = self.server_playback.next_flow(flow) if not rflow: return None - response = http.HTTPResponse.from_state(rflow.response.get_state()) + response = HTTPResponse.from_state(rflow.response.get_state()) response.is_replay = True if self.refresh_server_playback: response.refresh() @@ -836,10 +835,10 @@ class FlowMaster(controller.Master): sni=host, ssl_established=True )) - f = http.HTTPFlow(c, s) + f = HTTPFlow(c, s) headers = odict.ODictCaseless() - req = http.HTTPRequest( + req = HTTPRequest( "absolute", method, scheme, @@ -981,7 +980,7 @@ class FlowMaster(controller.Master): ) if err: self.add_event("Error in wsgi app. %s" % err, "error") - f.reply(protocol.KILL) + f.reply(Kill) return if f not in self.state.flows: # don't add again on replay self.state.add_flow(f) @@ -998,7 +997,7 @@ class FlowMaster(controller.Master): if self.stream_large_bodies: self.stream_large_bodies.run(f, False) except netlib.http.HttpError: - f.reply(protocol.KILL) + f.reply(Kill) return f.reply() diff --git a/libmproxy/models/__init__.py b/libmproxy/models/__init__.py new file mode 100644 index 00000000..3947847c --- /dev/null +++ b/libmproxy/models/__init__.py @@ -0,0 +1,16 @@ +from __future__ import (absolute_import, print_function, division) + +from .http import ( + HTTPFlow, HTTPRequest, HTTPResponse, decoded, + make_error_response, make_connect_request, make_connect_response +) +from .connections import ClientConnection, ServerConnection +from .flow import Flow, Error + +__all__ = [ + "HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded" + "make_error_response", "make_connect_request", + "make_connect_response", + "ClientConnection", "ServerConnection", + "Flow", "Error", +] diff --git a/libmproxy/models/connections.py b/libmproxy/models/connections.py new file mode 100644 index 00000000..98bae3cc --- /dev/null +++ b/libmproxy/models/connections.py @@ -0,0 +1,194 @@ +from __future__ import absolute_import + +import copy +import os + +from netlib import tcp, certutils +from .. import stateobject, utils + + +class ClientConnection(tcp.BaseHandler, stateobject.StateObject): + def __init__(self, client_connection, address, server): + # Eventually, this object is restored from state. We don't have a + # connection then. + if client_connection: + super(ClientConnection, self).__init__(client_connection, address, server) + else: + self.connection = None + self.server = None + self.wfile = None + self.rfile = None + self.address = None + self.clientcert = None + self.ssl_established = None + + self.timestamp_start = utils.timestamp() + self.timestamp_end = None + self.timestamp_ssl_setup = None + self.protocol = None + + def __nonzero__(self): + return bool(self.connection) and not self.finished + + def __repr__(self): + return "".format( + ssl="[ssl] " if self.ssl_established else "", + host=self.address.host, + port=self.address.port + ) + + @property + def tls_established(self): + return self.ssl_established + + _stateobject_attributes = dict( + ssl_established=bool, + timestamp_start=float, + timestamp_end=float, + timestamp_ssl_setup=float + ) + + def get_state(self, short=False): + d = super(ClientConnection, self).get_state(short) + d.update( + address={ + "address": self.address(), + "use_ipv6": self.address.use_ipv6}, + clientcert=self.cert.to_pem() if self.clientcert else None) + return d + + def load_state(self, state): + super(ClientConnection, self).load_state(state) + self.address = tcp.Address( + **state["address"]) if state["address"] else None + self.clientcert = certutils.SSLCert.from_pem( + state["clientcert"]) if state["clientcert"] else None + + def copy(self): + return copy.copy(self) + + def send(self, message): + if isinstance(message, list): + message = b''.join(message) + self.wfile.write(message) + self.wfile.flush() + + @classmethod + def from_state(cls, state): + f = cls(None, tuple(), None) + f.load_state(state) + return f + + def convert_to_ssl(self, *args, **kwargs): + super(ClientConnection, self).convert_to_ssl(*args, **kwargs) + self.timestamp_ssl_setup = utils.timestamp() + + def finish(self): + super(ClientConnection, self).finish() + self.timestamp_end = utils.timestamp() + + +class ServerConnection(tcp.TCPClient, stateobject.StateObject): + def __init__(self, address): + tcp.TCPClient.__init__(self, address) + + self.via = None + self.timestamp_start = None + self.timestamp_end = None + self.timestamp_tcp_setup = None + self.timestamp_ssl_setup = None + self.protocol = None + + def __nonzero__(self): + return bool(self.connection) and not self.finished + + def __repr__(self): + if self.ssl_established and self.sni: + ssl = "[ssl: {0}] ".format(self.sni) + elif self.ssl_established: + ssl = "[ssl] " + else: + ssl = "" + return "".format( + ssl=ssl, + host=self.address.host, + port=self.address.port + ) + + @property + def tls_established(self): + return self.ssl_established + + _stateobject_attributes = dict( + timestamp_start=float, + timestamp_end=float, + timestamp_tcp_setup=float, + timestamp_ssl_setup=float, + address=tcp.Address, + source_address=tcp.Address, + cert=certutils.SSLCert, + ssl_established=bool, + sni=str + ) + _stateobject_long_attributes = {"cert"} + + def get_state(self, short=False): + d = super(ServerConnection, self).get_state(short) + d.update( + address={"address": self.address(), + "use_ipv6": self.address.use_ipv6}, + source_address=({"address": self.source_address(), + "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), + cert=self.cert.to_pem() if self.cert else None + ) + return d + + def load_state(self, state): + super(ServerConnection, self).load_state(state) + + self.address = tcp.Address( + **state["address"]) if state["address"] else None + self.source_address = tcp.Address( + **state["source_address"]) if state["source_address"] else None + self.cert = certutils.SSLCert.from_pem( + state["cert"]) if state["cert"] else None + + @classmethod + def from_state(cls, state): + f = cls(tuple()) + f.load_state(state) + return f + + def copy(self): + return copy.copy(self) + + def connect(self): + self.timestamp_start = utils.timestamp() + tcp.TCPClient.connect(self) + self.timestamp_tcp_setup = utils.timestamp() + + def send(self, message): + if isinstance(message, list): + message = b''.join(message) + self.wfile.write(message) + self.wfile.flush() + + def establish_ssl(self, clientcerts, sni, **kwargs): + clientcert = None + if clientcerts: + path = os.path.join( + clientcerts, + self.address.host.encode("idna")) + ".pem" + if os.path.exists(path): + clientcert = path + + self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) + self.sni = sni + self.timestamp_ssl_setup = utils.timestamp() + + def finish(self): + tcp.TCPClient.finish(self) + self.timestamp_end = utils.timestamp() + + +ServerConnection._stateobject_attributes["via"] = ServerConnection diff --git a/libmproxy/models/flow.py b/libmproxy/models/flow.py new file mode 100644 index 00000000..58287e5b --- /dev/null +++ b/libmproxy/models/flow.py @@ -0,0 +1,166 @@ +from __future__ import absolute_import +import copy +import uuid + +from .. import stateobject, utils, version +from .connections import ClientConnection, ServerConnection + + +class Error(stateobject.StateObject): + """ + An Error. + + This is distinct from an protocol error response (say, a HTTP code 500), + which is represented by a normal HTTPResponse object. This class is + responsible for indicating errors that fall outside of normal protocol + communications, like interrupted connections, timeouts, protocol errors. + + Exposes the following attributes: + + flow: Flow object + msg: Message describing the error + timestamp: Seconds since the epoch + """ + + def __init__(self, msg, timestamp=None): + """ + @type msg: str + @type timestamp: float + """ + self.flow = None # will usually be set by the flow backref mixin + self.msg = msg + self.timestamp = timestamp or utils.timestamp() + + _stateobject_attributes = dict( + msg=str, + timestamp=float + ) + + def __str__(self): + return self.msg + + @classmethod + def from_state(cls, state): + # the default implementation assumes an empty constructor. Override + # accordingly. + f = cls(None) + f.load_state(state) + return f + + def copy(self): + c = copy.copy(self) + return c + + +class Flow(stateobject.StateObject): + """ + A Flow is a collection of objects representing a single transaction. + This class is usually subclassed for each protocol, e.g. HTTPFlow. + """ + + def __init__(self, type, client_conn, server_conn, live=None): + self.type = type + self.id = str(uuid.uuid4()) + self.client_conn = client_conn + """@type: ClientConnection""" + self.server_conn = server_conn + """@type: ServerConnection""" + self.live = live + """@type: LiveConnection""" + + self.error = None + """@type: Error""" + self.intercepted = False + """@type: bool""" + self._backup = None + self.reply = None + + _stateobject_attributes = dict( + id=str, + error=Error, + client_conn=ClientConnection, + server_conn=ServerConnection, + type=str, + intercepted=bool + ) + + def get_state(self, short=False): + d = super(Flow, self).get_state(short) + d.update(version=version.IVERSION) + if self._backup and self._backup != d: + if short: + d.update(modified=True) + else: + d.update(backup=self._backup) + return d + + def __eq__(self, other): + return self is other + + def copy(self): + f = copy.copy(self) + + f.id = str(uuid.uuid4()) + f.live = False + f.client_conn = self.client_conn.copy() + f.server_conn = self.server_conn.copy() + + if self.error: + f.error = self.error.copy() + return f + + def modified(self): + """ + Has this Flow been modified? + """ + if self._backup: + return self._backup != self.get_state() + else: + return False + + def backup(self, force=False): + """ + Save a backup of this Flow, which can be reverted to using a + call to .revert(). + """ + if not self._backup: + self._backup = self.get_state() + + def revert(self): + """ + Revert to the last backed up state. + """ + if self._backup: + self.load_state(self._backup) + self._backup = None + + def kill(self, master): + """ + Kill this request. + """ + from ..protocol import Kill + + self.error = Error("Connection killed") + self.intercepted = False + self.reply(Kill) + master.handle_error(self) + + def intercept(self, master): + """ + Intercept this Flow. Processing will stop until accept_intercept is + called. + """ + if self.intercepted: + return + self.intercepted = True + master.handle_intercept(self) + + def accept_intercept(self, master): + """ + Continue with the flow - called after an intercept(). + """ + if not self.intercepted: + return + self.intercepted = False + self.reply() + master.handle_accept_intercept(self) diff --git a/libmproxy/models/http.py b/libmproxy/models/http.py new file mode 100644 index 00000000..fb2f305b --- /dev/null +++ b/libmproxy/models/http.py @@ -0,0 +1,554 @@ +from __future__ import (absolute_import, print_function, division) +import Cookie +import copy +from email.utils import parsedate_tz, formatdate, mktime_tz +import time + +from libmproxy import utils +from netlib import odict, encoding +from netlib.http import status_codes +from netlib.tcp import Address +from netlib.http.semantics import Request, Response, CONTENT_MISSING +from .. import version, stateobject +from .flow import Flow + + +class MessageMixin(stateobject.StateObject): + _stateobject_attributes = dict( + httpversion=tuple, + headers=odict.ODictCaseless, + body=str, + timestamp_start=float, + timestamp_end=float + ) + _stateobject_long_attributes = {"body"} + + def get_state(self, short=False): + ret = super(MessageMixin, self).get_state(short) + if short: + if self.body: + ret["contentLength"] = len(self.body) + elif self.body == CONTENT_MISSING: + ret["contentLength"] = None + else: + ret["contentLength"] = 0 + return ret + + def get_decoded_content(self): + """ + Returns the decoded content based on the current Content-Encoding + header. + Doesn't change the message iteself or its headers. + """ + ce = self.headers.get_first("content-encoding") + if not self.body or ce not in encoding.ENCODINGS: + return self.body + return encoding.decode(ce, self.body) + + def decode(self): + """ + Decodes body based on the current Content-Encoding header, then + removes the header. If there is no Content-Encoding header, no + action is taken. + + Returns True if decoding succeeded, False otherwise. + """ + ce = self.headers.get_first("content-encoding") + if not self.body or ce not in encoding.ENCODINGS: + return False + data = encoding.decode(ce, self.body) + if data is None: + return False + self.body = data + del self.headers["content-encoding"] + return True + + def encode(self, e): + """ + Encodes body with the encoding e, where e is "gzip", "deflate" + or "identity". + """ + # FIXME: Error if there's an existing encoding header? + self.body = encoding.encode(e, self.body) + self.headers["content-encoding"] = [e] + + def copy(self): + c = copy.copy(self) + c.headers = self.headers.copy() + return c + + def replace(self, pattern, repl, *args, **kwargs): + """ + Replaces a regular expression pattern with repl in both the headers + and the body of the message. Encoded body will be decoded + before replacement, and re-encoded afterwards. + + Returns the number of replacements made. + """ + with decoded(self): + self.body, c = utils.safe_subn( + pattern, repl, self.body, *args, **kwargs + ) + c += self.headers.replace(pattern, repl, *args, **kwargs) + return c + + +class HTTPRequest(MessageMixin, Request): + """ + An HTTP request. + + Exposes the following attributes: + + method: HTTP method + + scheme: URL scheme (http/https) + + host: Target hostname of the request. This is not neccessarily the + directy upstream server (which could be another proxy), but it's always + the target server we want to reach at the end. This attribute is either + inferred from the request itself (absolute-form, authority-form) or from + the connection metadata (e.g. the host in reverse proxy mode). + + port: Destination port + + path: Path portion of the URL (not present in authority-form) + + httpversion: HTTP version tuple, e.g. (1,1) + + headers: odict.ODictCaseless object + + content: Content of the request, None, or CONTENT_MISSING if there + is content associated, but not present. CONTENT_MISSING evaluates + to False to make checking for the presence of content natural. + + form_in: The request form which mitmproxy has received. The following + values are possible: + + - relative (GET /index.html, OPTIONS *) (covers origin form and + asterisk form) + - absolute (GET http://example.com:80/index.html) + - authority-form (CONNECT example.com:443) + Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 + + form_out: The request form which mitmproxy will send out to the + destination + + timestamp_start: Timestamp indicating when request transmission started + + timestamp_end: Timestamp indicating when request transmission ended + """ + + def __init__( + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + body, + timestamp_start=None, + timestamp_end=None, + form_out=None, + ): + Request.__init__( + self, + form_in, + method, + scheme, + host, + port, + path, + httpversion, + headers, + body, + timestamp_start, + timestamp_end, + ) + self.form_out = form_out or form_in + + # Have this request's cookies been modified by sticky cookies or auth? + self.stickycookie = False + self.stickyauth = False + + # Is this request replayed? + self.is_replay = False + + _stateobject_attributes = MessageMixin._stateobject_attributes.copy() + _stateobject_attributes.update( + form_in=str, + method=str, + scheme=str, + host=str, + port=int, + path=str, + form_out=str, + is_replay=bool + ) + + @classmethod + def from_state(cls, state): + f = cls( + None, + None, + None, + None, + None, + None, + None, + None, + None, + None, + None) + f.load_state(state) + return f + + @classmethod + def from_protocol( + self, + protocol, + *args, + **kwargs + ): + req = protocol.read_request(*args, **kwargs) + return self.wrap(req) + + @classmethod + def wrap(self, request): + req = HTTPRequest( + form_in=request.form_in, + method=request.method, + scheme=request.scheme, + host=request.host, + port=request.port, + path=request.path, + httpversion=request.httpversion, + headers=request.headers, + body=request.body, + timestamp_start=request.timestamp_start, + timestamp_end=request.timestamp_end, + form_out=(request.form_out if hasattr(request, 'form_out') else None), + ) + if hasattr(request, 'stream_id'): + req.stream_id = request.stream_id + return req + + def __hash__(self): + return id(self) + + def replace(self, pattern, repl, *args, **kwargs): + """ + Replaces a regular expression pattern with repl in the headers, the + request path and the body of the request. Encoded content will be + decoded before replacement, and re-encoded afterwards. + + Returns the number of replacements made. + """ + c = MessageMixin.replace(self, pattern, repl, *args, **kwargs) + self.path, pc = utils.safe_subn( + pattern, repl, self.path, *args, **kwargs + ) + c += pc + return c + + +class HTTPResponse(MessageMixin, Response): + """ + An HTTP response. + + Exposes the following attributes: + + httpversion: HTTP version tuple, e.g. (1, 0), (1, 1), or (2, 0) + + status_code: HTTP response status code + + msg: HTTP response message + + headers: ODict Caseless object + + content: Content of the request, None, or CONTENT_MISSING if there + is content associated, but not present. CONTENT_MISSING evaluates + to False to make checking for the presence of content natural. + + timestamp_start: Timestamp indicating when request transmission started + + timestamp_end: Timestamp indicating when request transmission ended + """ + + def __init__( + self, + httpversion, + status_code, + msg, + headers, + body, + timestamp_start=None, + timestamp_end=None, + ): + Response.__init__( + self, + httpversion, + status_code, + msg, + headers, + body, + timestamp_start=timestamp_start, + timestamp_end=timestamp_end, + ) + + # Is this request replayed? + self.is_replay = False + self.stream = False + + _stateobject_attributes = MessageMixin._stateobject_attributes.copy() + _stateobject_attributes.update( + status_code=int, + msg=str + ) + + @classmethod + def from_state(cls, state): + f = cls(None, None, None, None, None) + f.load_state(state) + return f + + @classmethod + def from_protocol( + self, + protocol, + *args, + **kwargs + ): + resp = protocol.read_response(*args, **kwargs) + return self.wrap(resp) + + @classmethod + def wrap(self, response): + resp = HTTPResponse( + httpversion=response.httpversion, + status_code=response.status_code, + msg=response.msg, + headers=response.headers, + body=response.body, + timestamp_start=response.timestamp_start, + timestamp_end=response.timestamp_end, + ) + if hasattr(response, 'stream_id'): + resp.stream_id = response.stream_id + return resp + + def _refresh_cookie(self, c, delta): + """ + Takes a cookie string c and a time delta in seconds, and returns + a refreshed cookie string. + """ + c = Cookie.SimpleCookie(str(c)) + for i in c.values(): + if "expires" in i: + d = parsedate_tz(i["expires"]) + if d: + d = mktime_tz(d) + delta + i["expires"] = formatdate(d) + else: + # This can happen when the expires tag is invalid. + # reddit.com sends a an expires tag like this: "Thu, 31 Dec + # 2037 23:59:59 GMT", which is valid RFC 1123, but not + # strictly correct according to the cookie spec. Browsers + # appear to parse this tolerantly - maybe we should too. + # For now, we just ignore this. + del i["expires"] + return c.output(header="").strip() + + def refresh(self, now=None): + """ + This fairly complex and heuristic function refreshes a server + response for replay. + + - It adjusts date, expires and last-modified headers. + - It adjusts cookie expiration. + """ + if not now: + now = time.time() + delta = now - self.timestamp_start + refresh_headers = [ + "date", + "expires", + "last-modified", + ] + for i in refresh_headers: + if i in self.headers: + d = parsedate_tz(self.headers[i][0]) + if d: + new = mktime_tz(d) + delta + self.headers[i] = [formatdate(new)] + c = [] + for i in self.headers["set-cookie"]: + c.append(self._refresh_cookie(i, delta)) + if c: + self.headers["set-cookie"] = c + + +class HTTPFlow(Flow): + """ + A HTTPFlow is a collection of objects representing a single HTTP + transaction. The main attributes are: + + request: HTTPRequest object + response: HTTPResponse object + error: Error object + server_conn: ServerConnection object + client_conn: ClientConnection object + + Note that it's possible for a Flow to have both a response and an error + object. This might happen, for instance, when a response was received + from the server, but there was an error sending it back to the client. + + The following additional attributes are exposed: + + intercepted: Is this flow currently being intercepted? + live: Does this flow have a live client connection? + """ + + def __init__(self, client_conn, server_conn, live=None): + super(HTTPFlow, self).__init__("http", client_conn, server_conn, live) + self.request = None + """@type: HTTPRequest""" + self.response = None + """@type: HTTPResponse""" + + _stateobject_attributes = Flow._stateobject_attributes.copy() + _stateobject_attributes.update( + request=HTTPRequest, + response=HTTPResponse + ) + + @classmethod + def from_state(cls, state): + f = cls(None, None) + f.load_state(state) + return f + + def __repr__(self): + s = " + + %d %s + + %s + + """.strip() % (status_code, response, message) + + if not headers: + headers = odict.ODictCaseless() + headers["Server"] = [version.NAMEVERSION] + headers["Connection"] = ["close"] + headers["Content-Length"] = [len(body)] + headers["Content-Type"] = ["text/html"] + + return HTTPResponse( + (1, 1), # FIXME: Should be a string. + status_code, + response, + headers, + body, + ) + + +def make_connect_request(address): + address = Address.wrap(address) + return HTTPRequest( + "authority", "CONNECT", None, address.host, address.port, None, (1, 1), + odict.ODictCaseless(), "" + ) + + +def make_connect_response(httpversion): + headers = odict.ODictCaseless([ + ["Content-Length", "0"], + ["Proxy-Agent", version.NAMEVERSION] + ]) + return HTTPResponse( + httpversion, + 200, + "Connection established", + headers, + "", + ) diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index bbc20dba..c582592b 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -1 +1,12 @@ -from .primitives import * +from __future__ import (absolute_import, print_function, division) +from .base import Layer, ServerConnectionMixin, Log, Kill +from .http import Http1Layer, Http2Layer +from .tls import TlsLayer, is_tls_record_magic +from .rawtcp import RawTCPLayer + +__all__ = [ + "Layer", "ServerConnectionMixin", "Log", "Kill", + "Http1Layer", "Http2Layer", + "TlsLayer", "is_tls_record_magic", + "RawTCPLayer" +] diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py new file mode 100644 index 00000000..d22a71c6 --- /dev/null +++ b/libmproxy/protocol/base.py @@ -0,0 +1,152 @@ +""" +mitmproxy protocol architecture + +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +For example, the following scenarios depict possible settings (lowest layer first): + +Transparent HTTP proxy, no SSL: + TransparentProxy + Http1Layer + HttpLayer + +Regular proxy, CONNECT request with WebSockets over SSL: + HttpProxy + Http1Layer + HttpLayer + SslLayer + WebsocketLayer (or TcpLayer) + +Automated protocol detection by peeking into the buffer: + TransparentProxy + TLSLayer + Http2Layer + HttpLayer + +Communication between layers is done as follows: + - lower layers provide context information to higher layers + - higher layers can call functions provided by lower layers, + which are propagated until they reach a suitable layer. + +Further goals: + - Connections should always be peekable to make automatic protocol detection work. + - Upstream connections should be established as late as possible; + inline scripts shall have a chance to handle everything locally. +""" +from __future__ import (absolute_import, print_function, division) +from netlib import tcp +from ..models import ServerConnection +from ..exceptions import ProtocolException + + +class _LayerCodeCompletion(object): + """ + Dummy class that provides type hinting in PyCharm, which simplifies development a lot. + """ + + def __init__(self, *args, **kwargs): + super(_LayerCodeCompletion, self).__init__(*args, **kwargs) + if True: + return + self.config = None + """@type: libmproxy.proxy.config.ProxyConfig""" + self.client_conn = None + """@type: libmproxy.proxy.connection.ClientConnection""" + self.channel = None + """@type: libmproxy.controller.Channel""" + + +class Layer(_LayerCodeCompletion): + def __init__(self, ctx, *args, **kwargs): + """ + Args: + ctx: The (read-only) higher layer. + """ + super(Layer, self).__init__(*args, **kwargs) + self.ctx = ctx + + def __call__(self): + """ + Logic of the layer. + Raises: + ProtocolException in case of protocol exceptions. + """ + raise NotImplementedError + + def __getattr__(self, name): + """ + Attributes not present on the current layer may exist on a higher layer. + """ + return getattr(self.ctx, name) + + def log(self, msg, level, subs=()): + full_msg = [ + "{}: {}".format(repr(self.client_conn.address), msg) + ] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) + + @property + def layers(self): + return [self] + self.ctx.layers + + def __repr__(self): + return type(self).__name__ + + +class ServerConnectionMixin(object): + """ + Mixin that provides a layer with the capabilities to manage a server connection. + """ + + def __init__(self, server_address=None): + super(ServerConnectionMixin, self).__init__() + self.server_conn = ServerConnection(server_address) + + def reconnect(self): + address = self.server_conn.address + self._disconnect() + self.server_conn.address = address + self.connect() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + if depth == 1: + if self.server_conn: + self._disconnect() + self.log("Set new server address: " + repr(address), "debug") + self.server_conn.address = address + else: + self.ctx.set_server(address, server_tls, sni, depth - 1) + + def _disconnect(self): + """ + Deletes (and closes) an existing server connection. + """ + self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + self.server_conn.finish() + self.server_conn.close() + # self.channel.tell("serverdisconnect", self) + self.server_conn = ServerConnection(None) + + def connect(self): + if not self.server_conn.address: + raise ProtocolException("Cannot connect to server, no server address given.") + self.log("serverconnect", "debug", [repr(self.server_conn.address)]) + try: + self.server_conn.connect() + except tcp.NetLibError as e: + raise ProtocolException( + "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) + + +class Log(object): + def __init__(self, msg, level="info"): + self.msg = msg + self.level = level + + +class Kill(Exception): + """ + Kill a connection. + """ diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index bde7b088..fc57f6df 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,92 +1,538 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) -from .primitives import Flow +from netlib import tcp +from netlib.http import http1, HttpErrorConnClosed, HttpError +from netlib.http.semantics import CONTENT_MISSING +from netlib import odict +from netlib.tcp import NetLibError, Address +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol -from .http_wrappers import decoded, HTTPRequest, HTTPResponse +from .. import utils +from ..exceptions import InvalidCredentials, HttpException, ProtocolException +from ..models import ( + HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error +) +from .base import Layer, Kill -class HTTPFlow(Flow): - """ - A HTTPFlow is a collection of objects representing a single HTTP - transaction. The main attributes are: - request: HTTPRequest object - response: HTTPResponse object - error: Error object - server_conn: ServerConnection object - client_conn: ClientConnection object +class _HttpLayer(Layer): + supports_streaming = False + + def read_request(self): + raise NotImplementedError() + + def send_request(self, request): + raise NotImplementedError() + + def read_response(self, request_method): + raise NotImplementedError() + + def send_response(self, response): + raise NotImplementedError() + + +class _StreamingHttpLayer(_HttpLayer): + supports_streaming = True + + def read_response_headers(self): + raise NotImplementedError + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + raise NotImplementedError() + yield "this is a generator" + + def send_response_headers(self, response): + raise NotImplementedError + + def send_response_body(self, response, chunks): + raise NotImplementedError() + + +class Http1Layer(_StreamingHttpLayer): + def __init__(self, ctx, mode): + super(Http1Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP1Protocol(self.client_conn) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def read_request(self): + return HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + + def send_request(self, request): + self.server_conn.send(self.server_protocol.assemble(request)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True + ) + + def send_response(self, response): + self.client_conn.send(self.client_protocol.assemble(response)) + + def read_response_headers(self): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=None, # does not matter if we don't read the body. + body_size_limit=self.config.body_size_limit, + include_body=False + ) + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + return self.server_protocol.read_http_body_chunked( + headers, + self.config.body_size_limit, + request_method, + response_code, + False, + max_chunk_size + ) + + def send_response_headers(self, response): + h = self.client_protocol._assemble_response_first_line(response) + self.client_conn.wfile.write(h + "\r\n") + h = self.client_protocol._assemble_response_headers( + response, + preserve_transfer_encoding=True + ) + self.client_conn.send(h + "\r\n") + + def send_response_body(self, response, chunks): + if self.client_protocol.has_chunked_encoding(response.headers): + chunks = ( + "%d\r\n%s\r\n" % (len(chunk), chunk) + for chunk in chunks + ) + for chunk in chunks: + self.client_conn.send(chunk) - Note that it's possible for a Flow to have both a response and an error - object. This might happen, for instance, when a response was received - from the server, but there was an error sending it back to the client. + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP1Protocol(self.server_conn) - The following additional attributes are exposed: + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP1Protocol(self.server_conn) - intercepted: Is this flow currently being intercepted? - live: Does this flow have a live client connection? + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def __call__(self): + layer = HttpLayer(self, self.mode) + layer() + + +# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. +class Http2Layer(_HttpLayer): + def __init__(self, ctx, mode): + super(Http2Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + + def read_request(self): + request = HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + self._stream_id = request.stream_id + return request + + def send_request(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.server_conn.send(self.server_protocol.assemble(message)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True, + stream_id=self._stream_id + ) + + def send_response(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.client_conn.send(self.client_protocol.assemble(message)) + + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def __call__(self): + self.server_protocol.perform_connection_preface() + layer = HttpLayer(self, self.mode) + layer() + + def handle_unexpected_frame(self, frm): + self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") + + +class ConnectServerConnection(object): + """ + "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. """ - def __init__(self, client_conn, server_conn, live=None): - super(HTTPFlow, self).__init__("http", client_conn, server_conn, live) - self.request = None - """@type: HTTPRequest""" - self.response = None - """@type: HTTPResponse""" - - _stateobject_attributes = Flow._stateobject_attributes.copy() - _stateobject_attributes.update( - request=HTTPRequest, - response=HTTPResponse - ) - - @classmethod - def from_state(cls, state): - f = cls(None, None) - f.load_state(state) - return f - - def __repr__(self): - s = " clientconnect (transparent mode destination known) + # > serverconnect (required for client tls handshake) + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + self.reconnect() + get_response() + + # call the appropriate script hook - this is an opportunity for an + # inline script to set flow.stream = True + flow = self.channel.ask("responseheaders", flow) + if flow is None or flow == Kill: + raise Kill() + + if self.supports_streaming: + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = "".join(self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code + )) + flow.response.timestamp_end = utils.timestamp() + + # no further manipulation of self.server_conn beyond this point + # we can safely set it as the final attribute value here. + flow.server_conn = self.server_conn + + self.log( + "response", + "debug", + [repr(flow.response)] + ) + response_reply = self.channel.ask("response", flow) + if response_reply is None or response_reply == Kill: + raise Kill() + + def process_request_hook(self, flow): + # Determine .scheme, .host and .port attributes for inline scripts. + # For absolute-form requests, they are directly given in the request. + # For authority-form requests, we only need to determine the request scheme. + # For relative-form requests, we need to determine host and port as + # well. + if self.mode == "regular": + pass # only absolute-form at this point, nothing to do here. + elif self.mode == "upstream": + if flow.request.form_in == "authority": + flow.request.scheme = "http" # pseudo value + else: + flow.request.host = self.__original_server_conn.address.host + flow.request.port = self.__original_server_conn.address.port + flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" + + request_reply = self.channel.ask("request", flow) + if request_reply is None or request_reply == Kill: + raise Kill() + if isinstance(request_reply, HTTPResponse): + flow.response = request_reply + return + + def establish_server_connection(self, flow): + address = tcp.Address((flow.request.host, flow.request.port)) + tls = (flow.request.scheme == "https") + + if self.mode == "regular" or self.mode == "transparent": + # If there's an existing connection that doesn't match our expectations, kill it. + if address != self.server_conn.address or tls != self.server_conn.ssl_established: + self.set_server(address, tls, address.host) + # Establish connection is neccessary. + if not self.server_conn: + self.connect() + + # SetServer is not guaranteed to work with TLS: + # If there's not TlsLayer below which could catch the exception, + # TLS will not be established. + if tls and not self.server_conn.tls_established: + raise ProtocolException( + "Cannot upgrade to SSL, no TLS layer on the protocol stack.") + else: + if not self.server_conn: + self.connect() + if tls: + raise HttpException("Cannot change scheme in upstream proxy mode.") + """ + # This is a very ugly (untested) workaround to solve a very ugly problem. + if self.server_conn and self.server_conn.tls_established and not ssl: + self.reconnect() + elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: + if self.server_conn.tls_established: + self.reconnect() + + self.send_request(make_connect_request(address)) + tls_layer = TlsLayer(self, False, True) + tls_layer._establish_tls_with_server() + """ + + def validate_request(self, request): + if request.form_in == "absolute" and request.scheme != "http": + self.send_response( + make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + raise HttpException("Invalid request scheme: %s" % request.scheme) + + expected_request_forms = { + "regular": ("absolute",), # an authority request would already be handled. + "upstream": ("authority", "absolute"), + "transparent": ("relative",) + } + + allowed_request_forms = expected_request_forms[self.mode] + if request.form_in not in allowed_request_forms: + err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( + " or ".join(allowed_request_forms), request.form_in + ) + self.send_response(make_error_response(400, err_message)) + raise HttpException(err_message) + + if self.mode == "regular": + request.form_out = "relative" + + def authenticate(self, request): + if self.config.authenticator: + if self.config.authenticator.authenticate(request.headers): + self.config.authenticator.clean(request.headers) + else: + self.send_response(make_error_response( + 407, + "Proxy Authentication Required", + odict.ODictCaseless( + [ + [k, v] for k, v in + self.config.authenticator.auth_challenge_headers().items() + ]) + )) + raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py new file mode 100644 index 00000000..e0144c93 --- /dev/null +++ b/libmproxy/protocol/http_replay.py @@ -0,0 +1,95 @@ +import threading + +from netlib.http import HttpError +from netlib.http.http1 import HTTP1Protocol +from netlib.tcp import NetLibError +from ..controller import Channel +from ..models import Error, HTTPResponse, ServerConnection, make_connect_request +from .base import Log, Kill + + +# TODO: Doesn't really belong into libmproxy.protocol... + + +class RequestReplayThread(threading.Thread): + name = "RequestReplayThread" + + def __init__(self, config, flow, masterq, should_exit): + """ + masterqueue can be a queue or None, if no scripthooks should be + processed. + """ + self.config, self.flow = config, flow + if masterq: + self.channel = Channel(masterq, should_exit) + else: + self.channel = None + super(RequestReplayThread, self).__init__() + + def run(self): + r = self.flow.request + form_out_backup = r.form_out + try: + self.flow.response = None + + # If we have a channel, run script hooks. + if self.channel: + request_reply = self.channel.ask("request", self.flow) + if request_reply is None or request_reply == Kill: + raise Kill() + elif isinstance(request_reply, HTTPResponse): + self.flow.response = request_reply + + if not self.flow.response: + # In all modes, we directly connect to the server displayed + if self.config.mode == "upstream": + server_address = self.config.upstream_server.address + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + connect_request = make_connect_request((r.host, r.port)) + server.send(protocol.assemble(connect_request)) + resp = protocol.read_response("CONNECT") + if resp.code != 200: + raise HttpError(502, "Upstream server refuses CONNECT request") + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + protocol = HTTP1Protocol(server) + if r.scheme == "https": + server.establish_ssl( + self.config.clientcerts, + sni=self.flow.server_conn.sni + ) + r.form_out = "relative" + + server.send(protocol.assemble(r)) + self.flow.server_conn = server + self.flow.response = HTTPResponse.from_protocol( + protocol, + r.method, + body_size_limit=self.config.body_size_limit, + ) + if self.channel: + response_reply = self.channel.ask("response", self.flow) + if response_reply is None or response_reply == Kill: + raise Kill() + except (HttpError, NetLibError) as v: + self.flow.error = Error(repr(v)) + if self.channel: + self.channel.ask("error", self.flow) + except Kill: + # KillSignal should only be raised if there's a channel in the + # first place. + self.channel.tell("log", Log("Connection killed", "info")) + finally: + r.form_out = form_out_backup diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/protocol/http_wrappers.py deleted file mode 100644 index a26ddbb4..00000000 --- a/libmproxy/protocol/http_wrappers.py +++ /dev/null @@ -1,413 +0,0 @@ -from __future__ import absolute_import -import Cookie -import copy -import time -from email.utils import parsedate_tz, formatdate, mktime_tz - -from netlib import odict, encoding -from netlib.http import semantics, CONTENT_MISSING -from .. import utils, stateobject - - -class decoded(object): - """ - A context manager that decodes a request or response, and then - re-encodes it with the same encoding after execution of the block. - - Example: - with decoded(request): - request.content = request.content.replace("foo", "bar") - """ - - def __init__(self, o): - self.o = o - ce = o.headers.get_first("content-encoding") - if ce in encoding.ENCODINGS: - self.ce = ce - else: - self.ce = None - - def __enter__(self): - if self.ce: - self.o.decode() - - def __exit__(self, type, value, tb): - if self.ce: - self.o.encode(self.ce) - - -class MessageMixin(stateobject.StateObject): - _stateobject_attributes = dict( - httpversion=tuple, - headers=odict.ODictCaseless, - body=str, - timestamp_start=float, - timestamp_end=float - ) - _stateobject_long_attributes = {"body"} - - def get_state(self, short=False): - ret = super(MessageMixin, self).get_state(short) - if short: - if self.body: - ret["contentLength"] = len(self.body) - elif self.body == CONTENT_MISSING: - ret["contentLength"] = None - else: - ret["contentLength"] = 0 - return ret - - def get_decoded_content(self): - """ - Returns the decoded content based on the current Content-Encoding - header. - Doesn't change the message iteself or its headers. - """ - ce = self.headers.get_first("content-encoding") - if not self.body or ce not in encoding.ENCODINGS: - return self.body - return encoding.decode(ce, self.body) - - def decode(self): - """ - Decodes body based on the current Content-Encoding header, then - removes the header. If there is no Content-Encoding header, no - action is taken. - - Returns True if decoding succeeded, False otherwise. - """ - ce = self.headers.get_first("content-encoding") - if not self.body or ce not in encoding.ENCODINGS: - return False - data = encoding.decode(ce, self.body) - if data is None: - return False - self.body = data - del self.headers["content-encoding"] - return True - - def encode(self, e): - """ - Encodes body with the encoding e, where e is "gzip", "deflate" - or "identity". - """ - # FIXME: Error if there's an existing encoding header? - self.body = encoding.encode(e, self.body) - self.headers["content-encoding"] = [e] - - def copy(self): - c = copy.copy(self) - c.headers = self.headers.copy() - return c - - def replace(self, pattern, repl, *args, **kwargs): - """ - Replaces a regular expression pattern with repl in both the headers - and the body of the message. Encoded body will be decoded - before replacement, and re-encoded afterwards. - - Returns the number of replacements made. - """ - with decoded(self): - self.body, c = utils.safe_subn( - pattern, repl, self.body, *args, **kwargs - ) - c += self.headers.replace(pattern, repl, *args, **kwargs) - return c - - -class HTTPRequest(MessageMixin, semantics.Request): - """ - An HTTP request. - - Exposes the following attributes: - - method: HTTP method - - scheme: URL scheme (http/https) - - host: Target hostname of the request. This is not neccessarily the - directy upstream server (which could be another proxy), but it's always - the target server we want to reach at the end. This attribute is either - inferred from the request itself (absolute-form, authority-form) or from - the connection metadata (e.g. the host in reverse proxy mode). - - port: Destination port - - path: Path portion of the URL (not present in authority-form) - - httpversion: HTTP version tuple, e.g. (1,1) - - headers: odict.ODictCaseless object - - content: Content of the request, None, or CONTENT_MISSING if there - is content associated, but not present. CONTENT_MISSING evaluates - to False to make checking for the presence of content natural. - - form_in: The request form which mitmproxy has received. The following - values are possible: - - - relative (GET /index.html, OPTIONS *) (covers origin form and - asterisk form) - - absolute (GET http://example.com:80/index.html) - - authority-form (CONNECT example.com:443) - Details: http://tools.ietf.org/html/draft-ietf-httpbis-p1-messaging-25#section-5.3 - - form_out: The request form which mitmproxy will send out to the - destination - - timestamp_start: Timestamp indicating when request transmission started - - timestamp_end: Timestamp indicating when request transmission ended - """ - - def __init__( - self, - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - body, - timestamp_start=None, - timestamp_end=None, - form_out=None, - ): - semantics.Request.__init__( - self, - form_in, - method, - scheme, - host, - port, - path, - httpversion, - headers, - body, - timestamp_start, - timestamp_end, - ) - self.form_out = form_out or form_in - - # Have this request's cookies been modified by sticky cookies or auth? - self.stickycookie = False - self.stickyauth = False - - # Is this request replayed? - self.is_replay = False - - _stateobject_attributes = MessageMixin._stateobject_attributes.copy() - _stateobject_attributes.update( - form_in=str, - method=str, - scheme=str, - host=str, - port=int, - path=str, - form_out=str, - is_replay=bool - ) - - @classmethod - def from_state(cls, state): - f = cls( - None, - None, - None, - None, - None, - None, - None, - None, - None, - None, - None) - f.load_state(state) - return f - - @classmethod - def from_protocol( - self, - protocol, - *args, - **kwargs - ): - req = protocol.read_request(*args, **kwargs) - return self.wrap(req) - - @classmethod - def wrap(self, request): - req = HTTPRequest( - form_in=request.form_in, - method=request.method, - scheme=request.scheme, - host=request.host, - port=request.port, - path=request.path, - httpversion=request.httpversion, - headers=request.headers, - body=request.body, - timestamp_start=request.timestamp_start, - timestamp_end=request.timestamp_end, - form_out=(request.form_out if hasattr(request, 'form_out') else None), - ) - if hasattr(request, 'stream_id'): - req.stream_id = request.stream_id - return req - - def __hash__(self): - return id(self) - - def replace(self, pattern, repl, *args, **kwargs): - """ - Replaces a regular expression pattern with repl in the headers, the - request path and the body of the request. Encoded content will be - decoded before replacement, and re-encoded afterwards. - - Returns the number of replacements made. - """ - c = MessageMixin.replace(self, pattern, repl, *args, **kwargs) - self.path, pc = utils.safe_subn( - pattern, repl, self.path, *args, **kwargs - ) - c += pc - return c - - -class HTTPResponse(MessageMixin, semantics.Response): - """ - An HTTP response. - - Exposes the following attributes: - - httpversion: HTTP version tuple, e.g. (1, 0), (1, 1), or (2, 0) - - status_code: HTTP response status code - - msg: HTTP response message - - headers: ODict Caseless object - - content: Content of the request, None, or CONTENT_MISSING if there - is content associated, but not present. CONTENT_MISSING evaluates - to False to make checking for the presence of content natural. - - timestamp_start: Timestamp indicating when request transmission started - - timestamp_end: Timestamp indicating when request transmission ended - """ - - def __init__( - self, - httpversion, - status_code, - msg, - headers, - body, - timestamp_start=None, - timestamp_end=None, - ): - semantics.Response.__init__( - self, - httpversion, - status_code, - msg, - headers, - body, - timestamp_start=timestamp_start, - timestamp_end=timestamp_end, - ) - - # Is this request replayed? - self.is_replay = False - self.stream = False - - _stateobject_attributes = MessageMixin._stateobject_attributes.copy() - _stateobject_attributes.update( - status_code=int, - msg=str - ) - - @classmethod - def from_state(cls, state): - f = cls(None, None, None, None, None) - f.load_state(state) - return f - - @classmethod - def from_protocol( - self, - protocol, - *args, - **kwargs - ): - resp = protocol.read_response(*args, **kwargs) - return self.wrap(resp) - - @classmethod - def wrap(self, response): - resp = HTTPResponse( - httpversion=response.httpversion, - status_code=response.status_code, - msg=response.msg, - headers=response.headers, - body=response.body, - timestamp_start=response.timestamp_start, - timestamp_end=response.timestamp_end, - ) - if hasattr(response, 'stream_id'): - resp.stream_id = response.stream_id - return resp - - def _refresh_cookie(self, c, delta): - """ - Takes a cookie string c and a time delta in seconds, and returns - a refreshed cookie string. - """ - c = Cookie.SimpleCookie(str(c)) - for i in c.values(): - if "expires" in i: - d = parsedate_tz(i["expires"]) - if d: - d = mktime_tz(d) + delta - i["expires"] = formatdate(d) - else: - # This can happen when the expires tag is invalid. - # reddit.com sends a an expires tag like this: "Thu, 31 Dec - # 2037 23:59:59 GMT", which is valid RFC 1123, but not - # strictly correct according to the cookie spec. Browsers - # appear to parse this tolerantly - maybe we should too. - # For now, we just ignore this. - del i["expires"] - return c.output(header="").strip() - - def refresh(self, now=None): - """ - This fairly complex and heuristic function refreshes a server - response for replay. - - - It adjusts date, expires and last-modified headers. - - It adjusts cookie expiration. - """ - if not now: - now = time.time() - delta = now - self.timestamp_start - refresh_headers = [ - "date", - "expires", - "last-modified", - ] - for i in refresh_headers: - if i in self.headers: - d = parsedate_tz(self.headers[i][0]) - if d: - new = mktime_tz(d) + delta - self.headers[i] = [formatdate(new)] - c = [] - for i in self.headers["set-cookie"]: - c.append(self._refresh_cookie(i, delta)) - if c: - self.headers["set-cookie"] = c diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py deleted file mode 100644 index c663f0c5..00000000 --- a/libmproxy/protocol/primitives.py +++ /dev/null @@ -1,166 +0,0 @@ -from __future__ import absolute_import -import copy -import uuid - -from .. import stateobject, utils, version -from ..proxy.connection import ClientConnection, ServerConnection - -KILL = 0 # const for killed requests - - -class Error(stateobject.StateObject): - """ - An Error. - - This is distinct from an protocol error response (say, a HTTP code 500), - which is represented by a normal HTTPResponse object. This class is - responsible for indicating errors that fall outside of normal protocol - communications, like interrupted connections, timeouts, protocol errors. - - Exposes the following attributes: - - flow: Flow object - msg: Message describing the error - timestamp: Seconds since the epoch - """ - - def __init__(self, msg, timestamp=None): - """ - @type msg: str - @type timestamp: float - """ - self.flow = None # will usually be set by the flow backref mixin - self.msg = msg - self.timestamp = timestamp or utils.timestamp() - - _stateobject_attributes = dict( - msg=str, - timestamp=float - ) - - def __str__(self): - return self.msg - - @classmethod - def from_state(cls, state): - # the default implementation assumes an empty constructor. Override - # accordingly. - f = cls(None) - f.load_state(state) - return f - - def copy(self): - c = copy.copy(self) - return c - - -class Flow(stateobject.StateObject): - """ - A Flow is a collection of objects representing a single transaction. - This class is usually subclassed for each protocol, e.g. HTTPFlow. - """ - - def __init__(self, type, client_conn, server_conn, live=None): - self.type = type - self.id = str(uuid.uuid4()) - self.client_conn = client_conn - """@type: ClientConnection""" - self.server_conn = server_conn - """@type: ServerConnection""" - self.live = live - """@type: LiveConnection""" - - self.error = None - """@type: Error""" - self.intercepted = False - """@type: bool""" - self._backup = None - self.reply = None - - _stateobject_attributes = dict( - id=str, - error=Error, - client_conn=ClientConnection, - server_conn=ServerConnection, - type=str, - intercepted=bool - ) - - def get_state(self, short=False): - d = super(Flow, self).get_state(short) - d.update(version=version.IVERSION) - if self._backup and self._backup != d: - if short: - d.update(modified=True) - else: - d.update(backup=self._backup) - return d - - def __eq__(self, other): - return self is other - - def copy(self): - f = copy.copy(self) - - f.id = str(uuid.uuid4()) - f.live = False - f.client_conn = self.client_conn.copy() - f.server_conn = self.server_conn.copy() - - if self.error: - f.error = self.error.copy() - return f - - def modified(self): - """ - Has this Flow been modified? - """ - if self._backup: - return self._backup != self.get_state() - else: - return False - - def backup(self, force=False): - """ - Save a backup of this Flow, which can be reverted to using a - call to .revert(). - """ - if not self._backup: - self._backup = self.get_state() - - def revert(self): - """ - Revert to the last backed up state. - """ - if self._backup: - self.load_state(self._backup) - self._backup = None - - def kill(self, master): - """ - Kill this request. - """ - self.error = Error("Connection killed") - self.intercepted = False - self.reply(KILL) - master.handle_error(self) - - def intercept(self, master): - """ - Intercept this Flow. Processing will stop until accept_intercept is - called. - """ - if self.intercepted: - return - self.intercepted = True - master.handle_intercept(self) - - def accept_intercept(self, master): - """ - Continue with the flow - called after an intercept(). - """ - if not self.intercepted: - return - self.intercepted = False - self.reply() - master.handle_accept_intercept(self) diff --git a/libmproxy/protocol/rawtcp.py b/libmproxy/protocol/rawtcp.py new file mode 100644 index 00000000..86468773 --- /dev/null +++ b/libmproxy/protocol/rawtcp.py @@ -0,0 +1,66 @@ +from __future__ import (absolute_import, print_function, division) +import socket +import select + +from OpenSSL import SSL + +from netlib.tcp import NetLibError +from netlib.utils import cleanBin +from ..exceptions import ProtocolException +from .base import Layer + + +class RawTCPLayer(Layer): + chunk_size = 4096 + + def __init__(self, ctx, logging=True): + self.logging = logging + super(RawTCPLayer, self).__init__(ctx) + + def __call__(self): + self.connect() + + buf = memoryview(bytearray(self.chunk_size)) + + client = self.client_conn.connection + server = self.server_conn.connection + conns = [client, server] + + try: + while True: + r, _, _ = select.select(conns, [], [], 10) + for conn in r: + dst = server if conn == client else client + + size = conn.recv_into(buf, self.chunk_size) + if not size: + conns.remove(conn) + # Shutdown connection to the other peer + if isinstance(conn, SSL.Connection): + # We can't half-close a connection, so we just close everything here. + # Sockets will be cleaned up on a higher level. + return + else: + dst.shutdown(socket.SHUT_WR) + + if len(conns) == 0: + return + continue + + dst.sendall(buf[:size]) + + if self.logging: + # log messages are prepended with the client address, + # hence the "weird" direction string. + if dst == server: + direction = "-> tcp -> {}".format(repr(self.server_conn.address)) + else: + direction = "<- tcp <- {}".format(repr(self.server_conn.address)) + data = cleanBin(buf[:size].tobytes()) + self.log( + "{}\r\n{}".format(direction, data), + "info" + ) + + except (socket.error, NetLibError, SSL.Error) as e: + raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py new file mode 100644 index 00000000..b85a6595 --- /dev/null +++ b/libmproxy/protocol/tls.py @@ -0,0 +1,288 @@ +from __future__ import (absolute_import, print_function, division) + +import struct + +from construct import ConstructError + +from netlib.tcp import NetLibError, NetLibInvalidCertificateError +from netlib.http.http1 import HTTP1Protocol +from ..contrib.tls._constructs import ClientHello +from ..exceptions import ProtocolException +from .base import Layer + + +def is_tls_record_magic(d): + """ + Returns: + True, if the passed bytes start with the TLS record magic bytes. + False, otherwise. + """ + d = d[:3] + + # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 + # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello + return ( + len(d) == 3 and + d[0] == '\x16' and + d[1] == '\x03' and + d[2] in ('\x00', '\x01', '\x02', '\x03') + ) + + +class TlsLayer(Layer): + def __init__(self, ctx, client_tls, server_tls): + self.client_sni = None + self.client_alpn_protocols = None + + super(TlsLayer, self).__init__(ctx) + self._client_tls = client_tls + self._server_tls = server_tls + + self._sni_from_server_change = None + + def __call__(self): + """ + The strategy for establishing SSL is as follows: + First, we determine whether we need the server cert to establish ssl with the client. + If so, we first connect to the server and then to the client. + If not, we only connect to the client and do the server_ssl lazily on a Connect message. + + An additional complexity is that establish ssl with the server may require a SNI value from the client. + In an ideal world, we'd do the following: + 1. Start the SSL handshake with the client + 2. Check if the client sends a SNI. + 3. Pause the client handshake, establish SSL with the server. + 4. Finish the client handshake with the certificate from the server. + There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( + Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values. + + Further notes: + - OpenSSL 1.0.2 introduces a callback that would help here: + https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html + - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 + """ + + client_tls_requires_server_cert = ( + self._client_tls and self._server_tls and not self.config.no_upstream_cert + ) + + if self._client_tls: + self._parse_client_hello() + + if client_tls_requires_server_cert: + self._establish_tls_with_client_and_server() + elif self._client_tls: + self._establish_tls_with_client() + + layer = self.ctx.next_layer(self) + layer() + + def __repr__(self): + if self._client_tls and self._server_tls: + return "TlsLayer(client and server)" + elif self._client_tls: + return "TlsLayer(client)" + elif self._server_tls: + return "TlsLayer(server)" + else: + return "TlsLayer(inactive)" + + def _get_client_hello(self): + """ + Peek into the socket and read all records that contain the initial client hello message. + + Returns: + The raw handshake packet bytes, without TLS record header(s). + """ + client_hello = "" + client_hello_size = 1 + offset = 0 + while len(client_hello) < client_hello_size: + record_header = self.client_conn.rfile.peek(offset + 5)[offset:] + if not is_tls_record_magic(record_header) or len(record_header) != 5: + raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) + record_size = struct.unpack("!H", record_header[3:])[0] + 5 + record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] + if len(record_body) != record_size - 5: + raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) + client_hello += record_body + offset += record_size + client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 + return client_hello + + def _parse_client_hello(self): + """ + Peek into the connection, read the initial client hello and parse it to obtain ALPN values. + """ + try: + raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. + except ProtocolException as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + return + + try: + client_hello = ClientHello.parse(raw_client_hello) + except ConstructError as e: + self.log("Cannot parse Client Hello: %s" % repr(e), "error") + self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") + return + + for extension in client_hello.extensions: + if extension.type == 0x00: + if len(extension.server_names) != 1 or extension.server_names[0].type != 0: + self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") + self.client_sni = extension.server_names[0].name + elif extension.type == 0x10: + self.client_alpn_protocols = list(extension.alpn_protocols) + + self.log( + "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), + "debug" + ) + + def connect(self): + if not self.server_conn: + self.ctx.connect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def reconnect(self): + self.ctx.reconnect() + if self._server_tls and not self.server_conn.tls_established: + self._establish_tls_with_server() + + def set_server(self, address, server_tls=None, sni=None, depth=1): + self.ctx.set_server(address, server_tls, sni, depth) + if depth == 1 and server_tls is not None: + self._sni_from_server_change = sni + self._server_tls = server_tls + + @property + def sni_for_server_connection(self): + if self._sni_from_server_change is False: + return None + else: + return self._sni_from_server_change or self.client_sni + + @property + def alpn_for_client_connection(self): + return self.server_conn.get_alpn_proto_negotiated() + + def __alpn_select_callback(self, conn_, options): + """ + Once the client signals the alternate protocols it supports, + we reconnect upstream with the same list and pass the server's choice down to the client. + """ + + # This gets triggered if we haven't established an upstream connection yet. + default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 + # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 + + if self.alpn_for_client_connection in options: + choice = bytes(self.alpn_for_client_connection) + elif default_alpn in options: + choice = bytes(default_alpn) + else: + choice = options[0] + self.log("ALPN for client: %s" % choice, "debug") + return choice + + def _establish_tls_with_client_and_server(self): + self.ctx.connect() + + # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless + # to send an error message over TLS. + try: + self._establish_tls_with_server() + except Exception as e: + try: + self._establish_tls_with_client() + except: + pass + raise e + + self._establish_tls_with_client() + + def _establish_tls_with_client(self): + self.log("Establish TLS with client", "debug") + cert, key, chain_file = self._find_cert() + + try: + self.client_conn.convert_to_ssl( + cert, key, + method=self.config.openssl_method_client, + options=self.config.openssl_options_client, + cipher_list=self.config.ciphers_client, + dhparams=self.config.certstore.dhparams, + chain_file=chain_file, + alpn_select_callback=self.__alpn_select_callback, + ) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) + + def _establish_tls_with_server(self): + self.log("Establish TLS with server", "debug") + try: + # We only support http/1.1 and h2. + # If the server only supports spdy (next to http/1.1), it may select that + # and mitmproxy would enter TCP passthrough mode, which we want to avoid. + deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") + if self.client_alpn_protocols: + alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + else: + alpn = None + + self.server_conn.establish_ssl( + self.config.clientcerts, + self.sni_for_server_connection, + method=self.config.openssl_method_server, + options=self.config.openssl_options_server, + verify_options=self.config.openssl_verification_mode_server, + ca_path=self.config.openssl_trusted_cadir_server, + ca_pemfile=self.config.openssl_trusted_ca_server, + cipher_list=self.config.ciphers_server, + alpn_protos=alpn, + ) + tls_cert_err = self.server_conn.ssl_verification_error + if tls_cert_err is not None: + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Ignoring server verification error, continuing with connection", "error") + except NetLibInvalidCertificateError as e: + tls_cert_err = self.server_conn.ssl_verification_error + self.log( + "TLS verification failed for upstream server at depth %s with error: %s" % + (tls_cert_err['depth'], tls_cert_err['errno']), + "error") + self.log("Aborting connection attempt", "error") + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + except NetLibError as e: + raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) + + self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") + + def _find_cert(self): + host = self.server_conn.address.host + sans = set() + # Incorporate upstream certificate + use_upstream_cert = ( + self.server_conn and + self.server_conn.tls_established and + (not self.config.no_upstream_cert) + ) + if use_upstream_cert: + upstream_cert = self.server_conn.cert + sans.update(upstream_cert.altnames) + if upstream_cert.cn: + sans.add(host) + host = upstream_cert.cn.decode("utf8").encode("idna") + # Also add SNI values. + if self.client_sni: + sans.add(self.client_sni) + if self._sni_from_server_change: + sans.add(self._sni_from_server_change) + + sans.discard(host) + return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/protocol2/__init__.py deleted file mode 100644 index 61b9a77e..00000000 --- a/libmproxy/protocol2/__init__.py +++ /dev/null @@ -1,13 +0,0 @@ -from __future__ import (absolute_import, print_function, division) -from .root_context import RootContext -from .socks_proxy import Socks5Proxy -from .reverse_proxy import ReverseProxy -from .http_proxy import HttpProxy, HttpUpstreamProxy -from .transparent_proxy import TransparentProxy -from .http import make_error_response - -__all__ = [ - "RootContext", - "Socks5Proxy", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "TransparentProxy", - "make_error_response" -] diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py deleted file mode 100644 index a508ae8b..00000000 --- a/libmproxy/protocol2/http.py +++ /dev/null @@ -1,588 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from netlib import tcp -from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError -from netlib.http.semantics import CONTENT_MISSING -from netlib import odict -from netlib.tcp import NetLibError, Address -from netlib.http.http1 import HTTP1Protocol -from netlib.http.http2 import HTTP2Protocol - -from .. import version, utils -from ..exceptions import InvalidCredentials, HttpException, ProtocolException -from .layer import Layer -from ..proxy import Kill -from libmproxy.protocol import KILL, Error -from libmproxy.protocol.http import HTTPFlow -from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest - - -class _HttpLayer(Layer): - supports_streaming = False - - def read_request(self): - raise NotImplementedError() - - def send_request(self, request): - raise NotImplementedError() - - def read_response(self, request_method): - raise NotImplementedError() - - def send_response(self, response): - raise NotImplementedError() - - -class _StreamingHttpLayer(_HttpLayer): - supports_streaming = True - - def read_response_headers(self): - raise NotImplementedError - - def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): - raise NotImplementedError() - yield "this is a generator" - - def send_response_headers(self, response): - raise NotImplementedError - - def send_response_body(self, response, chunks): - raise NotImplementedError() - - -class Http1Layer(_StreamingHttpLayer): - def __init__(self, ctx, mode): - super(Http1Layer, self).__init__(ctx) - self.mode = mode - self.client_protocol = HTTP1Protocol(self.client_conn) - self.server_protocol = HTTP1Protocol(self.server_conn) - - def read_request(self): - return HTTPRequest.from_protocol( - self.client_protocol, - body_size_limit=self.config.body_size_limit - ) - - def send_request(self, request): - self.server_conn.send(self.server_protocol.assemble(request)) - - def read_response(self, request_method): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=request_method, - body_size_limit=self.config.body_size_limit, - include_body=True - ) - - def send_response(self, response): - self.client_conn.send(self.client_protocol.assemble(response)) - - def read_response_headers(self): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=None, # does not matter if we don't read the body. - body_size_limit=self.config.body_size_limit, - include_body=False - ) - - def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): - return self.server_protocol.read_http_body_chunked( - headers, - self.config.body_size_limit, - request_method, - response_code, - False, - max_chunk_size - ) - - def send_response_headers(self, response): - h = self.client_protocol._assemble_response_first_line(response) - self.client_conn.wfile.write(h + "\r\n") - h = self.client_protocol._assemble_response_headers( - response, - preserve_transfer_encoding=True - ) - self.client_conn.send(h + "\r\n") - - def send_response_body(self, response, chunks): - if self.client_protocol.has_chunked_encoding(response.headers): - chunks = ( - "%d\r\n%s\r\n" % (len(chunk), chunk) - for chunk in chunks - ) - for chunk in chunks: - self.client_conn.send(chunk) - - def connect(self): - self.ctx.connect() - self.server_protocol = HTTP1Protocol(self.server_conn) - - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP1Protocol(self.server_conn) - - def set_server(self, *args, **kwargs): - self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP1Protocol(self.server_conn) - - def __call__(self): - layer = HttpLayer(self, self.mode) - layer() - - -# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. -class Http2Layer(_HttpLayer): - def __init__(self, ctx, mode): - super(Http2Layer, self).__init__(ctx) - self.mode = mode - self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - - def read_request(self): - request = HTTPRequest.from_protocol( - self.client_protocol, - body_size_limit=self.config.body_size_limit - ) - self._stream_id = request.stream_id - return request - - def send_request(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames - self.server_conn.send(self.server_protocol.assemble(message)) - - def read_response(self, request_method): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=request_method, - body_size_limit=self.config.body_size_limit, - include_body=True, - stream_id=self._stream_id - ) - - def send_response(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames - self.client_conn.send(self.client_protocol.assemble(message)) - - def connect(self): - self.ctx.connect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def set_server(self, *args, **kwargs): - self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def __call__(self): - self.server_protocol.perform_connection_preface() - layer = HttpLayer(self, self.mode) - layer() - - def handle_unexpected_frame(self, frm): - self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") - - -def make_error_response(status_code, message, headers=None): - response = status_codes.RESPONSES.get(status_code, "Unknown") - body = """ - - - %d %s - - %s - - """.strip() % (status_code, response, message) - - if not headers: - headers = odict.ODictCaseless() - headers["Server"] = [version.NAMEVERSION] - headers["Connection"] = ["close"] - headers["Content-Length"] = [len(body)] - headers["Content-Type"] = ["text/html"] - - return HTTPResponse( - (1, 1), # FIXME: Should be a string. - status_code, - response, - headers, - body, - ) - - -def make_connect_request(address): - address = Address.wrap(address) - return HTTPRequest( - "authority", "CONNECT", None, address.host, address.port, None, (1, 1), - odict.ODictCaseless(), "" - ) - - -def make_connect_response(httpversion): - headers = odict.ODictCaseless([ - ["Content-Length", "0"], - ["Proxy-Agent", version.NAMEVERSION] - ]) - return HTTPResponse( - httpversion, - 200, - "Connection established", - headers, - "", - ) - - -class ConnectServerConnection(object): - """ - "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. - """ - - def __init__(self, address, ctx): - self.address = tcp.Address.wrap(address) - self._ctx = ctx - - @property - def via(self): - return self._ctx.server_conn - - def __getattr__(self, item): - return getattr(self.via, item) - - -class UpstreamConnectLayer(Layer): - def __init__(self, ctx, connect_request): - super(UpstreamConnectLayer, self).__init__(ctx) - self.connect_request = connect_request - self.server_conn = ConnectServerConnection( - (connect_request.host, connect_request.port), - self.ctx - ) - - def __call__(self): - layer = self.ctx.next_layer(self) - layer() - - def connect(self): - if not self.server_conn: - self.ctx.connect() - self.send_request(self.connect_request) - else: - pass # swallow the message - - def reconnect(self): - self.ctx.reconnect() - self.send_request(self.connect_request) - resp = self.read_response("CONNECT") - if resp.code != 200: - raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") - - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1: - if self.ctx.server_conn: - self.ctx.reconnect() - address = Address.wrap(address) - self.connect_request.host = address.host - self.connect_request.port = address.port - self.server_conn.address = address - else: - self.ctx.set_server(address, server_tls, sni, depth - 1) - - -class HttpLayer(Layer): - def __init__(self, ctx, mode): - super(HttpLayer, self).__init__(ctx) - self.mode = mode - self.__original_server_conn = None - "Contains the original destination in transparent mode, which needs to be restored" - "if an inline script modified the target server for a single http request" - - def __call__(self): - if self.mode == "transparent": - self.__original_server_conn = self.server_conn - while True: - try: - flow = HTTPFlow(self.client_conn, self.server_conn, live=self) - - try: - request = self.read_request() - except tcp.NetLibError: - # don't throw an error for disconnects that happen - # before/between requests. - return - - self.log("request", "debug", [repr(request)]) - - # Handle Proxy Authentication - self.authenticate(request) - - # Regular Proxy Mode: Handle CONNECT - if self.mode == "regular" and request.form_in == "authority": - self.handle_regular_mode_connect(request) - return - - # Make sure that the incoming request matches our expectations - self.validate_request(request) - - flow.request = request - self.process_request_hook(flow) - - if not flow.response: - self.establish_server_connection(flow) - self.get_response_from_server(flow) - - self.send_response_to_client(flow) - - if self.check_close_connection(flow): - return - - # TODO: Implement HTTP Upgrade - - # Upstream Proxy Mode: Handle CONNECT - if flow.request.form_in == "authority" and flow.response.code == 200: - self.handle_upstream_mode_connect(flow.request.copy()) - return - - except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: - if flow.request and not flow.response: - flow.error = Error(repr(e)) - self.channel.ask("error", flow) - try: - self.send_response(make_error_response( - getattr(e, "code", 502), - repr(e) - )) - except NetLibError: - pass - if isinstance(e, ProtocolException): - raise e - else: - raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) - finally: - flow.live = False - - def handle_regular_mode_connect(self, request): - self.set_server((request.host, request.port)) - self.send_response(make_connect_response(request.httpversion)) - layer = self.ctx.next_layer(self) - layer() - - def handle_upstream_mode_connect(self, connect_request): - layer = UpstreamConnectLayer(self, connect_request) - layer() - - def check_close_connection(self, flow): - """ - Checks if the connection should be closed depending on the HTTP - semantics. Returns True, if so. - """ - - # TODO: add logic for HTTP/2 - - close_connection = ( - http1.HTTP1Protocol.connection_close( - flow.request.httpversion, - flow.request.headers - ) or http1.HTTP1Protocol.connection_close( - flow.response.httpversion, - flow.response.headers - ) or http1.HTTP1Protocol.expected_http_body_size( - flow.response.headers, - False, - flow.request.method, - flow.response.code) == -1 - ) - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 - # and no Content-Length header - - return False - return close_connection - - def send_response_to_client(self, flow): - if not (self.supports_streaming and flow.response.stream): - # no streaming: - # we already received the full response from the server and can - # send it to the client straight away. - self.send_response(flow.response) - else: - # streaming: - # First send the headers and then transfer the response incrementally - self.send_response_headers(flow.response) - chunks = self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code, - max_chunk_size=4096 - ) - if callable(flow.response.stream): - chunks = flow.response.stream(chunks) - self.send_response_body(flow.response, chunks) - flow.response.timestamp_end = utils.timestamp() - - def get_response_from_server(self, flow): - def get_response(): - self.send_request(flow.request) - if self.supports_streaming: - flow.response = self.read_response_headers() - else: - flow.response = self.read_response() - - try: - get_response() - except (tcp.NetLibError, HttpErrorConnClosed) as v: - self.log( - "server communication error: %s" % repr(v), - level="debug" - ) - # In any case, we try to reconnect at least once. This is - # necessary because it might be possible that we already - # initiated an upstream connection after clientconnect that - # has already been expired, e.g consider the following event - # log: - # > clientconnect (transparent mode destination known) - # > serverconnect (required for client tls handshake) - # > read n% of large request - # > server detects timeout, disconnects - # > read (100-n)% of large request - # > send large request upstream - self.reconnect() - get_response() - - # call the appropriate script hook - this is an opportunity for an - # inline script to set flow.stream = True - flow = self.channel.ask("responseheaders", flow) - if flow is None or flow == KILL: - raise Kill() - - if self.supports_streaming: - if flow.response.stream: - flow.response.content = CONTENT_MISSING - else: - flow.response.content = "".join(self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code - )) - flow.response.timestamp_end = utils.timestamp() - - # no further manipulation of self.server_conn beyond this point - # we can safely set it as the final attribute value here. - flow.server_conn = self.server_conn - - self.log( - "response", - "debug", - [repr(flow.response)] - ) - response_reply = self.channel.ask("response", flow) - if response_reply is None or response_reply == KILL: - raise Kill() - - def process_request_hook(self, flow): - # Determine .scheme, .host and .port attributes for inline scripts. - # For absolute-form requests, they are directly given in the request. - # For authority-form requests, we only need to determine the request scheme. - # For relative-form requests, we need to determine host and port as - # well. - if self.mode == "regular": - pass # only absolute-form at this point, nothing to do here. - elif self.mode == "upstream": - if flow.request.form_in == "authority": - flow.request.scheme = "http" # pseudo value - else: - flow.request.host = self.__original_server_conn.address.host - flow.request.port = self.__original_server_conn.address.port - flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" - - request_reply = self.channel.ask("request", flow) - if request_reply is None or request_reply == KILL: - raise Kill() - if isinstance(request_reply, HTTPResponse): - flow.response = request_reply - return - - def establish_server_connection(self, flow): - address = tcp.Address((flow.request.host, flow.request.port)) - tls = (flow.request.scheme == "https") - - if self.mode == "regular" or self.mode == "transparent": - # If there's an existing connection that doesn't match our expectations, kill it. - if address != self.server_conn.address or tls != self.server_conn.ssl_established: - self.set_server(address, tls, address.host) - # Establish connection is neccessary. - if not self.server_conn: - self.connect() - - # SetServer is not guaranteed to work with TLS: - # If there's not TlsLayer below which could catch the exception, - # TLS will not be established. - if tls and not self.server_conn.tls_established: - raise ProtocolException( - "Cannot upgrade to SSL, no TLS layer on the protocol stack.") - else: - if not self.server_conn: - self.connect() - if tls: - raise HttpException("Cannot change scheme in upstream proxy mode.") - """ - # This is a very ugly (untested) workaround to solve a very ugly problem. - if self.server_conn and self.server_conn.tls_established and not ssl: - self.reconnect() - elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: - if self.server_conn.tls_established: - self.reconnect() - - self.send_request(make_connect_request(address)) - tls_layer = TlsLayer(self, False, True) - tls_layer._establish_tls_with_server() - """ - - def validate_request(self, request): - if request.form_in == "absolute" and request.scheme != "http": - self.send_response( - make_error_response(400, "Invalid request scheme: %s" % request.scheme)) - raise HttpException("Invalid request scheme: %s" % request.scheme) - - expected_request_forms = { - "regular": ("absolute",), # an authority request would already be handled. - "upstream": ("authority", "absolute"), - "transparent": ("relative",) - } - - allowed_request_forms = expected_request_forms[self.mode] - if request.form_in not in allowed_request_forms: - err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( - " or ".join(allowed_request_forms), request.form_in - ) - self.send_response(make_error_response(400, err_message)) - raise HttpException(err_message) - - if self.mode == "regular": - request.form_out = "relative" - - def authenticate(self, request): - if self.config.authenticator: - if self.config.authenticator.authenticate(request.headers): - self.config.authenticator.clean(request.headers) - else: - self.send_response(make_error_response( - 407, - "Proxy Authentication Required", - odict.ODictCaseless( - [ - [k, v] for k, v in - self.config.authenticator.auth_challenge_headers().items() - ]) - )) - raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/protocol2/http_proxy.py deleted file mode 100644 index 2876c022..00000000 --- a/libmproxy/protocol2/http_proxy.py +++ /dev/null @@ -1,26 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from .layer import Layer, ServerConnectionMixin - - -class HttpProxy(Layer, ServerConnectionMixin): - def __call__(self): - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() - - -class HttpUpstreamProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx, server_address): - super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) - - def __call__(self): - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() diff --git a/libmproxy/protocol2/http_replay.py b/libmproxy/protocol2/http_replay.py deleted file mode 100644 index 872ef9cd..00000000 --- a/libmproxy/protocol2/http_replay.py +++ /dev/null @@ -1,95 +0,0 @@ -import threading -from netlib.http import HttpError -from netlib.http.http1 import HTTP1Protocol -from netlib.tcp import NetLibError - -from ..controller import Channel -from ..protocol import KILL, Error -from ..protocol.http_wrappers import HTTPResponse -from ..proxy import Log, Kill -from ..proxy.connection import ServerConnection -from .http import make_connect_request - - -class RequestReplayThread(threading.Thread): - name = "RequestReplayThread" - - def __init__(self, config, flow, masterq, should_exit): - """ - masterqueue can be a queue or None, if no scripthooks should be - processed. - """ - self.config, self.flow = config, flow - if masterq: - self.channel = Channel(masterq, should_exit) - else: - self.channel = None - super(RequestReplayThread, self).__init__() - - def run(self): - r = self.flow.request - form_out_backup = r.form_out - try: - self.flow.response = None - - # If we have a channel, run script hooks. - if self.channel: - request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == KILL: - raise Kill() - elif isinstance(request_reply, HTTPResponse): - self.flow.response = request_reply - - if not self.flow.response: - # In all modes, we directly connect to the server displayed - if self.config.mode == "upstream": - server_address = self.config.upstream_server.address - server = ServerConnection(server_address) - server.connect() - protocol = HTTP1Protocol(server) - if r.scheme == "https": - connect_request = make_connect_request((r.host, r.port)) - server.send(protocol.assemble(connect_request)) - resp = protocol.read_response("CONNECT") - if resp.code != 200: - raise HttpError(502, "Upstream server refuses CONNECT request") - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - else: - r.form_out = "absolute" - else: - server_address = (r.host, r.port) - server = ServerConnection(server_address) - server.connect() - protocol = HTTP1Protocol(server) - if r.scheme == "https": - server.establish_ssl( - self.config.clientcerts, - sni=self.flow.server_conn.sni - ) - r.form_out = "relative" - - server.send(protocol.assemble(r)) - self.flow.server_conn = server - self.flow.response = HTTPResponse.from_protocol( - protocol, - r.method, - body_size_limit=self.config.body_size_limit, - ) - if self.channel: - response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == KILL: - raise Kill() - except (HttpError, NetLibError) as v: - self.flow.error = Error(repr(v)) - if self.channel: - self.channel.ask("error", self.flow) - except Kill: - # KillSignal should only be raised if there's a channel in the - # first place. - self.channel.tell("log", Log("Connection killed", "info")) - finally: - r.form_out = form_out_backup diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol2/layer.py deleted file mode 100644 index 2b47cc26..00000000 --- a/libmproxy/protocol2/layer.py +++ /dev/null @@ -1,138 +0,0 @@ -""" -mitmproxy protocol architecture - -In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. -For example, the following scenarios depict possible scenarios (lowest layer first): - -Transparent HTTP proxy, no SSL: - TransparentModeLayer - HttpLayer - -Regular proxy, CONNECT request with WebSockets over SSL: - RegularModeLayer - HttpLayer - SslLayer - WebsocketLayer (or TcpLayer) - -Automated protocol detection by peeking into the buffer: - TransparentModeLayer - SslLayer - Http2Layer - -Communication between layers is done as follows: - - lower layers provide context information to higher layers - - higher layers can call functions provided by lower layers, - which are propagated until they reach a suitable layer. - -Further goals: - - Connections should always be peekable to make automatic protocol detection work. - - Upstream connections should be established as late as possible; - inline scripts shall have a chance to handle everything locally. -""" -from __future__ import (absolute_import, print_function, division) -from netlib import tcp -from ..proxy import Log -from ..proxy.connection import ServerConnection -from ..exceptions import ProtocolException - - -class _LayerCodeCompletion(object): - """ - Dummy class that provides type hinting in PyCharm, which simplifies development a lot. - """ - - def __init__(self, *args, **kwargs): - super(_LayerCodeCompletion, self).__init__(*args, **kwargs) - if True: - return - self.config = None - """@type: libmproxy.proxy.config.ProxyConfig""" - self.client_conn = None - """@type: libmproxy.proxy.connection.ClientConnection""" - self.channel = None - """@type: libmproxy.controller.Channel""" - - -class Layer(_LayerCodeCompletion): - def __init__(self, ctx, *args, **kwargs): - """ - Args: - ctx: The (read-only) higher layer. - """ - super(Layer, self).__init__(*args, **kwargs) - self.ctx = ctx - - def __call__(self): - """ - Logic of the layer. - Raises: - ProtocolException in case of protocol exceptions. - """ - raise NotImplementedError - - def __getattr__(self, name): - """ - Attributes not present on the current layer may exist on a higher layer. - """ - return getattr(self.ctx, name) - - def log(self, msg, level, subs=()): - full_msg = [ - "{}: {}".format(repr(self.client_conn.address), msg) - ] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - - @property - def layers(self): - return [self] + self.ctx.layers - - def __repr__(self): - return type(self).__name__ - - -class ServerConnectionMixin(object): - """ - Mixin that provides a layer with the capabilities to manage a server connection. - """ - - def __init__(self, server_address=None): - super(ServerConnectionMixin, self).__init__() - self.server_conn = ServerConnection(server_address) - - def reconnect(self): - address = self.server_conn.address - self._disconnect() - self.server_conn.address = address - self.connect() - - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1: - if self.server_conn: - self._disconnect() - self.log("Set new server address: " + repr(address), "debug") - self.server_conn.address = address - else: - self.ctx.set_server(address, server_tls, sni, depth - 1) - - def _disconnect(self): - """ - Deletes (and closes) an existing server connection. - """ - self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) - self.server_conn.finish() - self.server_conn.close() - # self.channel.tell("serverdisconnect", self) - self.server_conn = ServerConnection(None) - - def connect(self): - if not self.server_conn.address: - raise ProtocolException("Cannot connect to server, no server address given.") - self.log("serverconnect", "debug", [repr(self.server_conn.address)]) - try: - self.server_conn.connect() - except tcp.NetLibError as e: - raise ProtocolException( - "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol2/rawtcp.py deleted file mode 100644 index b10217f1..00000000 --- a/libmproxy/protocol2/rawtcp.py +++ /dev/null @@ -1,66 +0,0 @@ -from __future__ import (absolute_import, print_function, division) -import socket -import select - -from OpenSSL import SSL - -from netlib.tcp import NetLibError -from netlib.utils import cleanBin -from ..exceptions import ProtocolException -from .layer import Layer - - -class RawTcpLayer(Layer): - chunk_size = 4096 - - def __init__(self, ctx, logging=True): - self.logging = logging - super(RawTcpLayer, self).__init__(ctx) - - def __call__(self): - self.connect() - - buf = memoryview(bytearray(self.chunk_size)) - - client = self.client_conn.connection - server = self.server_conn.connection - conns = [client, server] - - try: - while True: - r, _, _ = select.select(conns, [], [], 10) - for conn in r: - dst = server if conn == client else client - - size = conn.recv_into(buf, self.chunk_size) - if not size: - conns.remove(conn) - # Shutdown connection to the other peer - if isinstance(conn, SSL.Connection): - # We can't half-close a connection, so we just close everything here. - # Sockets will be cleaned up on a higher level. - return - else: - dst.shutdown(socket.SHUT_WR) - - if len(conns) == 0: - return - continue - - dst.sendall(buf[:size]) - - if self.logging: - # log messages are prepended with the client address, - # hence the "weird" direction string. - if dst == server: - direction = "-> tcp -> {}".format(repr(self.server_conn.address)) - else: - direction = "<- tcp <- {}".format(repr(self.server_conn.address)) - data = cleanBin(buf[:size].tobytes()) - self.log( - "{}\r\n{}".format(direction, data), - "info" - ) - - except (socket.error, NetLibError, SSL.Error) as e: - raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/protocol2/reverse_proxy.py deleted file mode 100644 index 3ca998d5..00000000 --- a/libmproxy/protocol2/reverse_proxy.py +++ /dev/null @@ -1,17 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from .layer import Layer, ServerConnectionMixin - - -class ReverseProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx, server_address, server_tls): - super(ReverseProxy, self).__init__(ctx, server_address=server_address) - self.server_tls = server_tls - - def __call__(self): - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/protocol2/root_context.py deleted file mode 100644 index daea54bd..00000000 --- a/libmproxy/protocol2/root_context.py +++ /dev/null @@ -1,95 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from netlib.http.http1 import HTTP1Protocol -from netlib.http.http2 import HTTP2Protocol - -from .rawtcp import RawTcpLayer -from .tls import TlsLayer, is_tls_record_magic -from .http import Http1Layer, Http2Layer -from .layer import ServerConnectionMixin -from .http_proxy import HttpProxy, HttpUpstreamProxy -from .reverse_proxy import ReverseProxy - - -class RootContext(object): - """ - The outmost context provided to the root layer. - As a consequence, every layer has .client_conn, .channel, .next_layer() and .config. - """ - - def __init__(self, client_conn, config, channel): - self.client_conn = client_conn # Client Connection - self.channel = channel # provides .ask() method to communicate with FlowMaster - self.config = config # Proxy Configuration - - def next_layer(self, top_layer): - """ - This function determines the next layer in the protocol stack. - - Arguments: - top_layer: the current top layer. - - Returns: - The next layer - """ - - # 1. Check for --ignore. - if self.config.check_ignore(top_layer.server_conn.address): - return RawTcpLayer(top_layer, logging=False) - - d = top_layer.client_conn.rfile.peek(3) - client_tls = is_tls_record_magic(d) - - # 2. Always insert a TLS layer, even if there's neither client nor server tls. - # An inline script may upgrade from http to https, - # in which case we need some form of TLS layer. - if isinstance(top_layer, ReverseProxy): - return TlsLayer(top_layer, client_tls, top_layer.server_tls) - if isinstance(top_layer, ServerConnectionMixin): - return TlsLayer(top_layer, client_tls, client_tls) - - # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. - if isinstance(top_layer, TlsLayer): - if isinstance(top_layer.ctx, HttpProxy): - return Http1Layer(top_layer, "regular") - if isinstance(top_layer.ctx, HttpUpstreamProxy): - return Http1Layer(top_layer, "upstream") - - # 4. Check for other TLS cases (e.g. after CONNECT). - if client_tls: - return TlsLayer(top_layer, True, True) - - # 4. Check for --tcp - if self.config.check_tcp(top_layer.server_conn.address): - return RawTcpLayer(top_layer) - - # 5. Check for TLS ALPN (HTTP1/HTTP2) - if isinstance(top_layer, TlsLayer): - alpn = top_layer.client_conn.get_alpn_proto_negotiated() - if alpn == HTTP2Protocol.ALPN_PROTO_H2: - return Http2Layer(top_layer, 'transparent') - if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: - return Http1Layer(top_layer, 'transparent') - - # 6. Assume HTTP1 by default - return Http1Layer(top_layer, 'transparent') - - # In a future version, we want to implement TCP passthrough as the last fallback, - # but we don't have the UI part ready for that. - # - # d = top_layer.client_conn.rfile.peek(3) - # is_ascii = ( - # len(d) == 3 and - # # better be safe here and don't expect uppercase... - # all(x in string.ascii_letters for x in d) - # ) - # # TODO: This could block if there are not enough bytes available? - # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) - # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) - - @property - def layers(self): - return [] - - def __repr__(self): - return "RootContext" diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/protocol2/socks_proxy.py deleted file mode 100644 index 525520e8..00000000 --- a/libmproxy/protocol2/socks_proxy.py +++ /dev/null @@ -1,59 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from netlib import socks -from netlib.tcp import NetLibError -from ..exceptions import Socks5Exception -from .layer import Layer, ServerConnectionMixin - - -class Socks5Proxy(Layer, ServerConnectionMixin): - def __call__(self): - try: - # Parse Client Greeting - client_greet = socks.ClientGreeting.from_file(self.client_conn.rfile, fail_early=True) - client_greet.assert_socks5() - if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: - raise socks.SocksError( - socks.METHOD.NO_ACCEPTABLE_METHODS, - "mitmproxy only supports SOCKS without authentication" - ) - - # Send Server Greeting - server_greet = socks.ServerGreeting( - socks.VERSION.SOCKS5, - socks.METHOD.NO_AUTHENTICATION_REQUIRED - ) - server_greet.to_file(self.client_conn.wfile) - self.client_conn.wfile.flush() - - # Parse Connect Request - connect_request = socks.Message.from_file(self.client_conn.rfile) - connect_request.assert_socks5() - if connect_request.msg != socks.CMD.CONNECT: - raise socks.SocksError( - socks.REP.COMMAND_NOT_SUPPORTED, - "mitmproxy only supports SOCKS5 CONNECT." - ) - - # We always connect lazily, but we need to pretend to the client that we connected. - connect_reply = socks.Message( - socks.VERSION.SOCKS5, - socks.REP.SUCCEEDED, - connect_request.atyp, - # dummy value, we don't have an upstream connection yet. - connect_request.addr - ) - connect_reply.to_file(self.client_conn.wfile) - self.client_conn.wfile.flush() - - except (socks.SocksError, NetLibError) as e: - raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e) - - self.server_conn.address = connect_request.addr - - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol2/tls.py deleted file mode 100644 index 73bb12f3..00000000 --- a/libmproxy/protocol2/tls.py +++ /dev/null @@ -1,288 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -import struct - -from construct import ConstructError - -from netlib.tcp import NetLibError, NetLibInvalidCertificateError -from netlib.http.http1 import HTTP1Protocol -from ..contrib.tls._constructs import ClientHello -from ..exceptions import ProtocolException -from .layer import Layer - - -def is_tls_record_magic(d): - """ - Returns: - True, if the passed bytes start with the TLS record magic bytes. - False, otherwise. - """ - d = d[:3] - - # TLS ClientHello magic, works for SSLv3, TLSv1.0, TLSv1.1, TLSv1.2 - # http://www.moserware.com/2009/06/first-few-milliseconds-of-https.html#client-hello - return ( - len(d) == 3 and - d[0] == '\x16' and - d[1] == '\x03' and - d[2] in ('\x00', '\x01', '\x02', '\x03') - ) - - -class TlsLayer(Layer): - def __init__(self, ctx, client_tls, server_tls): - self.client_sni = None - self.client_alpn_protocols = None - - super(TlsLayer, self).__init__(ctx) - self._client_tls = client_tls - self._server_tls = server_tls - - self._sni_from_server_change = None - - def __call__(self): - """ - The strategy for establishing SSL is as follows: - First, we determine whether we need the server cert to establish ssl with the client. - If so, we first connect to the server and then to the client. - If not, we only connect to the client and do the server_ssl lazily on a Connect message. - - An additional complexity is that establish ssl with the server may require a SNI value from the client. - In an ideal world, we'd do the following: - 1. Start the SSL handshake with the client - 2. Check if the client sends a SNI. - 3. Pause the client handshake, establish SSL with the server. - 4. Finish the client handshake with the certificate from the server. - There's just one issue: We cannot get a callback from OpenSSL if the client doesn't send a SNI. :( - Thus, we manually peek into the connection and parse the ClientHello message to obtain both SNI and ALPN values. - - Further notes: - - OpenSSL 1.0.2 introduces a callback that would help here: - https://www.openssl.org/docs/ssl/SSL_CTX_set_cert_cb.html - - The original mitmproxy issue is https://github.com/mitmproxy/mitmproxy/issues/427 - """ - - client_tls_requires_server_cert = ( - self._client_tls and self._server_tls and not self.config.no_upstream_cert - ) - - if self._client_tls: - self._parse_client_hello() - - if client_tls_requires_server_cert: - self._establish_tls_with_client_and_server() - elif self._client_tls: - self._establish_tls_with_client() - - layer = self.ctx.next_layer(self) - layer() - - def __repr__(self): - if self._client_tls and self._server_tls: - return "TlsLayer(client and server)" - elif self._client_tls: - return "TlsLayer(client)" - elif self._server_tls: - return "TlsLayer(server)" - else: - return "TlsLayer(inactive)" - - def _get_client_hello(self): - """ - Peek into the socket and read all records that contain the initial client hello message. - - Returns: - The raw handshake packet bytes, without TLS record header(s). - """ - client_hello = "" - client_hello_size = 1 - offset = 0 - while len(client_hello) < client_hello_size: - record_header = self.client_conn.rfile.peek(offset + 5)[offset:] - if not is_tls_record_magic(record_header) or len(record_header) != 5: - raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) - record_size = struct.unpack("!H", record_header[3:])[0] + 5 - record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] - if len(record_body) != record_size - 5: - raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) - client_hello += record_body - offset += record_size - client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 - return client_hello - - def _parse_client_hello(self): - """ - Peek into the connection, read the initial client hello and parse it to obtain ALPN values. - """ - try: - raw_client_hello = self._get_client_hello()[4:] # exclude handshake header. - except ProtocolException as e: - self.log("Cannot parse Client Hello: %s" % repr(e), "error") - return - - try: - client_hello = ClientHello.parse(raw_client_hello) - except ConstructError as e: - self.log("Cannot parse Client Hello: %s" % repr(e), "error") - self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") - return - - for extension in client_hello.extensions: - if extension.type == 0x00: - if len(extension.server_names) != 1 or extension.server_names[0].type != 0: - self.log("Unknown Server Name Indication: %s" % extension.server_names, "error") - self.client_sni = extension.server_names[0].name - elif extension.type == 0x10: - self.client_alpn_protocols = list(extension.alpn_protocols) - - self.log( - "Parsed Client Hello: sni=%s, alpn=%s" % (self.client_sni, self.client_alpn_protocols), - "debug" - ) - - def connect(self): - if not self.server_conn: - self.ctx.connect() - if self._server_tls and not self.server_conn.tls_established: - self._establish_tls_with_server() - - def reconnect(self): - self.ctx.reconnect() - if self._server_tls and not self.server_conn.tls_established: - self._establish_tls_with_server() - - def set_server(self, address, server_tls=None, sni=None, depth=1): - self.ctx.set_server(address, server_tls, sni, depth) - if depth == 1 and server_tls is not None: - self._sni_from_server_change = sni - self._server_tls = server_tls - - @property - def sni_for_server_connection(self): - if self._sni_from_server_change is False: - return None - else: - return self._sni_from_server_change or self.client_sni - - @property - def alpn_for_client_connection(self): - return self.server_conn.get_alpn_proto_negotiated() - - def __alpn_select_callback(self, conn_, options): - """ - Once the client signals the alternate protocols it supports, - we reconnect upstream with the same list and pass the server's choice down to the client. - """ - - # This gets triggered if we haven't established an upstream connection yet. - default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 - # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 - - if self.alpn_for_client_connection in options: - choice = bytes(self.alpn_for_client_connection) - elif default_alpn in options: - choice = bytes(default_alpn) - else: - choice = options[0] - self.log("ALPN for client: %s" % choice, "debug") - return choice - - def _establish_tls_with_client_and_server(self): - self.ctx.connect() - - # If establishing TLS with the server fails, we try to establish TLS with the client nonetheless - # to send an error message over TLS. - try: - self._establish_tls_with_server() - except Exception as e: - try: - self._establish_tls_with_client() - except: - pass - raise e - - self._establish_tls_with_client() - - def _establish_tls_with_client(self): - self.log("Establish TLS with client", "debug") - cert, key, chain_file = self._find_cert() - - try: - self.client_conn.convert_to_ssl( - cert, key, - method=self.config.openssl_method_client, - options=self.config.openssl_options_client, - cipher_list=self.config.ciphers_client, - dhparams=self.config.certstore.dhparams, - chain_file=chain_file, - alpn_select_callback=self.__alpn_select_callback, - ) - except NetLibError as e: - raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) - - def _establish_tls_with_server(self): - self.log("Establish TLS with server", "debug") - try: - # We only support http/1.1 and h2. - # If the server only supports spdy (next to http/1.1), it may select that - # and mitmproxy would enter TCP passthrough mode, which we want to avoid. - deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") - if self.client_alpn_protocols: - alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) - else: - alpn = None - - self.server_conn.establish_ssl( - self.config.clientcerts, - self.sni_for_server_connection, - method=self.config.openssl_method_server, - options=self.config.openssl_options_server, - verify_options=self.config.openssl_verification_mode_server, - ca_path=self.config.openssl_trusted_cadir_server, - ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, - alpn_protos=alpn, - ) - tls_cert_err = self.server_conn.ssl_verification_error - if tls_cert_err is not None: - self.log( - "TLS verification failed for upstream server at depth %s with error: %s" % - (tls_cert_err['depth'], tls_cert_err['errno']), - "error") - self.log("Ignoring server verification error, continuing with connection", "error") - except NetLibInvalidCertificateError as e: - tls_cert_err = self.server_conn.ssl_verification_error - self.log( - "TLS verification failed for upstream server at depth %s with error: %s" % - (tls_cert_err['depth'], tls_cert_err['errno']), - "error") - self.log("Aborting connection attempt", "error") - raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) - except NetLibError as e: - raise ProtocolException("Cannot establish TLS with server: %s" % repr(e), e) - - self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") - - def _find_cert(self): - host = self.server_conn.address.host - sans = set() - # Incorporate upstream certificate - use_upstream_cert = ( - self.server_conn and - self.server_conn.tls_established and - (not self.config.no_upstream_cert) - ) - if use_upstream_cert: - upstream_cert = self.server_conn.cert - sans.update(upstream_cert.altnames) - if upstream_cert.cn: - sans.add(host) - host = upstream_cert.cn.decode("utf8").encode("idna") - # Also add SNI values. - if self.client_sni: - sans.add(self.client_sni) - if self._sni_from_server_change: - sans.add(self._sni_from_server_change) - - sans.discard(host) - return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/protocol2/transparent_proxy.py deleted file mode 100644 index e6ebf115..00000000 --- a/libmproxy/protocol2/transparent_proxy.py +++ /dev/null @@ -1,24 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from ..exceptions import ProtocolException -from .. import platform -from .layer import Layer, ServerConnectionMixin - - -class TransparentProxy(Layer, ServerConnectionMixin): - def __init__(self, ctx): - super(TransparentProxy, self).__init__(ctx) - self.resolver = platform.resolver() - - def __call__(self): - try: - self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) - except Exception as e: - raise ProtocolException("Transparent mode failure: %s" % repr(e), e) - - layer = self.ctx.next_layer(self) - try: - layer() - finally: - if self.server_conn: - self._disconnect() diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index 709654cb..d5297cb1 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1,11 +1,9 @@ from __future__ import (absolute_import, print_function, division) -from .primitives import Log, Kill +from .server import ProxyServer, DummyServer from .config import ProxyConfig -from .connection import ClientConnection, ServerConnection __all__ = [ - "Log", "Kill", + "ProxyServer", "DummyServer", "ProxyConfig", - "ClientConnection", "ServerConnection" -] \ No newline at end of file +] diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index b360abbd..65029087 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -6,9 +6,9 @@ from OpenSSL import SSL from netlib import certutils, tcp from netlib.http import authentication +from netlib.tcp import Address, sslversion_choices from .. import utils, platform -from netlib.tcp import Address, sslversion_choices CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py deleted file mode 100644 index 94f318f6..00000000 --- a/libmproxy/proxy/connection.py +++ /dev/null @@ -1,193 +0,0 @@ -from __future__ import absolute_import - -import copy -import os - -from netlib import tcp, certutils -from .. import stateobject, utils - - -class ClientConnection(tcp.BaseHandler, stateobject.StateObject): - def __init__(self, client_connection, address, server): - # Eventually, this object is restored from state. We don't have a - # connection then. - if client_connection: - super(ClientConnection, self).__init__(client_connection, address, server) - else: - self.connection = None - self.server = None - self.wfile = None - self.rfile = None - self.address = None - self.clientcert = None - self.ssl_established = None - - self.timestamp_start = utils.timestamp() - self.timestamp_end = None - self.timestamp_ssl_setup = None - self.protocol = None - - def __nonzero__(self): - return bool(self.connection) and not self.finished - - def __repr__(self): - return "".format( - ssl="[ssl] " if self.ssl_established else "", - host=self.address.host, - port=self.address.port - ) - - @property - def tls_established(self): - return self.ssl_established - - _stateobject_attributes = dict( - ssl_established=bool, - timestamp_start=float, - timestamp_end=float, - timestamp_ssl_setup=float - ) - - def get_state(self, short=False): - d = super(ClientConnection, self).get_state(short) - d.update( - address={ - "address": self.address(), - "use_ipv6": self.address.use_ipv6}, - clientcert=self.cert.to_pem() if self.clientcert else None) - return d - - def load_state(self, state): - super(ClientConnection, self).load_state(state) - self.address = tcp.Address( - **state["address"]) if state["address"] else None - self.clientcert = certutils.SSLCert.from_pem( - state["clientcert"]) if state["clientcert"] else None - - def copy(self): - return copy.copy(self) - - def send(self, message): - if isinstance(message, list): - message = b''.join(message) - self.wfile.write(message) - self.wfile.flush() - - @classmethod - def from_state(cls, state): - f = cls(None, tuple(), None) - f.load_state(state) - return f - - def convert_to_ssl(self, *args, **kwargs): - super(ClientConnection, self).convert_to_ssl(*args, **kwargs) - self.timestamp_ssl_setup = utils.timestamp() - - def finish(self): - super(ClientConnection, self).finish() - self.timestamp_end = utils.timestamp() - - -class ServerConnection(tcp.TCPClient, stateobject.StateObject): - def __init__(self, address): - tcp.TCPClient.__init__(self, address) - - self.via = None - self.timestamp_start = None - self.timestamp_end = None - self.timestamp_tcp_setup = None - self.timestamp_ssl_setup = None - self.protocol = None - - def __nonzero__(self): - return bool(self.connection) and not self.finished - - def __repr__(self): - if self.ssl_established and self.sni: - ssl = "[ssl: {0}] ".format(self.sni) - elif self.ssl_established: - ssl = "[ssl] " - else: - ssl = "" - return "".format( - ssl=ssl, - host=self.address.host, - port=self.address.port - ) - - @property - def tls_established(self): - return self.ssl_established - - _stateobject_attributes = dict( - timestamp_start=float, - timestamp_end=float, - timestamp_tcp_setup=float, - timestamp_ssl_setup=float, - address=tcp.Address, - source_address=tcp.Address, - cert=certutils.SSLCert, - ssl_established=bool, - sni=str - ) - _stateobject_long_attributes = {"cert"} - - def get_state(self, short=False): - d = super(ServerConnection, self).get_state(short) - d.update( - address={"address": self.address(), - "use_ipv6": self.address.use_ipv6}, - source_address=({"address": self.source_address(), - "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), - cert=self.cert.to_pem() if self.cert else None - ) - return d - - def load_state(self, state): - super(ServerConnection, self).load_state(state) - - self.address = tcp.Address( - **state["address"]) if state["address"] else None - self.source_address = tcp.Address( - **state["source_address"]) if state["source_address"] else None - self.cert = certutils.SSLCert.from_pem( - state["cert"]) if state["cert"] else None - - @classmethod - def from_state(cls, state): - f = cls(tuple()) - f.load_state(state) - return f - - def copy(self): - return copy.copy(self) - - def connect(self): - self.timestamp_start = utils.timestamp() - tcp.TCPClient.connect(self) - self.timestamp_tcp_setup = utils.timestamp() - - def send(self, message): - if isinstance(message, list): - message = b''.join(message) - self.wfile.write(message) - self.wfile.flush() - - def establish_ssl(self, clientcerts, sni, **kwargs): - clientcert = None - if clientcerts: - path = os.path.join( - clientcerts, - self.address.host.encode("idna")) + ".pem" - if os.path.exists(path): - clientcert = path - - self.convert_to_ssl(cert=clientcert, sni=sni, **kwargs) - self.sni = sni - self.timestamp_ssl_setup = utils.timestamp() - - def finish(self): - tcp.TCPClient.finish(self) - self.timestamp_end = utils.timestamp() - -ServerConnection._stateobject_attributes["via"] = ServerConnection diff --git a/libmproxy/proxy/modes/__init__.py b/libmproxy/proxy/modes/__init__.py new file mode 100644 index 00000000..f014ed98 --- /dev/null +++ b/libmproxy/proxy/modes/__init__.py @@ -0,0 +1,12 @@ +from __future__ import (absolute_import, print_function, division) +from .http_proxy import HttpProxy, HttpUpstreamProxy +from .reverse_proxy import ReverseProxy +from .socks_proxy import Socks5Proxy +from .transparent_proxy import TransparentProxy + +__all__ = [ + "HttpProxy", "HttpUpstreamProxy", + "ReverseProxy", + "Socks5Proxy", + "TransparentProxy" +] diff --git a/libmproxy/proxy/modes/http_proxy.py b/libmproxy/proxy/modes/http_proxy.py new file mode 100644 index 00000000..90c54cc6 --- /dev/null +++ b/libmproxy/proxy/modes/http_proxy.py @@ -0,0 +1,26 @@ +from __future__ import (absolute_import, print_function, division) + +from ...protocol import Layer, ServerConnectionMixin + + +class HttpProxy(Layer, ServerConnectionMixin): + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() + + +class HttpUpstreamProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address): + super(HttpUpstreamProxy, self).__init__(ctx, server_address=server_address) + + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/modes/reverse_proxy.py b/libmproxy/proxy/modes/reverse_proxy.py new file mode 100644 index 00000000..b57ac5eb --- /dev/null +++ b/libmproxy/proxy/modes/reverse_proxy.py @@ -0,0 +1,17 @@ +from __future__ import (absolute_import, print_function, division) + +from ...protocol import Layer, ServerConnectionMixin + + +class ReverseProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx, server_address, server_tls): + super(ReverseProxy, self).__init__(ctx, server_address=server_address) + self.server_tls = server_tls + + def __call__(self): + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/modes/socks_proxy.py b/libmproxy/proxy/modes/socks_proxy.py new file mode 100644 index 00000000..ebaf939e --- /dev/null +++ b/libmproxy/proxy/modes/socks_proxy.py @@ -0,0 +1,60 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib import socks +from netlib.tcp import NetLibError + +from ...exceptions import Socks5Exception +from ...protocol import Layer, ServerConnectionMixin + + +class Socks5Proxy(Layer, ServerConnectionMixin): + def __call__(self): + try: + # Parse Client Greeting + client_greet = socks.ClientGreeting.from_file(self.client_conn.rfile, fail_early=True) + client_greet.assert_socks5() + if socks.METHOD.NO_AUTHENTICATION_REQUIRED not in client_greet.methods: + raise socks.SocksError( + socks.METHOD.NO_ACCEPTABLE_METHODS, + "mitmproxy only supports SOCKS without authentication" + ) + + # Send Server Greeting + server_greet = socks.ServerGreeting( + socks.VERSION.SOCKS5, + socks.METHOD.NO_AUTHENTICATION_REQUIRED + ) + server_greet.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + # Parse Connect Request + connect_request = socks.Message.from_file(self.client_conn.rfile) + connect_request.assert_socks5() + if connect_request.msg != socks.CMD.CONNECT: + raise socks.SocksError( + socks.REP.COMMAND_NOT_SUPPORTED, + "mitmproxy only supports SOCKS5 CONNECT." + ) + + # We always connect lazily, but we need to pretend to the client that we connected. + connect_reply = socks.Message( + socks.VERSION.SOCKS5, + socks.REP.SUCCEEDED, + connect_request.atyp, + # dummy value, we don't have an upstream connection yet. + connect_request.addr + ) + connect_reply.to_file(self.client_conn.wfile) + self.client_conn.wfile.flush() + + except (socks.SocksError, NetLibError) as e: + raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e) + + self.server_conn.address = connect_request.addr + + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/modes/transparent_proxy.py b/libmproxy/proxy/modes/transparent_proxy.py new file mode 100644 index 00000000..96ad86c4 --- /dev/null +++ b/libmproxy/proxy/modes/transparent_proxy.py @@ -0,0 +1,24 @@ +from __future__ import (absolute_import, print_function, division) + +from ... import platform +from ...exceptions import ProtocolException +from ...protocol import Layer, ServerConnectionMixin + + +class TransparentProxy(Layer, ServerConnectionMixin): + def __init__(self, ctx): + super(TransparentProxy, self).__init__(ctx) + self.resolver = platform.resolver() + + def __call__(self): + try: + self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) + except Exception as e: + raise ProtocolException("Transparent mode failure: %s" % repr(e), e) + + layer = self.ctx.next_layer(self) + try: + layer() + finally: + if self.server_conn: + self._disconnect() diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py deleted file mode 100644 index 2e440fe8..00000000 --- a/libmproxy/proxy/primitives.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import absolute_import -import collections -from netlib import socks, tcp - - -class Log(object): - def __init__(self, msg, level="info"): - self.msg = msg - self.level = level - - -class Kill(Exception): - """ - Kill a connection. - """ \ No newline at end of file diff --git a/libmproxy/proxy/root_context.py b/libmproxy/proxy/root_context.py new file mode 100644 index 00000000..35909612 --- /dev/null +++ b/libmproxy/proxy/root_context.py @@ -0,0 +1,93 @@ +from __future__ import (absolute_import, print_function, division) + +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol + +from ..protocol import ( + RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin +) +from .modes import HttpProxy, HttpUpstreamProxy, ReverseProxy + + +class RootContext(object): + """ + The outmost context provided to the root layer. + As a consequence, every layer has .client_conn, .channel, .next_layer() and .config. + """ + + def __init__(self, client_conn, config, channel): + self.client_conn = client_conn # Client Connection + self.channel = channel # provides .ask() method to communicate with FlowMaster + self.config = config # Proxy Configuration + + def next_layer(self, top_layer): + """ + This function determines the next layer in the protocol stack. + + Arguments: + top_layer: the current top layer. + + Returns: + The next layer + """ + + # 1. Check for --ignore. + if self.config.check_ignore(top_layer.server_conn.address): + return RawTCPLayer(top_layer, logging=False) + + d = top_layer.client_conn.rfile.peek(3) + client_tls = is_tls_record_magic(d) + + # 2. Always insert a TLS layer, even if there's neither client nor server tls. + # An inline script may upgrade from http to https, + # in which case we need some form of TLS layer. + if isinstance(top_layer, ReverseProxy): + return TlsLayer(top_layer, client_tls, top_layer.server_tls) + if isinstance(top_layer, ServerConnectionMixin): + return TlsLayer(top_layer, client_tls, client_tls) + + # 3. In Http Proxy mode and Upstream Proxy mode, the next layer is fixed. + if isinstance(top_layer, TlsLayer): + if isinstance(top_layer.ctx, HttpProxy): + return Http1Layer(top_layer, "regular") + if isinstance(top_layer.ctx, HttpUpstreamProxy): + return Http1Layer(top_layer, "upstream") + + # 4. Check for other TLS cases (e.g. after CONNECT). + if client_tls: + return TlsLayer(top_layer, True, True) + + # 4. Check for --tcp + if self.config.check_tcp(top_layer.server_conn.address): + return RawTCPLayer(top_layer) + + # 5. Check for TLS ALPN (HTTP1/HTTP2) + if isinstance(top_layer, TlsLayer): + alpn = top_layer.client_conn.get_alpn_proto_negotiated() + if alpn == HTTP2Protocol.ALPN_PROTO_H2: + return Http2Layer(top_layer, 'transparent') + if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: + return Http1Layer(top_layer, 'transparent') + + # 6. Assume HTTP1 by default + return Http1Layer(top_layer, 'transparent') + + # In a future version, we want to implement TCP passthrough as the last fallback, + # but we don't have the UI part ready for that. + # + # d = top_layer.client_conn.rfile.peek(3) + # is_ascii = ( + # len(d) == 3 and + # # better be safe here and don't expect uppercase... + # all(x in string.ascii_letters for x in d) + # ) + # # TODO: This could block if there are not enough bytes available? + # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) + # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) + + @property + def layers(self): + return [] + + def __repr__(self): + return "RootContext" diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 5abd0877..2a451ba1 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -3,14 +3,15 @@ from __future__ import absolute_import, print_function import traceback import sys import socket + from netlib import tcp from netlib.http.http1 import HTTP1Protocol from netlib.tcp import NetLibError - -from .. import protocol2 from ..exceptions import ProtocolException, ServerException -from .primitives import Log, Kill -from .connection import ClientConnection +from ..protocol import Log, Kill +from ..models import ClientConnection, make_error_response +from .modes import HttpUpstreamProxy, HttpProxy, ReverseProxy, TransparentProxy, Socks5Proxy +from .root_context import RootContext class DummyServer: @@ -71,7 +72,7 @@ class ConnectionHandler(object): """@type: libmproxy.controller.Channel""" def _create_root_layer(self): - root_context = protocol2.RootContext( + root_context = RootContext( self.client_conn, self.config, self.channel @@ -79,23 +80,23 @@ class ConnectionHandler(object): mode = self.config.mode if mode == "upstream": - return protocol2.HttpUpstreamProxy( + return HttpUpstreamProxy( root_context, self.config.upstream_server.address ) elif mode == "transparent": - return protocol2.TransparentProxy(root_context) + return TransparentProxy(root_context) elif mode == "reverse": server_tls = self.config.upstream_server.scheme == "https" - return protocol2.ReverseProxy( + return ReverseProxy( root_context, self.config.upstream_server.address, server_tls ) elif mode == "socks5": - return protocol2.Socks5Proxy(root_context) + return Socks5Proxy(root_context) elif mode == "regular": - return protocol2.HttpProxy(root_context) + return HttpProxy(root_context) elif callable(mode): # pragma: nocover return mode(root_context) else: # pragma: nocover @@ -116,7 +117,7 @@ class ConnectionHandler(object): # we send an HTTP error response, which is both # understandable by HTTP clients and humans. try: - error_response = protocol2.make_error_response(502, repr(e)) + error_response = make_error_response(502, repr(e)) self.client_conn.send(HTTP1Protocol().assemble(error_response)) except NetLibError: pass -- cgit v1.2.3