diff options
Diffstat (limited to 'libmproxy')
| -rw-r--r-- | libmproxy/cmdline.py | 18 | ||||
| -rw-r--r-- | libmproxy/console/common.py | 4 | ||||
| -rw-r--r-- | libmproxy/console/flowview.py | 19 | ||||
| -rw-r--r-- | libmproxy/contentview.py | 15 | ||||
| -rw-r--r-- | libmproxy/dump.py | 2 | ||||
| -rw-r--r-- | libmproxy/exceptions.py | 25 | ||||
| -rw-r--r-- | libmproxy/filt.py | 55 | ||||
| -rw-r--r-- | libmproxy/flow.py | 62 | ||||
| -rw-r--r-- | libmproxy/models/http.py | 62 | ||||
| -rw-r--r-- | libmproxy/protocol/__init__.py | 31 | ||||
| -rw-r--r-- | libmproxy/protocol/base.py | 184 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 229 | ||||
| -rw-r--r-- | libmproxy/protocol/http_replay.py | 5 | ||||
| -rw-r--r-- | libmproxy/protocol/rawtcp.py | 12 | ||||
| -rw-r--r-- | libmproxy/protocol/tls.py | 280 | ||||
| -rw-r--r-- | libmproxy/proxy/__init__.py | 2 | ||||
| -rw-r--r-- | libmproxy/proxy/config.py | 8 | ||||
| -rw-r--r-- | libmproxy/proxy/modes/http_proxy.py | 4 | ||||
| -rw-r--r-- | libmproxy/proxy/modes/reverse_proxy.py | 2 | ||||
| -rw-r--r-- | libmproxy/proxy/modes/socks_proxy.py | 4 | ||||
| -rw-r--r-- | libmproxy/proxy/modes/transparent_proxy.py | 4 | ||||
| -rw-r--r-- | libmproxy/proxy/root_context.py | 78 | ||||
| -rw-r--r-- | libmproxy/proxy/server.py | 28 | ||||
| -rw-r--r-- | libmproxy/script.py | 5 | ||||
| -rw-r--r-- | libmproxy/web/app.py | 7 |
25 files changed, 779 insertions, 366 deletions
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 7f6f69ef..3779953f 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -1,11 +1,11 @@ from __future__ import absolute_import import os import re + import configargparse -from netlib.tcp import Address, sslversion_choices +from netlib.tcp import Address, sslversion_choices import netlib.utils - from . import filt, utils, version from .proxy import config @@ -358,6 +358,20 @@ def proxy_options(parser): action="store", type=int, dest="port", default=8080, help="Proxy service port." ) + http2 = group.add_mutually_exclusive_group() + http2.add_argument("--http2", action="store_true", dest="http2") + http2.add_argument("--no-http2", action="store_false", dest="http2", + help="Explicitly enable/disable experimental HTTP2 support. " + "Disabled by default. " + "Default value will change in a future version." + ) + rawtcp = group.add_mutually_exclusive_group() + rawtcp.add_argument("--raw-tcp", action="store_true", dest="rawtcp") + rawtcp.add_argument("--no-raw-tcp", action="store_false", dest="rawtcp", + help="Explicitly enable/disable experimental raw tcp support. " + "Disabled by default. " + "Default value will change in a future version." + ) def proxy_ssl_options(parser): diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index c25f7267..ae3dd61e 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -415,9 +415,9 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2, resp_clen = contentdesc, roundtrip = roundtrip, )) - t = f.response.headers["content-type"] + t = f.response.headers.get("content-type") if t: - d["resp_ctype"] = t[0].split(";")[0] + d["resp_ctype"] = t.split(";")[0] else: d["resp_ctype"] = "" return flowcache.get( diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 958ab176..e33d4c43 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -4,7 +4,7 @@ import sys import urwid from netlib import odict -from netlib.http.semantics import CONTENT_MISSING +from netlib.http.semantics import CONTENT_MISSING, Headers from . import common, grideditor, signals, searchable, tabs from . import flowdetailview @@ -182,7 +182,7 @@ class FlowView(tabs.Tabs): description, text_objects = cache.get( contentview.get_content_view, viewmode, - tuple(tuple(i) for i in conn.headers.lst), + conn.headers, conn.content, limit, isinstance(conn, HTTPRequest), @@ -200,7 +200,7 @@ class FlowView(tabs.Tabs): def conn_text(self, conn): if conn: txt = common.format_keyvals( - [(h + ":", v) for (h, v) in conn.headers.lst], + [(h + ":", v) for (h, v) in conn.headers.fields], key = "header", val = "text" ) @@ -285,8 +285,8 @@ class FlowView(tabs.Tabs): response.msg = msg signals.flow_change.send(self, flow = self.flow) - def set_headers(self, lst, conn): - conn.headers = odict.ODictCaseless(lst) + def set_headers(self, fields, conn): + conn.headers = Headers(fields) signals.flow_change.send(self, flow = self.flow) def set_query(self, lst, conn): @@ -331,7 +331,7 @@ class FlowView(tabs.Tabs): if not self.flow.response: self.flow.response = HTTPResponse( self.flow.request.httpversion, - 200, "OK", odict.ODictCaseless(), "" + 200, "OK", Headers(), "" ) self.flow.response.reply = controller.DummyReply() message = self.flow.response @@ -382,7 +382,7 @@ class FlowView(tabs.Tabs): self.master.view_grideditor( grideditor.HeaderEditor( self.master, - message.headers.lst, + message.headers.fields, self.set_headers, message ) @@ -617,8 +617,7 @@ class FlowView(tabs.Tabs): key = None elif key == "v": if conn.content: - t = conn.headers["content-type"] or [None] - t = t[0] + t = conn.headers.get("content-type") if "EDITOR" in os.environ or "PAGER" in os.environ: self.master.spawn_external_viewer(conn.content, t) else: @@ -627,7 +626,7 @@ class FlowView(tabs.Tabs): ) elif key == "z": self.flow.backup() - e = conn.headers.get_first("content-encoding", "identity") + e = conn.headers.get("content-encoding", "identity") if e != "identity": if not conn.decode(): signals.status_message.send( diff --git a/libmproxy/contentview.py b/libmproxy/contentview.py index 45c1f2f1..a9b6cf95 100644 --- a/libmproxy/contentview.py +++ b/libmproxy/contentview.py @@ -9,14 +9,13 @@ import lxml.html import lxml.etree from PIL import Image from PIL.ExifTags import TAGS -import urwid import html2text import netlib.utils -from netlib import odict, encoding from . import utils from .contrib import jsbeautifier from .contrib.wbxml.ASCommandResponse import ASCommandResponse +from netlib import encoding try: import pyamf @@ -129,7 +128,7 @@ class ViewAuto(View): content_types = [] def __call__(self, hdrs, content, limit): - ctype = hdrs.get_first("content-type") + ctype = hdrs.get("content-type") if ctype: ct = netlib.utils.parse_content_type(ctype) if ctype else None ct = "%s/%s" % (ct[0], ct[1]) @@ -536,7 +535,7 @@ def get(name): return i -def get_content_view(viewmode, hdrItems, content, limit, is_request, log=None): +def get_content_view(viewmode, headers, content, limit, is_request, log=None): """ Returns: A (msg, body) tuple. @@ -551,16 +550,14 @@ def get_content_view(viewmode, hdrItems, content, limit, is_request, log=None): return "No content", "" msg = [] - hdrs = odict.ODictCaseless([list(i) for i in hdrItems]) - - enc = hdrs.get_first("content-encoding") + enc = headers.get("content-encoding") if enc and enc != "identity": decoded = encoding.decode(enc, content) if decoded: content = decoded msg.append("[decoded %s]" % enc) try: - ret = viewmode(hdrs, content, limit) + ret = viewmode(headers, content, limit) # Third-party viewers can fail in unexpected ways... except Exception: if log: @@ -569,7 +566,7 @@ def get_content_view(viewmode, hdrItems, content, limit, is_request, log=None): log(s, "error") ret = None if not ret: - ret = get("Raw")(hdrs, content, limit) + ret = get("Raw")(headers, content, limit) msg.append("Couldn't parse: falling back to Raw") else: msg.append(ret[0]) diff --git a/libmproxy/dump.py b/libmproxy/dump.py index bf409803..17b47dd2 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -174,7 +174,7 @@ class DumpMaster(flow.FlowMaster): def _print_message(self, message): if self.o.flow_detail >= 2: - print(self.indent(4, message.headers.format()), file=self.outfile) + print(self.indent(4, str(message.headers)), file=self.outfile) if self.o.flow_detail >= 3: if message.content == CONTENT_MISSING: print(self.indent(4, "(content missing)"), file=self.outfile) diff --git a/libmproxy/exceptions.py b/libmproxy/exceptions.py index f34d9707..0e11c136 100644 --- a/libmproxy/exceptions.py +++ b/libmproxy/exceptions.py @@ -1,23 +1,42 @@ +""" +We try to be very hygienic regarding the exceptions we throw: +Every Exception mitmproxy raises shall be a subclass of ProxyException. + + +See also: http://lucumr.pocoo.org/2014/10/16/on-error-handling/ +""" from __future__ import (absolute_import, print_function, division) class ProxyException(Exception): """ Base class for all exceptions thrown by libmproxy. + + Args: + message: the error message + cause: (optional) an error object that caused this exception, e.g. an IOError. """ - def __init__(self, message, cause=None): + def __init__(self, message): """ :param message: Error Message - :param cause: Exception object that caused this exception to be thrown. """ super(ProxyException, self).__init__(message) - self.cause = cause class ProtocolException(ProxyException): pass +class TlsException(ProtocolException): + pass + + +class ClientHandshakeException(TlsException): + def __init__(self, message, server): + super(ClientHandshakeException, self).__init__(message) + self.server = server + + class Socks5Exception(ProtocolException): pass diff --git a/libmproxy/filt.py b/libmproxy/filt.py index cfd3a1bc..7cd0f4df 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -35,7 +35,6 @@ from __future__ import absolute_import import re import sys import pyparsing as pp -from .models import decoded class _Token: @@ -78,17 +77,19 @@ class FResp(_Action): class _Rex(_Action): + flags = 0 + def __init__(self, expr): self.expr = expr try: - self.re = re.compile(self.expr) + self.re = re.compile(self.expr, self.flags) except: raise ValueError("Cannot compile expression.") def _check_content_type(expr, o): - val = o.headers["content-type"] - if val and re.search(expr, val[0]): + val = o.headers.get("content-type") + if val and re.search(expr, val): return True return False @@ -146,11 +147,12 @@ class FResponseContentType(_Rex): class FHead(_Rex): code = "h" help = "Header" + flags = re.MULTILINE def __call__(self, f): - if f.request.headers.match_re(self.expr): + if f.request and self.re.search(str(f.request.headers)): return True - elif f.response and f.response.headers.match_re(self.expr): + if f.response and self.re.search(str(f.response.headers)): return True return False @@ -158,18 +160,20 @@ class FHead(_Rex): class FHeadRequest(_Rex): code = "hq" help = "Request header" + flags = re.MULTILINE def __call__(self, f): - if f.request.headers.match_re(self.expr): + if f.request and self.re.search(str(f.request.headers)): return True class FHeadResponse(_Rex): code = "hs" help = "Response header" + flags = re.MULTILINE def __call__(self, f): - if f.response and f.response.headers.match_re(self.expr): + if f.response and self.re.search(str(f.response.headers)): return True @@ -179,13 +183,11 @@ class FBod(_Rex): def __call__(self, f): if f.request and f.request.content: - with decoded(f.request): - if re.search(self.expr, f.request.content): - return True + if self.re.search(f.request.get_decoded_content()): + return True if f.response and f.response.content: - with decoded(f.response): - if re.search(self.expr, f.response.content): - return True + if self.re.search(f.response.get_decoded_content()): + return True return False @@ -195,9 +197,8 @@ class FBodRequest(_Rex): def __call__(self, f): if f.request and f.request.content: - with decoded(f.request): - if re.search(self.expr, f.request.content): - return True + if self.re.search(f.request.get_decoded_content()): + return True class FBodResponse(_Rex): @@ -206,25 +207,26 @@ class FBodResponse(_Rex): def __call__(self, f): if f.response and f.response.content: - with decoded(f.response): - if re.search(self.expr, f.response.content): - return True + if self.re.search(f.response.get_decoded_content()): + return True class FMethod(_Rex): code = "m" help = "Method" + flags = re.IGNORECASE def __call__(self, f): - return bool(re.search(self.expr, f.request.method, re.IGNORECASE)) + return bool(self.re.search(f.request.method)) class FDomain(_Rex): code = "d" help = "Domain" + flags = re.IGNORECASE def __call__(self, f): - return bool(re.search(self.expr, f.request.host, re.IGNORECASE)) + return bool(self.re.search(f.request.host)) class FUrl(_Rex): @@ -239,21 +241,24 @@ class FUrl(_Rex): return klass(*toks) def __call__(self, f): - return re.search(self.expr, f.request.url) + return self.re.search(f.request.url) + class FSrc(_Rex): code = "src" help = "Match source address" def __call__(self, f): - return f.client_conn.address and re.search(self.expr, repr(f.client_conn.address)) + return f.client_conn.address and self.re.search(repr(f.client_conn.address)) + class FDst(_Rex): code = "dst" help = "Match destination address" def __call__(self, f): - return f.server_conn.address and re.search(self.expr, repr(f.server_conn.address)) + return f.server_conn.address and self.re.search(repr(f.server_conn.address)) + class _Int(_Action): def __init__(self, num): diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 5eac8da9..d037d36e 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -11,8 +11,8 @@ import re import urlparse -from netlib import odict, wsgi -from netlib.http.semantics import CONTENT_MISSING +from netlib import wsgi +from netlib.http.semantics import CONTENT_MISSING, Headers import netlib.http from . import controller, tnetstring, filt, script, version from .onboarding import app @@ -45,7 +45,7 @@ class AppRegistry: if (request.host, request.port) in self.apps: return self.apps[(request.host, request.port)] if "host" in request.headers: - host = request.headers["host"][0] + host = request.headers["host"] return self.apps.get((host, request.port), None) @@ -144,15 +144,15 @@ class SetHeaders: for _, header, value, cpatt in self.lst: if cpatt(f): if f.response: - del f.response.headers[header] + f.response.headers.pop(header, None) else: - del f.request.headers[header] + f.request.headers.pop(header, None) for _, header, value, cpatt in self.lst: if cpatt(f): if f.response: - f.response.headers.add(header, value) + f.response.headers.fields.append((header, value)) else: - f.request.headers.add(header, value) + f.request.headers.fields.append((header, value)) class StreamLargeBodies(object): @@ -278,14 +278,11 @@ class ServerPlaybackState: key.append(p[1]) if self.headers: - hdrs = [] + headers = [] for i in self.headers: - v = r.headers[i] - # Slightly subtle: we need to convert everything to strings - # to prevent a mismatch between unicode/non-unicode. - v = [str(x) for x in v] - hdrs.append((i, v)) - key.append(hdrs) + v = r.headers.get(i) + headers.append((i, v)) + key.append(headers) return hashlib.sha256(repr(key)).digest() def next_flow(self, request): @@ -329,7 +326,7 @@ class StickyCookieState: return False def handle_response(self, f): - for i in f.response.headers["set-cookie"]: + for i in f.response.headers.get_all("set-cookie"): # FIXME: We now know that Cookie.py screws up some cookies with # valid RFC 822/1123 datetime specifications for expiry. Sigh. c = Cookie.SimpleCookie(str(i)) @@ -351,7 +348,7 @@ class StickyCookieState: l.append(self.jar[i].output(header="").strip()) if l: f.request.stickycookie = True - f.request.headers["cookie"] = l + f.request.headers.set_all("cookie",l) class StickyAuthState: @@ -836,7 +833,7 @@ class FlowMaster(controller.Master): ssl_established=True )) f = HTTPFlow(c, s) - headers = odict.ODictCaseless() + headers = Headers() req = HTTPRequest( "absolute", @@ -930,8 +927,7 @@ class FlowMaster(controller.Master): f.backup() f.request.is_replay = True if f.request.content: - f.request.headers[ - "Content-Length"] = [str(len(f.request.content))] + f.request.headers["Content-Length"] = str(len(f.request.content)) f.response = None f.error = None self.process_new_request(f) @@ -949,21 +945,25 @@ class FlowMaster(controller.Master): self.add_event(l.msg, l.level) l.reply() - def handle_clientconnect(self, cc): - self.run_script_hook("clientconnect", cc) - cc.reply() + def handle_clientconnect(self, root_layer): + self.run_script_hook("clientconnect", root_layer) + root_layer.reply() - def handle_clientdisconnect(self, r): - self.run_script_hook("clientdisconnect", r) - r.reply() + def handle_clientdisconnect(self, root_layer): + self.run_script_hook("clientdisconnect", root_layer) + root_layer.reply() - def handle_serverconnect(self, sc): - self.run_script_hook("serverconnect", sc) - sc.reply() + def handle_serverconnect(self, server_conn): + self.run_script_hook("serverconnect", server_conn) + server_conn.reply() - def handle_serverdisconnect(self, sc): - self.run_script_hook("serverdisconnect", sc) - sc.reply() + def handle_serverdisconnect(self, server_conn): + self.run_script_hook("serverdisconnect", server_conn) + server_conn.reply() + + def handle_next_layer(self, top_layer): + self.run_script_hook("next_layer", top_layer) + top_layer.reply() def handle_error(self, f): self.state.update_flow(f) diff --git a/libmproxy/models/http.py b/libmproxy/models/http.py index fb2f305b..0d5e53b5 100644 --- a/libmproxy/models/http.py +++ b/libmproxy/models/http.py @@ -5,8 +5,8 @@ from email.utils import parsedate_tz, formatdate, mktime_tz import time from libmproxy import utils -from netlib import odict, encoding -from netlib.http import status_codes +from netlib import encoding +from netlib.http import status_codes, Headers from netlib.tcp import Address from netlib.http.semantics import Request, Response, CONTENT_MISSING from .. import version, stateobject @@ -16,7 +16,7 @@ from .flow import Flow class MessageMixin(stateobject.StateObject): _stateobject_attributes = dict( httpversion=tuple, - headers=odict.ODictCaseless, + headers=Headers, body=str, timestamp_start=float, timestamp_end=float @@ -40,7 +40,7 @@ class MessageMixin(stateobject.StateObject): header. Doesn't change the message iteself or its headers. """ - ce = self.headers.get_first("content-encoding") + ce = self.headers.get("content-encoding") if not self.body or ce not in encoding.ENCODINGS: return self.body return encoding.decode(ce, self.body) @@ -53,14 +53,14 @@ class MessageMixin(stateobject.StateObject): Returns True if decoding succeeded, False otherwise. """ - ce = self.headers.get_first("content-encoding") + ce = self.headers.get("content-encoding") if not self.body or ce not in encoding.ENCODINGS: return False data = encoding.decode(ce, self.body) if data is None: return False self.body = data - del self.headers["content-encoding"] + self.headers.pop("content-encoding", None) return True def encode(self, e): @@ -70,7 +70,7 @@ class MessageMixin(stateobject.StateObject): """ # FIXME: Error if there's an existing encoding header? self.body = encoding.encode(e, self.body) - self.headers["content-encoding"] = [e] + self.headers["content-encoding"] = e def copy(self): c = copy.copy(self) @@ -86,11 +86,18 @@ class MessageMixin(stateobject.StateObject): Returns the number of replacements made. """ with decoded(self): - self.body, c = utils.safe_subn( + self.body, count = utils.safe_subn( pattern, repl, self.body, *args, **kwargs ) - c += self.headers.replace(pattern, repl, *args, **kwargs) - return c + fields = [] + for name, value in self.headers.fields: + name, c = utils.safe_subn(pattern, repl, name, *args, **kwargs) + count += c + value, c = utils.safe_subn(pattern, repl, value, *args, **kwargs) + count += c + fields.append([name, value]) + self.headers.fields = fields + return count class HTTPRequest(MessageMixin, Request): @@ -115,7 +122,7 @@ class HTTPRequest(MessageMixin, Request): httpversion: HTTP version tuple, e.g. (1,1) - headers: odict.ODictCaseless object + headers: Headers object content: Content of the request, None, or CONTENT_MISSING if there is content associated, but not present. CONTENT_MISSING evaluates @@ -266,7 +273,7 @@ class HTTPResponse(MessageMixin, Response): msg: HTTP response message - headers: ODict Caseless object + headers: Headers object content: Content of the request, None, or CONTENT_MISSING if there is content associated, but not present. CONTENT_MISSING evaluates @@ -379,15 +386,15 @@ class HTTPResponse(MessageMixin, Response): ] for i in refresh_headers: if i in self.headers: - d = parsedate_tz(self.headers[i][0]) + d = parsedate_tz(self.headers[i]) if d: new = mktime_tz(d) + delta - self.headers[i] = [formatdate(new)] + self.headers[i] = formatdate(new) c = [] - for i in self.headers["set-cookie"]: + for i in self.headers.get_all("set-cookie"): c.append(self._refresh_cookie(i, delta)) if c: - self.headers["set-cookie"] = c + self.headers.set_all("set-cookie", c) class HTTPFlow(Flow): @@ -490,7 +497,7 @@ class decoded(object): def __init__(self, o): self.o = o - ce = o.headers.get_first("content-encoding") + ce = o.headers.get("content-encoding") if ce in encoding.ENCODINGS: self.ce = ce else: @@ -517,11 +524,12 @@ def make_error_response(status_code, message, headers=None): """.strip() % (status_code, response, message) if not headers: - headers = odict.ODictCaseless() - headers["Server"] = [version.NAMEVERSION] - headers["Connection"] = ["close"] - headers["Content-Length"] = [len(body)] - headers["Content-Type"] = ["text/html"] + headers = Headers( + Server=version.NAMEVERSION, + Connection="close", + Content_Length=str(len(body)), + Content_Type="text/html" + ) return HTTPResponse( (1, 1), # FIXME: Should be a string. @@ -536,15 +544,15 @@ def make_connect_request(address): address = Address.wrap(address) return HTTPRequest( "authority", "CONNECT", None, address.host, address.port, None, (1, 1), - odict.ODictCaseless(), "" + Headers(), "" ) def make_connect_response(httpversion): - headers = odict.ODictCaseless([ - ["Content-Length", "0"], - ["Proxy-Agent", version.NAMEVERSION] - ]) + headers = Headers( + Content_Length="0", + Proxy_Agent=version.NAMEVERSION + ) return HTTPResponse( httpversion, 200, diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index c582592b..35d59f28 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -1,11 +1,38 @@ +""" +In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. +The first layer is usually the proxy mode, e.g. transparent proxy or normal HTTP proxy. Next, +various protocol layers are stacked on top of each other - imagine WebSockets on top of an HTTP +Upgrade request. An actual mitmproxy connection may look as follows (outermost layer first): + + Transparent HTTP proxy, no TLS: + - TransparentProxy + - Http1Layer + - HttpLayer + + Regular proxy, CONNECT request with WebSockets over SSL: + - ReverseProxy + - Http1Layer + - HttpLayer + - TLSLayer + - WebsocketLayer (or TCPLayer) + +Every layer acts as a read-only context for its inner layers (see :py:class:`Layer`). To communicate +with an outer layer, a layer can use functions provided in the context. The next layer is always +determined by a call to :py:meth:`.next_layer() <libmproxy.proxy.RootContext.next_layer>`, +which is provided by the root context. + +Another subtle design goal of this architecture is that upstream connections should be established +as late as possible; this makes server replay without any outgoing connections possible. +""" + from __future__ import (absolute_import, print_function, division) -from .base import Layer, ServerConnectionMixin, Log, Kill +from .base import Layer, ServerConnectionMixin, Kill from .http import Http1Layer, Http2Layer from .tls import TlsLayer, is_tls_record_magic from .rawtcp import RawTCPLayer __all__ = [ - "Layer", "ServerConnectionMixin", "Log", "Kill", + "Layer", "ServerConnectionMixin", "Kill", "Http1Layer", "Http2Layer", "TlsLayer", "is_tls_record_magic", "RawTCPLayer" diff --git a/libmproxy/protocol/base.py b/libmproxy/protocol/base.py index 40ec0536..b92aeea1 100644 --- a/libmproxy/protocol/base.py +++ b/libmproxy/protocol/base.py @@ -1,38 +1,8 @@ -""" -mitmproxy protocol architecture - -In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. -For example, the following scenarios depict possible settings (lowest layer first): - -Transparent HTTP proxy, no SSL: - TransparentProxy - Http1Layer - HttpLayer - -Regular proxy, CONNECT request with WebSockets over SSL: - HttpProxy - Http1Layer - HttpLayer - SslLayer - WebsocketLayer (or TcpLayer) - -Automated protocol detection by peeking into the buffer: - TransparentProxy - TLSLayer - Http2Layer - HttpLayer - -Communication between layers is done as follows: - - lower layers provide context information to higher layers - - higher layers can call functions provided by lower layers, - which are propagated until they reach a suitable layer. - -Further goals: - - Connections should always be peekable to make automatic protocol detection work. - - Upstream connections should be established as late as possible; - inline scripts shall have a chance to handle everything locally. -""" from __future__ import (absolute_import, print_function, division) +import sys + +import six + from netlib import tcp from ..models import ServerConnection from ..exceptions import ProtocolException @@ -43,8 +13,8 @@ class _LayerCodeCompletion(object): Dummy class that provides type hinting in PyCharm, which simplifies development a lot. """ - def __init__(self, *args, **kwargs): # pragma: nocover - super(_LayerCodeCompletion, self).__init__(*args, **kwargs) + def __init__(self, **mixin_args): # pragma: nocover + super(_LayerCodeCompletion, self).__init__(**mixin_args) if True: return self.config = None @@ -55,43 +25,64 @@ class _LayerCodeCompletion(object): """@type: libmproxy.models.ServerConnection""" self.channel = None """@type: libmproxy.controller.Channel""" + self.ctx = None + """@type: libmproxy.protocol.Layer""" class Layer(_LayerCodeCompletion): - def __init__(self, ctx, *args, **kwargs): + """ + Base class for all layers. All other protocol layers should inherit from this class. + """ + + def __init__(self, ctx, **mixin_args): """ + Each layer usually passes itself to its child layers as a context. Properties of the + context are transparently mapped to the layer, so that the following works: + + .. code-block:: python + + root_layer = Layer(None) + root_layer.client_conn = 42 + sub_layer = Layer(root_layer) + print(sub_layer.client_conn) # 42 + + The root layer is passed a :py:class:`libmproxy.proxy.RootContext` object, + which provides access to :py:attr:`.client_conn <libmproxy.proxy.RootContext.client_conn>`, + :py:attr:`.next_layer <libmproxy.proxy.RootContext.next_layer>` and other basic attributes. + Args: - ctx: The (read-only) higher layer. + ctx: The (read-only) parent layer / context. """ self.ctx = ctx - """@type: libmproxy.protocol.Layer""" - super(Layer, self).__init__(*args, **kwargs) + """ + The parent layer. - def __call__(self): + :type: :py:class:`Layer` """ - Logic of the layer. + super(Layer, self).__init__(**mixin_args) + + def __call__(self): + """Logic of the layer. + + Returns: + Once the protocol has finished without exceptions. + Raises: - ProtocolException in case of protocol exceptions. + ~libmproxy.exceptions.ProtocolException: if an exception occurs. No other exceptions must be raised. """ raise NotImplementedError() def __getattr__(self, name): """ - Attributes not present on the current layer may exist on a higher layer. + Attributes not present on the current layer are looked up on the context. """ return getattr(self.ctx, name) - def log(self, msg, level, subs=()): - full_msg = [ - "{}: {}".format(repr(self.client_conn.address), msg) - ] - for i in subs: - full_msg.append(" -> " + i) - full_msg = "\n".join(full_msg) - self.channel.tell("log", Log(full_msg, level)) - @property def layers(self): + """ + List of all layers, including the current layer (``[self, self.ctx, self.ctx.ctx, ...]``) + """ return [self] + self.ctx.layers def __repr__(self): @@ -101,20 +92,28 @@ class Layer(_LayerCodeCompletion): class ServerConnectionMixin(object): """ Mixin that provides a layer with the capabilities to manage a server connection. + The server address can be passed in the constructor or set by calling :py:meth:`set_server`. + Subclasses are responsible for calling :py:meth:`disconnect` before returning. + + Recommended Usage: + + .. code-block:: python + + class MyLayer(Layer, ServerConnectionMixin): + def __call__(self): + try: + # Do something. + finally: + if self.server_conn: + self.disconnect() """ def __init__(self, server_address=None): super(ServerConnectionMixin, self).__init__() self.server_conn = ServerConnection(server_address) - self._check_self_connect() - - def reconnect(self): - address = self.server_conn.address - self._disconnect() - self.server_conn.address = address - self.connect() + self.__check_self_connect() - def _check_self_connect(self): + def __check_self_connect(self): """ We try to protect the proxy from _accidentally_ connecting to itself, e.g. because of a failed transparent lookup or an invalid configuration. @@ -131,31 +130,45 @@ class ServerConnectionMixin(object): "The proxy shall not connect to itself.".format(repr(address)) ) - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1: - if self.server_conn: - self._disconnect() - self.log("Set new server address: " + repr(address), "debug") - self.server_conn.address = address - self._check_self_connect() - if server_tls: - raise ProtocolException( - "Cannot upgrade to TLS, no TLS layer on the protocol stack." - ) - else: - self.ctx.set_server(address, server_tls, sni, depth - 1) + def set_server(self, address, server_tls=None, sni=None): + """ + Sets a new server address. If there is an existing connection, it will be closed. + + Raises: + ~libmproxy.exceptions.ProtocolException: + if ``server_tls`` is ``True``, but there was no TLS layer on the + protocol stack which could have processed this. + """ + if self.server_conn: + self.disconnect() + self.log("Set new server address: " + repr(address), "debug") + self.server_conn.address = address + self.__check_self_connect() + if server_tls: + raise ProtocolException( + "Cannot upgrade to TLS, no TLS layer on the protocol stack." + ) - def _disconnect(self): + def disconnect(self): """ Deletes (and closes) an existing server connection. + Must not be called if there is no existing connection. """ self.log("serverdisconnect", "debug", [repr(self.server_conn.address)]) + address = self.server_conn.address self.server_conn.finish() self.server_conn.close() self.channel.tell("serverdisconnect", self.server_conn) - self.server_conn = ServerConnection(None) + self.server_conn = ServerConnection(address) def connect(self): + """ + Establishes a server connection. + Must not be called if there is an existing connection. + + Raises: + ~libmproxy.exceptions.ProtocolException: if the connection could not be established. + """ if not self.server_conn.address: raise ProtocolException("Cannot connect to server, no server address given.") self.log("serverconnect", "debug", [repr(self.server_conn.address)]) @@ -163,17 +176,18 @@ class ServerConnectionMixin(object): try: self.server_conn.connect() except tcp.NetLibError as e: - raise ProtocolException( - "Server connection to %s failed: %s" % (repr(self.server_conn.address), e), e) - - -class Log(object): - def __init__(self, msg, level="info"): - self.msg = msg - self.level = level + six.reraise( + ProtocolException, + ProtocolException( + "Server connection to {} failed: {}".format( + repr(self.server_conn.address), str(e) + ) + ), + sys.exc_info()[2] + ) class Kill(Exception): """ - Kill a connection. + Signal that both client and server connection(s) should be killed immediately. """ diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 7f57d17c..3a415320 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,14 +1,16 @@ from __future__ import (absolute_import, print_function, division) +import itertools +import sys + +import six from netlib import tcp -from netlib.http import http1, HttpErrorConnClosed, HttpError +from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers from netlib.http.semantics import CONTENT_MISSING -from netlib import odict from netlib.tcp import NetLibError, Address from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol -from netlib.http.http2.frame import WindowUpdateFrame - +from netlib.http.http2.frame import GoAwayFrame, PriorityFrame, WindowUpdateFrame from .. import utils from ..exceptions import InvalidCredentials, HttpException, ProtocolException from ..models import ( @@ -32,6 +34,9 @@ class _HttpLayer(Layer): def send_response(self, response): raise NotImplementedError() + def check_close_connection(self, flow): + raise NotImplementedError() + class _StreamingHttpLayer(_HttpLayer): supports_streaming = True @@ -43,12 +48,25 @@ class _StreamingHttpLayer(_HttpLayer): raise NotImplementedError() yield "this is a generator" # pragma: no cover + def read_response(self, request_method): + response = self.read_response_headers() + response.body = "".join( + self.read_response_body(response.headers, request_method, response.code) + ) + return response + def send_response_headers(self, response): raise NotImplementedError def send_response_body(self, response, chunks): raise NotImplementedError() + def send_response(self, response): + if response.body == CONTENT_MISSING: + raise HttpError(502, "Cannot assemble flow with CONTENT_MISSING") + self.send_response_headers(response) + self.send_response_body(response, [response.body]) + class Http1Layer(_StreamingHttpLayer): def __init__(self, ctx, mode): @@ -66,17 +84,6 @@ class Http1Layer(_StreamingHttpLayer): def send_request(self, request): self.server_conn.send(self.server_protocol.assemble(request)) - def read_response(self, request_method): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=request_method, - body_size_limit=self.config.body_size_limit, - include_body=True - ) - - def send_response(self, response): - self.client_conn.send(self.client_protocol.assemble(response)) - def read_response_headers(self): return HTTPResponse.from_protocol( self.server_protocol, @@ -102,25 +109,49 @@ class Http1Layer(_StreamingHttpLayer): response, preserve_transfer_encoding=True ) - self.client_conn.send(h + "\r\n") + self.client_conn.wfile.write(h + "\r\n") + self.client_conn.wfile.flush() def send_response_body(self, response, chunks): if self.client_protocol.has_chunked_encoding(response.headers): - chunks = ( - "%d\r\n%s\r\n" % (len(chunk), chunk) - for chunk in chunks + chunks = itertools.chain( + ( + "{:x}\r\n{}\r\n".format(len(chunk), chunk) + for chunk in chunks if chunk + ), + ("0\r\n\r\n",) ) for chunk in chunks: - self.client_conn.send(chunk) + self.client_conn.wfile.write(chunk) + self.client_conn.wfile.flush() + + def check_close_connection(self, flow): + close_connection = ( + http1.HTTP1Protocol.connection_close( + flow.request.httpversion, + flow.request.headers + ) or http1.HTTP1Protocol.connection_close( + flow.response.httpversion, + flow.response.headers + ) or http1.HTTP1Protocol.expected_http_body_size( + flow.response.headers, + False, + flow.request.method, + flow.response.code) == -1 + ) + if flow.request.form_in == "authority" and flow.response.code == 200: + # Workaround for + # https://github.com/mitmproxy/mitmproxy/issues/313: Some + # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 + # and no Content-Length header + + return False + return close_connection def connect(self): self.ctx.connect() self.server_protocol = HTTP1Protocol(self.server_conn) - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP1Protocol(self.server_conn) - def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) self.server_protocol = HTTP1Protocol(self.server_conn) @@ -136,9 +167,9 @@ class Http2Layer(_HttpLayer): super(Http2Layer, self).__init__(ctx) self.mode = mode self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_client) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_server) def read_request(self): request = HTTPRequest.from_protocol( @@ -162,25 +193,24 @@ class Http2Layer(_HttpLayer): ) def send_response(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames + # TODO: implement flow control to prevent client buffer filling up + # maintain a send buffer size, and read WindowUpdateFrames from client to increase the send buffer self.client_conn.send(self.client_protocol.assemble(message)) + def check_close_connection(self, flow): + # TODO: add a timer to disconnect after a 10 second timeout + return False + def connect(self): self.ctx.connect() self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_server) self.server_protocol.perform_connection_preface() def set_server(self, *args, **kwargs): self.ctx.set_server(*args, **kwargs) self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) + unhandled_frame_cb=self.handle_unexpected_frame_from_server) self.server_protocol.perform_connection_preface() def __call__(self): @@ -188,7 +218,10 @@ class Http2Layer(_HttpLayer): layer = HttpLayer(self, self.mode) layer() - def handle_unexpected_frame(self, frame): + # terminate the connection + self.client_conn.send(GoAwayFrame().to_bytes()) + + def handle_unexpected_frame_from_client(self, frame): if isinstance(frame, WindowUpdateFrame): # Clients are sending WindowUpdate frames depending on their flow control algorithm. # Since we cannot predict these frames, and we do not need to respond to them, @@ -196,7 +229,34 @@ class Http2Layer(_HttpLayer): # Ideally we should keep track of our own flow control window and # stall transmission if the outgoing flow control buffer is full. return - self.log("Unexpected HTTP2 Frame: %s" % frame.human_readable(), "info") + if isinstance(frame, PriorityFrame): + # Clients are sending Priority frames depending on their implementation. + # The RFC does not clearly state when or which priority preferences should be set. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should forward them to the server. + return + if isinstance(frame, GoAwayFrame): + # Client wants to terminate the connection, + # relay it to the server. + self.server_conn.send(frame.to_bytes()) + return + self.log("Unexpected HTTP2 frame from client: %s" % frame.human_readable(), "info") + + def handle_unexpected_frame_from_server(self, frame): + if isinstance(frame, WindowUpdateFrame): + # Servers are sending WindowUpdate frames depending on their flow control algorithm. + # Since we cannot predict these frames, and we do not need to respond to them, + # simply accept them, and hide them from the log. + # Ideally we should keep track of our own flow control window and + # stall transmission if the outgoing flow control buffer is full. + return + if isinstance(frame, GoAwayFrame): + # Server wants to terminate the connection, + # relay it to the client. + self.client_conn.send(frame.to_bytes()) + return + self.log("Unexpected HTTP2 frame from server: %s" % frame.human_readable(), "info") class ConnectServerConnection(object): @@ -245,20 +305,22 @@ class UpstreamConnectLayer(Layer): else: pass # swallow the message - def reconnect(self): - self.ctx.reconnect() - self._send_connect_request() - - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1: - if self.ctx.server_conn: - self.ctx.reconnect() - address = Address.wrap(address) - self.connect_request.host = address.host - self.connect_request.port = address.port - self.server_conn.address = address - else: - self.ctx.set_server(address, server_tls, sni, depth - 1) + def change_upstream_proxy_server(self, address): + if address != self.server_conn.via.address: + self.ctx.set_server(address) + + def set_server(self, address, server_tls=None, sni=None): + if self.ctx.server_conn: + self.ctx.disconnect() + address = Address.wrap(address) + self.connect_request.host = address.host + self.connect_request.port = address.port + self.server_conn.address = address + + if server_tls: + raise ProtocolException( + "Cannot upgrade to TLS, no TLS layer on the protocol stack." + ) class HttpLayer(Layer): @@ -308,7 +370,13 @@ class HttpLayer(Layer): if self.check_close_connection(flow): return - # TODO: Implement HTTP Upgrade + # Handle 101 Switching Protocols + # It may be useful to pass additional args (such as the upgrade header) + # to next_layer in the future + if flow.response.status_code == 101: + layer = self.ctx.next_layer(self) + layer() + return # Upstream Proxy Mode: Handle CONNECT if flow.request.form_in == "authority" and flow.response.code == 200: @@ -327,12 +395,18 @@ class HttpLayer(Layer): except NetLibError: pass if isinstance(e, ProtocolException): - raise e + six.reraise(ProtocolException, e, sys.exc_info()[2]) else: - raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) + six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) finally: flow.live = False + def change_upstream_proxy_server(self, address): + # Make set_upstream_proxy_server always available, + # even if there's no UpstreamConnectLayer + if address != self.server_conn.address: + return self.set_server(address) + def handle_regular_mode_connect(self, request): self.set_server((request.host, request.port)) self.send_response(make_connect_response(request.httpversion)) @@ -343,36 +417,6 @@ class HttpLayer(Layer): layer = UpstreamConnectLayer(self, connect_request) layer() - def check_close_connection(self, flow): - """ - Checks if the connection should be closed depending on the HTTP - semantics. Returns True, if so. - """ - - # TODO: add logic for HTTP/2 - - close_connection = ( - http1.HTTP1Protocol.connection_close( - flow.request.httpversion, - flow.request.headers - ) or http1.HTTP1Protocol.connection_close( - flow.response.httpversion, - flow.response.headers - ) or http1.HTTP1Protocol.expected_http_body_size( - flow.response.headers, - False, - flow.request.method, - flow.response.code) == -1 - ) - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 - # and no Content-Length header - - return False - return close_connection - def send_response_to_client(self, flow): if not (self.supports_streaming and flow.response.stream): # no streaming: @@ -420,7 +464,8 @@ class HttpLayer(Layer): # > server detects timeout, disconnects # > read (100-n)% of large request # > send large request upstream - self.reconnect() + self.disconnect() + self.connect() get_response() # call the appropriate script hook - this is an opportunity for an @@ -482,7 +527,7 @@ class HttpLayer(Layer): if self.mode == "regular" or self.mode == "transparent": # If there's an existing connection that doesn't match our expectations, kill it. - if address != self.server_conn.address or tls != self.server_conn.ssl_established: + if address != self.server_conn.address or tls != self.server_conn.tls_established: self.set_server(address, tls, address.host) # Establish connection is neccessary. if not self.server_conn: @@ -495,10 +540,12 @@ class HttpLayer(Layer): """ # This is a very ugly (untested) workaround to solve a very ugly problem. if self.server_conn and self.server_conn.tls_established and not ssl: - self.reconnect() + self.disconnect() + self.connect() elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: if self.server_conn.tls_established: - self.reconnect() + self.disconnect() + self.connect() self.send_request(make_connect_request(address)) tls_layer = TlsLayer(self, False, True) @@ -536,10 +583,6 @@ class HttpLayer(Layer): self.send_response(make_error_response( 407, "Proxy Authentication Required", - odict.ODictCaseless( - [ - [k, v] for k, v in - self.config.authenticator.auth_challenge_headers().items() - ]) + Headers(**self.config.authenticator.auth_challenge_headers()) )) raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py index 2759a019..a9ee5506 100644 --- a/libmproxy/protocol/http_replay.py +++ b/libmproxy/protocol/http_replay.py @@ -6,7 +6,7 @@ from netlib.http.http1 import HTTP1Protocol from netlib.tcp import NetLibError from ..controller import Channel from ..models import Error, HTTPResponse, ServerConnection, make_connect_request -from .base import Log, Kill +from .base import Kill # TODO: Doesn't really belong into libmproxy.protocol... @@ -89,8 +89,9 @@ class RequestReplayThread(threading.Thread): if self.channel: self.channel.ask("error", self.flow) except Kill: - # KillSignal should only be raised if there's a channel in the + # Kill should only be raised if there's a channel in the # first place. + from ..proxy.root_context import Log self.channel.tell("log", Log("Connection killed", "info")) finally: r.form_out = form_out_backup diff --git a/libmproxy/protocol/rawtcp.py b/libmproxy/protocol/rawtcp.py index 86468773..9b155412 100644 --- a/libmproxy/protocol/rawtcp.py +++ b/libmproxy/protocol/rawtcp.py @@ -1,10 +1,12 @@ from __future__ import (absolute_import, print_function, division) import socket import select +import six +import sys from OpenSSL import SSL -from netlib.tcp import NetLibError +from netlib.tcp import NetLibError, ssl_read_select from netlib.utils import cleanBin from ..exceptions import ProtocolException from .base import Layer @@ -28,7 +30,7 @@ class RawTCPLayer(Layer): try: while True: - r, _, _ = select.select(conns, [], [], 10) + r = ssl_read_select(conns, 10) for conn in r: dst = server if conn == client else client @@ -63,4 +65,8 @@ class RawTCPLayer(Layer): ) except (socket.error, NetLibError, SSL.Error) as e: - raise ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e)), e) + six.reraise( + ProtocolException, + ProtocolException("TCP connection closed unexpectedly: {}".format(repr(e))), + sys.exc_info()[2] + ) diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index a8dc8bb2..4f7c9300 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -1,16 +1,210 @@ from __future__ import (absolute_import, print_function, division) import struct +import sys from construct import ConstructError +import six from netlib.tcp import NetLibError, NetLibInvalidCertificateError from netlib.http.http1 import HTTP1Protocol from ..contrib.tls._constructs import ClientHello -from ..exceptions import ProtocolException +from ..exceptions import ProtocolException, TlsException, ClientHandshakeException from .base import Layer + +# taken from https://testssl.sh/openssl-rfc.mappping.html +CIPHER_ID_NAME_MAP = { + 0x00: 'NULL-MD5', + 0x01: 'NULL-MD5', + 0x02: 'NULL-SHA', + 0x03: 'EXP-RC4-MD5', + 0x04: 'RC4-MD5', + 0x05: 'RC4-SHA', + 0x06: 'EXP-RC2-CBC-MD5', + 0x07: 'IDEA-CBC-SHA', + 0x08: 'EXP-DES-CBC-SHA', + 0x09: 'DES-CBC-SHA', + 0x0a: 'DES-CBC3-SHA', + 0x0b: 'EXP-DH-DSS-DES-CBC-SHA', + 0x0c: 'DH-DSS-DES-CBC-SHA', + 0x0d: 'DH-DSS-DES-CBC3-SHA', + 0x0e: 'EXP-DH-RSA-DES-CBC-SHA', + 0x0f: 'DH-RSA-DES-CBC-SHA', + 0x10: 'DH-RSA-DES-CBC3-SHA', + 0x11: 'EXP-EDH-DSS-DES-CBC-SHA', + 0x12: 'EDH-DSS-DES-CBC-SHA', + 0x13: 'EDH-DSS-DES-CBC3-SHA', + 0x14: 'EXP-EDH-RSA-DES-CBC-SHA', + 0x15: 'EDH-RSA-DES-CBC-SHA', + 0x16: 'EDH-RSA-DES-CBC3-SHA', + 0x17: 'EXP-ADH-RC4-MD5', + 0x18: 'ADH-RC4-MD5', + 0x19: 'EXP-ADH-DES-CBC-SHA', + 0x1a: 'ADH-DES-CBC-SHA', + 0x1b: 'ADH-DES-CBC3-SHA', + # 0x1c: , + # 0x1d: , + 0x1e: 'KRB5-DES-CBC-SHA', + 0x1f: 'KRB5-DES-CBC3-SHA', + 0x20: 'KRB5-RC4-SHA', + 0x21: 'KRB5-IDEA-CBC-SHA', + 0x22: 'KRB5-DES-CBC-MD5', + 0x23: 'KRB5-DES-CBC3-MD5', + 0x24: 'KRB5-RC4-MD5', + 0x25: 'KRB5-IDEA-CBC-MD5', + 0x26: 'EXP-KRB5-DES-CBC-SHA', + 0x27: 'EXP-KRB5-RC2-CBC-SHA', + 0x28: 'EXP-KRB5-RC4-SHA', + 0x29: 'EXP-KRB5-DES-CBC-MD5', + 0x2a: 'EXP-KRB5-RC2-CBC-MD5', + 0x2b: 'EXP-KRB5-RC4-MD5', + 0x2f: 'AES128-SHA', + 0x30: 'DH-DSS-AES128-SHA', + 0x31: 'DH-RSA-AES128-SHA', + 0x32: 'DHE-DSS-AES128-SHA', + 0x33: 'DHE-RSA-AES128-SHA', + 0x34: 'ADH-AES128-SHA', + 0x35: 'AES256-SHA', + 0x36: 'DH-DSS-AES256-SHA', + 0x37: 'DH-RSA-AES256-SHA', + 0x38: 'DHE-DSS-AES256-SHA', + 0x39: 'DHE-RSA-AES256-SHA', + 0x3a: 'ADH-AES256-SHA', + 0x3b: 'NULL-SHA256', + 0x3c: 'AES128-SHA256', + 0x3d: 'AES256-SHA256', + 0x3e: 'DH-DSS-AES128-SHA256', + 0x3f: 'DH-RSA-AES128-SHA256', + 0x40: 'DHE-DSS-AES128-SHA256', + 0x41: 'CAMELLIA128-SHA', + 0x42: 'DH-DSS-CAMELLIA128-SHA', + 0x43: 'DH-RSA-CAMELLIA128-SHA', + 0x44: 'DHE-DSS-CAMELLIA128-SHA', + 0x45: 'DHE-RSA-CAMELLIA128-SHA', + 0x46: 'ADH-CAMELLIA128-SHA', + 0x62: 'EXP1024-DES-CBC-SHA', + 0x63: 'EXP1024-DHE-DSS-DES-CBC-SHA', + 0x64: 'EXP1024-RC4-SHA', + 0x65: 'EXP1024-DHE-DSS-RC4-SHA', + 0x66: 'DHE-DSS-RC4-SHA', + 0x67: 'DHE-RSA-AES128-SHA256', + 0x68: 'DH-DSS-AES256-SHA256', + 0x69: 'DH-RSA-AES256-SHA256', + 0x6a: 'DHE-DSS-AES256-SHA256', + 0x6b: 'DHE-RSA-AES256-SHA256', + 0x6c: 'ADH-AES128-SHA256', + 0x6d: 'ADH-AES256-SHA256', + 0x80: 'GOST94-GOST89-GOST89', + 0x81: 'GOST2001-GOST89-GOST89', + 0x82: 'GOST94-NULL-GOST94', + 0x83: 'GOST2001-GOST89-GOST89', + 0x84: 'CAMELLIA256-SHA', + 0x85: 'DH-DSS-CAMELLIA256-SHA', + 0x86: 'DH-RSA-CAMELLIA256-SHA', + 0x87: 'DHE-DSS-CAMELLIA256-SHA', + 0x88: 'DHE-RSA-CAMELLIA256-SHA', + 0x89: 'ADH-CAMELLIA256-SHA', + 0x8a: 'PSK-RC4-SHA', + 0x8b: 'PSK-3DES-EDE-CBC-SHA', + 0x8c: 'PSK-AES128-CBC-SHA', + 0x8d: 'PSK-AES256-CBC-SHA', + # 0x8e: , + # 0x8f: , + # 0x90: , + # 0x91: , + # 0x92: , + # 0x93: , + # 0x94: , + # 0x95: , + 0x96: 'SEED-SHA', + 0x97: 'DH-DSS-SEED-SHA', + 0x98: 'DH-RSA-SEED-SHA', + 0x99: 'DHE-DSS-SEED-SHA', + 0x9a: 'DHE-RSA-SEED-SHA', + 0x9b: 'ADH-SEED-SHA', + 0x9c: 'AES128-GCM-SHA256', + 0x9d: 'AES256-GCM-SHA384', + 0x9e: 'DHE-RSA-AES128-GCM-SHA256', + 0x9f: 'DHE-RSA-AES256-GCM-SHA384', + 0xa0: 'DH-RSA-AES128-GCM-SHA256', + 0xa1: 'DH-RSA-AES256-GCM-SHA384', + 0xa2: 'DHE-DSS-AES128-GCM-SHA256', + 0xa3: 'DHE-DSS-AES256-GCM-SHA384', + 0xa4: 'DH-DSS-AES128-GCM-SHA256', + 0xa5: 'DH-DSS-AES256-GCM-SHA384', + 0xa6: 'ADH-AES128-GCM-SHA256', + 0xa7: 'ADH-AES256-GCM-SHA384', + 0x5600: 'TLS_FALLBACK_SCSV', + 0xc001: 'ECDH-ECDSA-NULL-SHA', + 0xc002: 'ECDH-ECDSA-RC4-SHA', + 0xc003: 'ECDH-ECDSA-DES-CBC3-SHA', + 0xc004: 'ECDH-ECDSA-AES128-SHA', + 0xc005: 'ECDH-ECDSA-AES256-SHA', + 0xc006: 'ECDHE-ECDSA-NULL-SHA', + 0xc007: 'ECDHE-ECDSA-RC4-SHA', + 0xc008: 'ECDHE-ECDSA-DES-CBC3-SHA', + 0xc009: 'ECDHE-ECDSA-AES128-SHA', + 0xc00a: 'ECDHE-ECDSA-AES256-SHA', + 0xc00b: 'ECDH-RSA-NULL-SHA', + 0xc00c: 'ECDH-RSA-RC4-SHA', + 0xc00d: 'ECDH-RSA-DES-CBC3-SHA', + 0xc00e: 'ECDH-RSA-AES128-SHA', + 0xc00f: 'ECDH-RSA-AES256-SHA', + 0xc010: 'ECDHE-RSA-NULL-SHA', + 0xc011: 'ECDHE-RSA-RC4-SHA', + 0xc012: 'ECDHE-RSA-DES-CBC3-SHA', + 0xc013: 'ECDHE-RSA-AES128-SHA', + 0xc014: 'ECDHE-RSA-AES256-SHA', + 0xc015: 'AECDH-NULL-SHA', + 0xc016: 'AECDH-RC4-SHA', + 0xc017: 'AECDH-DES-CBC3-SHA', + 0xc018: 'AECDH-AES128-SHA', + 0xc019: 'AECDH-AES256-SHA', + 0xc01a: 'SRP-3DES-EDE-CBC-SHA', + 0xc01b: 'SRP-RSA-3DES-EDE-CBC-SHA', + 0xc01c: 'SRP-DSS-3DES-EDE-CBC-SHA', + 0xc01d: 'SRP-AES-128-CBC-SHA', + 0xc01e: 'SRP-RSA-AES-128-CBC-SHA', + 0xc01f: 'SRP-DSS-AES-128-CBC-SHA', + 0xc020: 'SRP-AES-256-CBC-SHA', + 0xc021: 'SRP-RSA-AES-256-CBC-SHA', + 0xc022: 'SRP-DSS-AES-256-CBC-SHA', + 0xc023: 'ECDHE-ECDSA-AES128-SHA256', + 0xc024: 'ECDHE-ECDSA-AES256-SHA384', + 0xc025: 'ECDH-ECDSA-AES128-SHA256', + 0xc026: 'ECDH-ECDSA-AES256-SHA384', + 0xc027: 'ECDHE-RSA-AES128-SHA256', + 0xc028: 'ECDHE-RSA-AES256-SHA384', + 0xc029: 'ECDH-RSA-AES128-SHA256', + 0xc02a: 'ECDH-RSA-AES256-SHA384', + 0xc02b: 'ECDHE-ECDSA-AES128-GCM-SHA256', + 0xc02c: 'ECDHE-ECDSA-AES256-GCM-SHA384', + 0xc02d: 'ECDH-ECDSA-AES128-GCM-SHA256', + 0xc02e: 'ECDH-ECDSA-AES256-GCM-SHA384', + 0xc02f: 'ECDHE-RSA-AES128-GCM-SHA256', + 0xc030: 'ECDHE-RSA-AES256-GCM-SHA384', + 0xc031: 'ECDH-RSA-AES128-GCM-SHA256', + 0xc032: 'ECDH-RSA-AES256-GCM-SHA384', + 0xcc13: 'ECDHE-RSA-CHACHA20-POLY1305', + 0xcc14: 'ECDHE-ECDSA-CHACHA20-POLY1305', + 0xcc15: 'DHE-RSA-CHACHA20-POLY1305', + 0xff00: 'GOST-MD5', + 0xff01: 'GOST-GOST94', + 0xff02: 'GOST-GOST89MAC', + 0xff03: 'GOST-GOST89STREAM', + 0x010080: 'RC4-MD5', + 0x020080: 'EXP-RC4-MD5', + 0x030080: 'RC2-CBC-MD5', + 0x040080: 'EXP-RC2-CBC-MD5', + 0x050080: 'IDEA-CBC-MD5', + 0x060040: 'DES-CBC-MD5', + 0x0700c0: 'DES-CBC3-MD5', + 0x080080: 'RC4-64-MD5', +} + + def is_tls_record_magic(d): """ Returns: @@ -47,8 +241,8 @@ class TlsLayer(Layer): If so, we first connect to the server and then to the client. If not, we only connect to the client and do the server_ssl lazily on a Connect message. - An additional complexity is that establish ssl with the server may require a SNI value from the client. - In an ideal world, we'd do the following: + An additional complexity is that establish ssl with the server may require a SNI value from + the client. In an ideal world, we'd do the following: 1. Start the SSL handshake with the client 2. Check if the client sends a SNI. 3. Pause the client handshake, establish SSL with the server. @@ -100,11 +294,11 @@ class TlsLayer(Layer): while len(client_hello) < client_hello_size: record_header = self.client_conn.rfile.peek(offset + 5)[offset:] if not is_tls_record_magic(record_header) or len(record_header) != 5: - raise ProtocolException('Expected TLS record, got "%s" instead.' % record_header) + raise TlsException('Expected TLS record, got "%s" instead.' % record_header) record_size = struct.unpack("!H", record_header[3:])[0] + 5 record_body = self.client_conn.rfile.peek(offset + record_size)[offset + 5:] if len(record_body) != record_size - 5: - raise ProtocolException("Unexpected EOF in TLS handshake: %s" % record_body) + raise TlsException("Unexpected EOF in TLS handshake: %s" % record_body) client_hello += record_body offset += record_size client_hello_size = struct.unpack("!I", '\x00' + client_hello[1:4])[0] + 4 @@ -127,6 +321,8 @@ class TlsLayer(Layer): self.log("Raw Client Hello:\r\n:%s" % raw_client_hello.encode("hex"), "debug") return + self.client_ciphers = client_hello.cipher_suites.cipher_suites + for extension in client_hello.extensions: if extension.type == 0x00: if len(extension.server_names) != 1 or extension.server_names[0].type != 0: @@ -146,18 +342,11 @@ class TlsLayer(Layer): if self._server_tls and not self.server_conn.tls_established: self._establish_tls_with_server() - def reconnect(self): - self.ctx.reconnect() - if self._server_tls and not self.server_conn.tls_established: - self._establish_tls_with_server() - - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1 and server_tls is not None: - self.ctx.set_server(address, None, None, 1) + def set_server(self, address, server_tls=None, sni=None): + if server_tls is not None: self._sni_from_server_change = sni self._server_tls = server_tls - else: - self.ctx.set_server(address, server_tls, sni, depth) + self.ctx.set_server(address, None, None) @property def sni_for_server_connection(self): @@ -201,7 +390,7 @@ class TlsLayer(Layer): self._establish_tls_with_client() except: pass - raise e + six.reraise(*sys.exc_info()) self._establish_tls_with_client() @@ -219,8 +408,22 @@ class TlsLayer(Layer): chain_file=chain_file, alpn_select_callback=self.__alpn_select_callback, ) + # Some TLS clients will not fail the handshake, + # but will immediately throw an "unexpected eof" error on the first read. + # The reason for this might be difficult to find, so we try to peek here to see if it + # raises ann error. + self.client_conn.rfile.peek(1) except NetLibError as e: - raise ProtocolException("Cannot establish TLS with client: %s" % repr(e), e) + six.reraise( + ClientHandshakeException, + ClientHandshakeException( + "Cannot establish TLS with client (sni: {sni}): {e}".format( + sni=self.client_sni, e=repr(e) + ), + self.client_sni or repr(self.server_conn.address) + ), + sys.exc_info()[2] + ) def _establish_tls_with_server(self): self.log("Establish TLS with server", "debug") @@ -230,9 +433,19 @@ class TlsLayer(Layer): # and mitmproxy would enter TCP passthrough mode, which we want to avoid. deprecated_http2_variant = lambda x: x.startswith("h2-") or x.startswith("spdy") if self.client_alpn_protocols: - alpn = filter(lambda x: not deprecated_http2_variant(x), self.client_alpn_protocols) + alpn = [x for x in self.client_alpn_protocols if not deprecated_http2_variant(x)] else: alpn = None + if alpn and "h2" in alpn and not self.config.http2 : + alpn.remove("h2") + + ciphers_server = self.config.ciphers_server + if not ciphers_server: + ciphers_server = [] + for id in self.client_ciphers: + if id in CIPHER_ID_NAME_MAP.keys(): + ciphers_server.append(CIPHER_ID_NAME_MAP[id]) + ciphers_server = ':'.join(ciphers_server) self.server_conn.establish_ssl( self.config.clientcerts, @@ -242,7 +455,7 @@ class TlsLayer(Layer): verify_options=self.config.openssl_verification_mode_server, ca_path=self.config.openssl_trusted_cadir_server, ca_pemfile=self.config.openssl_trusted_ca_server, - cipher_list=self.config.ciphers_server, + cipher_list=ciphers_server, alpn_protos=alpn, ) tls_cert_err = self.server_conn.ssl_verification_error @@ -259,17 +472,25 @@ class TlsLayer(Layer): (tls_cert_err['depth'], tls_cert_err['errno']), "error") self.log("Aborting connection attempt", "error") - raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( - address=repr(self.server_conn.address), - sni=self.sni_for_server_connection, - e=repr(e), - ), e) + six.reraise( + TlsException, + TlsException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + address=repr(self.server_conn.address), + sni=self.sni_for_server_connection, + e=repr(e), + )), + sys.exc_info()[2] + ) except NetLibError as e: - raise ProtocolException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( - address=repr(self.server_conn.address), - sni=self.sni_for_server_connection, - e=repr(e), - ), e) + six.reraise( + TlsException, + TlsException("Cannot establish TLS with {address} (sni: {sni}): {e}".format( + address=repr(self.server_conn.address), + sni=self.sni_for_server_connection, + e=repr(e), + )), + sys.exc_info()[2] + ) self.log("ALPN selected by server: %s" % self.alpn_for_client_connection, "debug") @@ -294,5 +515,4 @@ class TlsLayer(Layer): if self._sni_from_server_change: sans.add(self._sni_from_server_change) - sans.discard(host) return self.config.certstore.get_cert(host, list(sans)) diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index d5297cb1..be7f5207 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -2,8 +2,10 @@ from __future__ import (absolute_import, print_function, division) from .server import ProxyServer, DummyServer from .config import ProxyConfig +from .root_context import RootContext, Log __all__ = [ "ProxyServer", "DummyServer", "ProxyConfig", + "RootContext", "Log", ] diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 2a1b84cb..cd9eda5a 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -54,6 +54,8 @@ class ProxyConfig: authenticator=None, ignore_hosts=tuple(), tcp_hosts=tuple(), + http2=False, + rawtcp=False, ciphers_client=None, ciphers_server=None, certs=tuple(), @@ -78,6 +80,8 @@ class ProxyConfig: self.check_ignore = HostMatcher(ignore_hosts) self.check_tcp = HostMatcher(tcp_hosts) + self.http2 = http2 + self.rawtcp = rawtcp self.authenticator = authenticator self.cadir = os.path.expanduser(cadir) self.certstore = certutils.CertStore.from_store( @@ -183,6 +187,8 @@ def process_proxy_options(parser, options): upstream_server=upstream_server, ignore_hosts=options.ignore_hosts, tcp_hosts=options.tcp_hosts, + http2=options.http2, + rawtcp=options.rawtcp, authenticator=authenticator, ciphers_client=options.ciphers_client, ciphers_server=options.ciphers_server, @@ -192,4 +198,4 @@ def process_proxy_options(parser, options): ssl_verify_upstream_cert=options.ssl_verify_upstream_cert, ssl_verify_upstream_trusted_cadir=options.ssl_verify_upstream_trusted_cadir, ssl_verify_upstream_trusted_ca=options.ssl_verify_upstream_trusted_ca - )
\ No newline at end of file + ) diff --git a/libmproxy/proxy/modes/http_proxy.py b/libmproxy/proxy/modes/http_proxy.py index 90c54cc6..c7502c24 100644 --- a/libmproxy/proxy/modes/http_proxy.py +++ b/libmproxy/proxy/modes/http_proxy.py @@ -10,7 +10,7 @@ class HttpProxy(Layer, ServerConnectionMixin): layer() finally: if self.server_conn: - self._disconnect() + self.disconnect() class HttpUpstreamProxy(Layer, ServerConnectionMixin): @@ -23,4 +23,4 @@ class HttpUpstreamProxy(Layer, ServerConnectionMixin): layer() finally: if self.server_conn: - self._disconnect() + self.disconnect() diff --git a/libmproxy/proxy/modes/reverse_proxy.py b/libmproxy/proxy/modes/reverse_proxy.py index b57ac5eb..28f4e6f8 100644 --- a/libmproxy/proxy/modes/reverse_proxy.py +++ b/libmproxy/proxy/modes/reverse_proxy.py @@ -14,4 +14,4 @@ class ReverseProxy(Layer, ServerConnectionMixin): layer() finally: if self.server_conn: - self._disconnect() + self.disconnect() diff --git a/libmproxy/proxy/modes/socks_proxy.py b/libmproxy/proxy/modes/socks_proxy.py index ebaf939e..545c38d6 100644 --- a/libmproxy/proxy/modes/socks_proxy.py +++ b/libmproxy/proxy/modes/socks_proxy.py @@ -48,7 +48,7 @@ class Socks5Proxy(Layer, ServerConnectionMixin): self.client_conn.wfile.flush() except (socks.SocksError, NetLibError) as e: - raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e), e) + raise Socks5Exception("SOCKS5 mode failure: %s" % repr(e)) self.server_conn.address = connect_request.addr @@ -57,4 +57,4 @@ class Socks5Proxy(Layer, ServerConnectionMixin): layer() finally: if self.server_conn: - self._disconnect() + self.disconnect() diff --git a/libmproxy/proxy/modes/transparent_proxy.py b/libmproxy/proxy/modes/transparent_proxy.py index 96ad86c4..da1d4632 100644 --- a/libmproxy/proxy/modes/transparent_proxy.py +++ b/libmproxy/proxy/modes/transparent_proxy.py @@ -14,11 +14,11 @@ class TransparentProxy(Layer, ServerConnectionMixin): try: self.server_conn.address = self.resolver.original_addr(self.client_conn.connection) except Exception as e: - raise ProtocolException("Transparent mode failure: %s" % repr(e), e) + raise ProtocolException("Transparent mode failure: %s" % repr(e)) layer = self.ctx.next_layer(self) try: layer() finally: if self.server_conn: - self._disconnect() + self.disconnect() diff --git a/libmproxy/proxy/root_context.py b/libmproxy/proxy/root_context.py index 35909612..54bea1db 100644 --- a/libmproxy/proxy/root_context.py +++ b/libmproxy/proxy/root_context.py @@ -1,8 +1,13 @@ from __future__ import (absolute_import, print_function, division) +import string +import sys +import six + +from libmproxy.exceptions import ProtocolException from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol - +from netlib.tcp import NetLibError from ..protocol import ( RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin ) @@ -11,31 +16,47 @@ from .modes import HttpProxy, HttpUpstreamProxy, ReverseProxy class RootContext(object): """ - The outmost context provided to the root layer. - As a consequence, every layer has .client_conn, .channel, .next_layer() and .config. + The outermost context provided to the root layer. + As a consequence, every layer has access to methods and attributes defined here. + + Attributes: + client_conn: + The :py:class:`client connection <libmproxy.models.ClientConnection>`. + channel: + A :py:class:`~libmproxy.controller.Channel` to communicate with the FlowMaster. + Provides :py:meth:`.ask() <libmproxy.controller.Channel.ask>` and + :py:meth:`.tell() <libmproxy.controller.Channel.tell>` methods. + config: + The :py:class:`proxy server's configuration <libmproxy.proxy.ProxyConfig>` """ def __init__(self, client_conn, config, channel): - self.client_conn = client_conn # Client Connection - self.channel = channel # provides .ask() method to communicate with FlowMaster - self.config = config # Proxy Configuration + self.client_conn = client_conn + self.channel = channel + self.config = config def next_layer(self, top_layer): """ This function determines the next layer in the protocol stack. Arguments: - top_layer: the current top layer. + top_layer: the current innermost layer. Returns: The next layer """ + layer = self._next_layer(top_layer) + return self.channel.ask("next_layer", layer) + def _next_layer(self, top_layer): # 1. Check for --ignore. if self.config.check_ignore(top_layer.server_conn.address): return RawTCPLayer(top_layer, logging=False) - d = top_layer.client_conn.rfile.peek(3) + try: + d = top_layer.client_conn.rfile.peek(3) + except NetLibError as e: + six.reraise(ProtocolException, ProtocolException(str(e)), sys.exc_info()[2]) client_tls = is_tls_record_magic(d) # 2. Always insert a TLS layer, even if there's neither client nor server tls. @@ -69,21 +90,30 @@ class RootContext(object): if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: return Http1Layer(top_layer, 'transparent') - # 6. Assume HTTP1 by default + # 6. Check for raw tcp mode + is_ascii = ( + len(d) == 3 and + # better be safe here and don't expect uppercase... + all(x in string.ascii_letters for x in d) + ) + if self.config.rawtcp and not is_ascii: + return RawTCPLayer(top_layer) + + # 7. Assume HTTP1 by default return Http1Layer(top_layer, 'transparent') - # In a future version, we want to implement TCP passthrough as the last fallback, - # but we don't have the UI part ready for that. - # - # d = top_layer.client_conn.rfile.peek(3) - # is_ascii = ( - # len(d) == 3 and - # # better be safe here and don't expect uppercase... - # all(x in string.ascii_letters for x in d) - # ) - # # TODO: This could block if there are not enough bytes available? - # d = top_layer.client_conn.rfile.peek(len(HTTP2Protocol.CLIENT_CONNECTION_PREFACE)) - # is_http2_magic = (d == HTTP2Protocol.CLIENT_CONNECTION_PREFACE) + def log(self, msg, level, subs=()): + """ + Send a log message to the master. + """ + + full_msg = [ + "{}: {}".format(repr(self.client_conn.address), msg) + ] + for i in subs: + full_msg.append(" -> " + i) + full_msg = "\n".join(full_msg) + self.channel.tell("log", Log(full_msg, level)) @property def layers(self): @@ -91,3 +121,9 @@ class RootContext(object): def __repr__(self): return "RootContext" + + +class Log(object): + def __init__(self, msg, level="info"): + self.msg = msg + self.level = level diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index e9e8df09..88448172 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -3,15 +3,16 @@ from __future__ import (absolute_import, print_function, division) import traceback import sys import socket +import six from netlib import tcp from netlib.http.http1 import HTTP1Protocol from netlib.tcp import NetLibError -from ..exceptions import ProtocolException, ServerException -from ..protocol import Log, Kill +from ..exceptions import ProtocolException, ServerException, ClientHandshakeException +from ..protocol import Kill from ..models import ClientConnection, make_error_response from .modes import HttpUpstreamProxy, HttpProxy, ReverseProxy, TransparentProxy, Socks5Proxy -from .root_context import RootContext +from .root_context import RootContext, Log class DummyServer: @@ -39,7 +40,11 @@ class ProxyServer(tcp.TCPServer): try: super(ProxyServer, self).__init__((config.host, config.port)) except socket.error as e: - raise ServerException('Error starting proxy server: ' + repr(e), e) + six.reraise( + ServerException, + ServerException('Error starting proxy server: ' + repr(e)), + sys.exc_info()[2] + ) self.channel = None def start_slave(self, klass, channel): @@ -116,7 +121,18 @@ class ConnectionHandler(object): except Kill: self.log("Connection killed", "info") except ProtocolException as e: - self.log(e, "info") + + if isinstance(e, ClientHandshakeException): + self.log( + "Client Handshake failed. " + "The client may not trust the proxy's certificate for {}.".format(e.server), + "error" + ) + self.log(repr(e), "debug") + else: + self.log(repr(e), "error") + + self.log(traceback.format_exc(), "debug") # If an error propagates to the topmost level, # we send an HTTP error response, which is both # understandable by HTTP clients and humans. @@ -137,4 +153,4 @@ class ConnectionHandler(object): def log(self, msg, level): msg = "{}: {}".format(repr(self.client_conn.address), msg) - self.channel.tell("log", Log(msg, level))
\ No newline at end of file + self.channel.tell("log", Log(msg, level)) diff --git a/libmproxy/script.py b/libmproxy/script.py index e13f0e2b..b4ecfbbf 100644 --- a/libmproxy/script.py +++ b/libmproxy/script.py @@ -95,8 +95,8 @@ class Script: """ if self.ns is not None: self.unload() - ns = {} script_dir = os.path.dirname(os.path.abspath(self.args[0])) + ns = {'__file__': os.path.abspath(self.args[0])} sys.path.append(script_dir) try: execfile(self.args[0], ns, ns) @@ -179,7 +179,8 @@ def concurrent(fn): "error", "clientconnect", "serverconnect", - "clientdisconnect"): + "clientdisconnect", + "next_layer"): def _concurrent(ctx, obj): _handle_concurrent_reply(fn, obj, ctx, obj) diff --git a/libmproxy/web/app.py b/libmproxy/web/app.py index d6082ee2..2517e7ad 100644 --- a/libmproxy/web/app.py +++ b/libmproxy/web/app.py @@ -27,8 +27,7 @@ class RequestHandler(tornado.web.RequestHandler): @property def json(self): - if not self.request.headers.get( - "Content-Type").startswith("application/json"): + if not self.request.headers.get("Content-Type").startswith("application/json"): return None return json.loads(self.request.body) @@ -186,12 +185,12 @@ class FlowContent(RequestHandler): if not message.content: raise APIError(400, "No content.") - content_encoding = message.headers.get_first("Content-Encoding", None) + content_encoding = message.headers.get("Content-Encoding", None) if content_encoding: content_encoding = re.sub(r"[^\w]", "", content_encoding) self.set_header("Content-Encoding", content_encoding) - original_cd = message.headers.get_first("Content-Disposition", None) + original_cd = message.headers.get("Content-Disposition", None) filename = None if original_cd: filename = re.search("filename=([\w\" \.\-\(\)]+)", original_cd) |