from __future__ import print_function import copy import logging import os import sys import threading import urllib from netlib import tcp, http, certutils, websockets from netlib.exceptions import HttpException, HttpReadDisconnect, TcpTimeout, TcpDisconnect, \ TlsException from . import version, app, language, utils, log, protocols import language.http import language.actions import language.exceptions import language.websockets DEFAULT_CERT_DOMAIN = "pathod.net" CONFDIR = "~/.mitmproxy" CERTSTORE_BASENAME = "mitmproxy" CA_CERT_NAME = "mitmproxy-ca.pem" DEFAULT_CRAFT_ANCHOR = "/p/" logger = logging.getLogger('pathod') class PathodError(Exception): pass class SSLOptions(object): def __init__( self, confdir=CONFDIR, cn=None, sans=(), not_after_connect=None, request_client_cert=False, ssl_version=tcp.SSL_DEFAULT_METHOD, ssl_options=tcp.SSL_DEFAULT_OPTIONS, ciphers=None, certs=None, alpn_select=b'h2', ): self.confdir = confdir self.cn = cn self.sans = sans self.not_after_connect = not_after_connect self.request_client_cert = request_client_cert self.ssl_version = ssl_version self.ssl_options = ssl_options self.ciphers = ciphers self.alpn_select = alpn_select self.certstore = certutils.CertStore.from_store( os.path.expanduser(confdir), CERTSTORE_BASENAME ) for i in certs or []: self.certstore.add_cert_file(*i) def get_cert(self, name): if self.cn: name = self.cn elif not name: name = DEFAULT_CERT_DOMAIN return self.certstore.get_cert(name, self.sans) class PathodHandler(tcp.BaseHandler): wbufsize = 0 sni = None def __init__( self, connection, address, server, logfp, settings, http2_framedump=False ): tcp.BaseHandler.__init__(self, connection, address, server) self.logfp = logfp self.settings = copy.copy(settings) self.protocol = None self.use_http2 = False self.http2_framedump = http2_framedump def handle_sni(self, connection): self.sni = connection.get_servername() def http_serve_crafted(self, crafted, logctx): error, crafted = self.server.check_policy( crafted, self.settings ) if error: err = self.make_http_error_response(error) language.serve(err, self.wfile, self.settings) return None, dict( type="error", msg=error ) if self.server.explain and not hasattr(crafted, 'is_error_response'): crafted = crafted.freeze(self.settings) logctx(">> Spec: %s" % crafted.spec()) response_log = language.serve( crafted, self.wfile, self.settings ) if response_log["disconnect"]: return None, response_log return self.handle_http_request, response_log def handle_http_request(self, logger): """ Returns a (handler, log) tuple. handler: Handler for the next request, or None to disconnect log: A dictionary, or None """ with logger.ctx() as lg: try: req = self.protocol.read_request(self.rfile) except HttpReadDisconnect: return None, None except HttpException as s: s = str(s) lg(s) return None, dict(type="error", msg=s) if req.method == 'CONNECT': return self.protocol.handle_http_connect([req.host, req.port, req.http_version], lg) method = req.method path = req.path http_version = req.http_version headers = req.headers body = req.content clientcert = None if self.clientcert: clientcert = dict( cn=self.clientcert.cn, subject=self.clientcert.subject, serial=self.clientcert.serial, notbefore=self.clientcert.notbefore.isoformat(), notafter=self.clientcert.notafter.isoformat(), keyinfo=self.clientcert.keyinfo, ) retlog = dict( type="crafted", protocol="http", request=dict( path=path, method=method, headers=headers.fields, http_version=http_version, sni=self.sni, remote_address=self.address(), clientcert=clientcert, ), cipher=None, ) if self.ssl_established: retlog["cipher"] = self.get_current_cipher() m = utils.MemBool() websocket_key = websockets.WebsocketsProtocol.check_client_handshake(headers) self.settings.websocket_key = websocket_key # If this is a websocket initiation, we respond with a proper # server response, unless over-ridden. if websocket_key: anchor_gen = language.parse_pathod("ws") else: anchor_gen = None for regex, spec in self.server.anchors: if regex.match(path): anchor_gen = language.parse_pathod(spec, self.use_http2) break else: if m(path.startswith(self.server.craftanchor)): spec = urllib.unquote(path)[len(self.server.craftanchor):] if spec: try: anchor_gen = language.parse_pathod(spec, self.use_http2) except language.ParseException as v: lg("Parse error: %s" % v.msg) anchor_gen = iter([self.make_http_error_response( "Parse Error", "Error parsing response spec: %s\n" % ( v.msg + v.marked() ) )]) else: if self.use_http2: anchor_gen = iter([self.make_http_error_response( "Spec Error", "HTTP/2 only supports request/response with the craft anchor point: %s" % self.server.craftanchor )]) if anchor_gen: spec = anchor_gen.next() if self.use_http2 and isinstance(spec, language.http2.Response): spec.stream_id = req.stream_id lg("crafting spec: %s" % spec) nexthandler, retlog["response"] = self.http_serve_crafted( spec, lg ) if nexthandler and websocket_key: self.protocol = protocols.websockets.WebsocketsProtocol(self) return self.protocol.handle_websocket, retlog else: return nexthandler, retlog else: return self.protocol.handle_http_app(method, path, headers, body, lg) def make_http_error_response(self, reason, body=None): resp = self.protocol.make_error_response(reason, body) resp.is_error_response = True return resp def handle(self): self.settimeout(self.server.timeout) if self.server.ssl: try: cert, key, _ = self.server.ssloptions.get_cert(None) self.convert_to_ssl( cert, key, handle_sni=self.handle_sni, request_client_cert=self.server.ssloptions.request_client_cert, cipher_list=self.server.ssloptions.ciphers, method=self.server.ssloptions.ssl_version, options=self.server.ssloptions.ssl_options, alpn_select=self.server.ssloptions.alpn_select, ) except TlsException as v: s = str(v) self.server.add_log( dict( type="error", msg=s ) ) log.write_raw(self.logfp, s) return alp = self.get_alpn_proto_negotiated() if alp == b'h2': self.protocol = protocols.http2.HTTP2Protocol(self) self.use_http2 = True if not self.protocol: self.protocol = protocols.http.HTTPProtocol(self) lr = self.rfile if self.server.logreq else None lw = self.wfile if self.server.logresp else None logger = log.ConnectionLogger(self.logfp, self.server.hexdump, lr, lw) self.settings.protocol = self.protocol handler = self.handle_http_request while not self.finished: handler, l = handler(logger) if l: self.addlog(l) if not handler: return def addlog(self, log): # FIXME: The bytes in the log should not be escaped. We do this at the # moment because JSON encoding can't handle binary data, and I don't # want to base64 everything. if self.server.logreq: encoded_bytes = self.rfile.get_log().encode("string_escape") log["request_bytes"] = encoded_bytes if self.server.logresp: encoded_bytes = self.wfile.get_log().encode("string_escape") log["response_bytes"] = encoded_bytes self.server.add_log(log) class Pathod(tcp.TCPServer): LOGBUF = 500 def __init__( self, addr, ssl=False, ssloptions=None, craftanchor=DEFAULT_CRAFT_ANCHOR, staticdir=None, anchors=(), sizelimit=None, noweb=False, nocraft=False, noapi=False, nohang=False, timeout=None, logreq=False, logresp=False, explain=False, hexdump=False, http2_framedump=False, webdebug=False, logfp=sys.stdout, ): """ addr: (address, port) tuple. If port is 0, a free port will be automatically chosen. ssloptions: an SSLOptions object. craftanchor: URL prefix specifying the path under which to anchor response generation. staticdir: path to a directory of static resources, or None. anchors: List of (regex object, language.Request object) tuples, or None. sizelimit: Limit size of served data. nocraft: Disable response crafting. noapi: Disable the API. nohang: Disable pauses. """ tcp.TCPServer.__init__(self, addr) self.ssl = ssl self.ssloptions = ssloptions or SSLOptions() self.staticdir = staticdir self.craftanchor = craftanchor self.sizelimit = sizelimit self.noweb, self.nocraft = noweb, nocraft self.noapi, self.nohang = noapi, nohang self.timeout, self.logreq = timeout, logreq self.logresp, self.hexdump = logresp, hexdump self.http2_framedump = http2_framedump self.explain = explain self.logfp = logfp self.app = app.make_app(noapi, webdebug) self.app.config["pathod"] = self self.log = [] self.logid = 0 self.anchors = anchors self.settings = language.Settings( staticdir=self.staticdir ) def check_policy(self, req, settings): """ A policy check that verifies the request size is within limits. """ if self.nocraft: return "Crafting disabled.", None try: req = req.resolve(settings) l = req.maximum_length(settings) except language.FileAccessDenied: return "File access denied.", None if self.sizelimit and l > self.sizelimit: return "Response too large.", None pauses = [isinstance(i, language.actions.PauseAt) for i in req.actions] if self.nohang and any(pauses): return "Pauses have been disabled.", None return None, req def handle_client_connection(self, request, client_address): h = PathodHandler( request, client_address, self, self.logfp, self.settings, self.http2_framedump, ) try: h.handle() h.finish() except TcpDisconnect: # pragma: no cover log.write_raw(self.logfp, "Disconnect") self.add_log( dict( type="error", msg="Disconnect" ) ) return except TcpTimeout: log.write_raw(self.logfp, "Timeout") self.add_log( dict( type="timeout", ) ) return def add_log(self, d): if not self.noapi: lock = threading.Lock() with lock: d["id"] = self.logid self.log.insert(0, d) if len(self.log) > self.LOGBUF: self.log.pop() self.logid += 1 return d["id"] def clear_log(self): lock = threading.Lock() with lock: self.log = [] def log_by_id(self, identifier): for i in self.log: if i["id"] == identifier: return i def get_log(self): return self.log def main(args): # pragma: no cover ssloptions = SSLOptions( cn=args.cn, confdir=args.confdir, not_after_connect=args.ssl_not_after_connect, ciphers=args.ciphers, ssl_version=args.ssl_version, ssl_options=args.ssl_options, certs=args.ssl_certs, sans=args.sans, ) root = logging.getLogger() if root.handlers: for handler in root.handlers: root.removeHandler(handler) log = logging.getLogger('pathod') log.setLevel(logging.DEBUG) fmt = logging.Formatter( '%(asctime)s: %(message)s', datefmt='%d-%m-%y %H:%M:%S', ) if args.logfile: fh = logging.handlers.WatchedFileHandler(args.logfile) fh.setFormatter(fmt) log.addHandler(fh) if not args.daemonize: sh = logging.StreamHandler() sh.setFormatter(fmt) log.addHandler(sh) try: pd = Pathod( (args.address, args.port), craftanchor=args.craftanchor, ssl=args.ssl, ssloptions=ssloptions, staticdir=args.staticdir, anchors=args.anchors, sizelimit=args.sizelimit, noweb=args.noweb, nocraft=args.nocraft, noapi=args.noapi, nohang=args.nohang, timeout=args.timeout, logreq=args.logreq, logresp=args.logresp, hexdump=args.hexdump, http2_framedump=args.http2_framedump, explain=args.explain, webdebug=args.webdebug ) except PathodError as v: print("Error: %s" % v, file=sys.stderr) sys.exit(1) except language.FileAccessDenied as v: print("Error: %s" % v, file=sys.stderr) if args.daemonize: utils.daemonize() try: print("%s listening on %s" % ( version.NAMEVERSION, repr(pd.address) )) pd.serve_forever() except KeyboardInterrupt: pass