Start refactoring towards netlib, adding SNI and client testing.

4 files changed, 454 insertions, 261 deletions

diff --git a/libpathod/netlib.py b/libpathod/netlib.py
new file mode 100644
index 00000000..08ccba09
--- /dev/null
+++ b/libpathod/netlib.py
@@ -0,0 +1,182 @@
+import select, socket, threading, traceback, sys
+from OpenSSL import SSL
+
+
+class NetLibError(Exception): pass
+
+
+class FileLike:
+    def __init__(self, o):
+        self.o = o
+
+    def __getattr__(self, attr):
+        return getattr(self.o, attr)
+
+    def flush(self):
+        pass
+
+    def read(self, length):
+        result = ''
+        while len(result) < length:
+            try:
+                data = self.o.read(length)
+            except SSL.ZeroReturnError:
+                break
+            if not data:
+                break
+            result += data
+        return result
+
+    def write(self, v):
+        self.o.sendall(v)
+
+    def readline(self, size = None):
+        result = ''
+        bytes_read = 0
+        while True:
+            if size is not None and bytes_read >= size:
+                break
+            ch = self.read(1)
+            bytes_read += 1
+            if not ch:
+                break
+            else:
+                result += ch
+                if ch == '\n':
+                    break
+        return result
+
+
+class TCPClient:
+    def __init__(self, ssl, host, port, clientcert):
+        self.ssl, self.host, self.port, self.clientcert = ssl, host, port, clientcert
+        self.connection, self.rfile, self.wfile = None, None, None
+        self.cert = None
+        self.connect()
+
+    def connect(self):
+        try:
+            addr = socket.gethostbyname(self.host)
+            server = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            if self.ssl:
+                context = SSL.Context(SSL.SSLv23_METHOD)
+                if self.clientcert:
+                    context.use_certificate_file(self.clientcert)
+                server = SSL.Connection(context, server)
+            server.connect((addr, self.port))
+            if self.ssl:
+                self.cert = server.get_peer_certificate()
+                self.rfile, self.wfile = FileLike(server), FileLike(server)
+            else:
+                self.rfile, self.wfile = server.makefile('rb'), server.makefile('wb')
+        except socket.error, err:
+            raise NetLibError('Error connecting to "%s": %s' % (self.host, err))
+        self.connection = server
+
+
+class BaseHandler:
+    rbufsize = -1
+    wbufsize = 0
+    def __init__(self, connection, client_address, server):
+        self.connection = connection
+        self.rfile = self.connection.makefile('rb', self.rbufsize)
+        self.wfile = self.connection.makefile('wb', self.wbufsize)
+
+        self.client_address = client_address
+        self.server = server
+        self.handle()
+        self.finish()
+
+    def convert_to_ssl(self, cert, key):
+        ctx = SSL.Context(SSL.SSLv23_METHOD)
+        ctx.use_privatekey_file(key)
+        ctx.use_certificate_file(cert)
+        self.connection = SSL.Connection(ctx, self.connection)
+        self.connection.set_accept_state()
+        self.rfile = FileLike(self.connection)
+        self.wfile = FileLike(self.connection)
+
+    def finish(self):
+        try:
+            if not getattr(self.wfile, "closed", False):
+                self.wfile.flush()
+            self.connection.close()
+            self.wfile.close()
+            self.rfile.close()
+        except IOError: # pragma: no cover
+            pass
+
+    def handle(self): # pragma: no cover
+        raise NotImplementedError
+
+
+class TCPServer:
+    request_queue_size = 20
+    def __init__(self, server_address):
+        self.server_address = server_address
+        self.__is_shut_down = threading.Event()
+        self.__shutdown_request = False
+        self.socket = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+        self.socket.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1)
+        self.socket.bind(self.server_address)
+        self.server_address = self.socket.getsockname()
+        self.socket.listen(self.request_queue_size)
+        self.port = self.socket.getsockname()[1]
+
+    def request_thread(self, request, client_address):
+        try:
+            self.handle_connection(request, client_address)
+            request.close()
+        except:
+            self.handle_error(request, client_address)
+            request.close()
+
+    def serve_forever(self, poll_interval=0.5):
+        self.__is_shut_down.clear()
+        try:
+            while not self.__shutdown_request:
+                r, w, e = select.select([self.socket], [], [], poll_interval)
+                if self.socket in r:
+                    try:
+                        request, client_address = self.socket.accept()
+                    except socket.error:
+                        return
+                    try:
+                        t = threading.Thread(
+                                target = self.request_thread,
+                                args = (request, client_address)
+                            )
+                        t.setDaemon(1)
+                        t.start()
+                    except:
+                        self.handle_error(request, client_address)
+                        request.close()
+        finally:
+            self.__shutdown_request = False
+            self.__is_shut_down.set()
+
+    def shutdown(self):
+        self.__shutdown_request = True
+        self.__is_shut_down.wait()
+        self.handle_shutdown()
+
+    def handle_error(self, request, client_address, fp=sys.stderr):
+        """
+            Called when handle_connection raises an exception.
+        """
+        print >> fp, '-'*40
+        print >> fp, "Error processing of request from %s:%s"%client_address
+        print >> fp, traceback.format_exc()
+        print >> fp, '-'*40
+
+    def handle_connection(self, request, client_address): # pragma: no cover
+        """
+            Called after client connection.
+        """
+        raise NotImplementedError
+
+    def handle_shutdown(self):
+        """
+            Called after server shutdown.
+        """
+        pass
diff --git a/libpathod/opathod.py b/libpathod/opathod.py
new file mode 100644
index 00000000..6ec8367d
--- /dev/null
+++ b/libpathod/opathod.py
@@ -0,0 +1,263 @@
+import urllib, pprint
+import tornado.web, tornado.template, tornado.ioloop, tornado.httpserver
+import rparse, utils, version
+
+
+class Pathod(object):
+    def __init__(self, spec, application, request, **settings):
+        self.application, self.request, self.settings = application, request, settings
+        try:
+            self.response = rparse.parse(self.settings, spec)
+        except rparse.ParseException, v:
+            self.response = rparse.InternalResponse(
+                800,
+                "Error parsing response spec: %s\n"%v.msg + v.marked()
+            )
+
+    def _execute(self, transforms, *args, **kwargs):
+        d = self.response.serve(self.request)
+        d["request"] = dict(
+            path = self.request.path,
+            method = self.request.method,
+            headers = self.request.headers,
+            host = self.request.host,
+            protocol = self.request.protocol,
+            remote_address = self.request.connection.address,
+            full_url = self.request.full_url(),
+            query = self.request.query,
+            version = self.request.version,
+            uri = self.request.uri,
+        )
+        self.application.add_log(d)
+
+
+class RequestPathod(Pathod):
+    anchor = "/p/"
+    def __init__(self, application, request, **settings):
+        spec = urllib.unquote(request.uri)[len(self.anchor):]
+        Pathod.__init__(self, spec, application, request, **settings)
+
+
+class PathodApp(tornado.web.Application):
+    LOGBUF = 500
+    def __init__(self, **settings):
+        self.appsettings = settings
+        tornado.web.Application.__init__(
+            self,
+            [
+                (r"/", Index),
+                (r"/log", Log),
+                (r"/log/clear", ClearLog),
+                (r"/log/([0-9]+)", OneLog),
+                (r"/help", Help),
+                (r"/preview", Preview),
+                (r"/api/shutdown", APIShutdown),
+                (r"/api/info", APIInfo),
+                (r"/api/log", APILog),
+                (r"/api/log/clear", APILogClear),
+                (r"/p/.*", RequestPathod, settings),
+            ],
+            static_path = utils.data.path("static"),
+            template_path = utils.data.path("templates"),
+            debug=True
+        )
+        self.log = []
+        self.logid = 0
+
+    def add_anchor(self, pattern, spec):
+        """
+            Anchors are added to the beginning of the handlers.
+        """
+        # We assume we have only one host...
+        l = self.handlers[0][1]
+        class FixedPathod(Pathod):
+            def __init__(self, application, request, **settings):
+                Pathod.__init__(self, spec, application, request, **settings)
+        FixedPathod.spec = spec
+        FixedPathod.pattern = pattern
+        l.insert(0, tornado.web.URLSpec(pattern, FixedPathod, self.appsettings))
+
+    def get_anchors(self):
+        """
+            Anchors are added to the beginning of the handlers.
+        """
+        l = self.handlers[0][1]
+        a = []
+        for i in l:
+            if i.handler_class.__name__ == "FixedPathod":
+                a.append(
+                    (
+                        i.handler_class.pattern,
+                        i.handler_class.spec
+                    )
+                )
+        return a
+
+    def remove_anchor(self, pattern, spec):
+        """
+            Anchors are added to the beginning of the handlers.
+        """
+        l = self.handlers[0][1]
+        for i, h in enumerate(l):
+            if h.handler_class.__name__ == "FixedPathod":
+                if (h.handler_class.pattern, h.handler_class.spec) == (pattern, spec):
+                    del l[i]
+                    return
+
+    def add_log(self, d):
+        d["id"] = self.logid
+        self.log.insert(0, d)
+        if len(self.log) > self.LOGBUF:
+            self.log.pop()
+        self.logid += 1
+
+    def log_by_id(self, id):
+        for i in self.log:
+            if i["id"] == id:
+                return i
+
+    def clear_log(self):
+        self.log = []
+
+    def get_log(self):
+        return self.log
+
+
+def make_app(staticdir=None, anchors=()):
+    """
+        staticdir: A directory for static assets referenced in response patterns.
+        anchors: A sequence of strings of the form "pattern=pagespec"
+    """
+    settings = dict(
+        staticdir=staticdir
+    )
+    application = PathodApp(**settings)
+    for i in anchors:
+        rex, spec = utils.parse_anchor_spec(i, settings)
+        application.add_anchor(rex, spec)
+    return application
+
+
+def make_server(application, port, address, ssl_options):
+    """
+        Returns a (server, port) tuple.
+
+        The returned port will match the passed port, unless the passed port
+        was 0. In that case, an arbitrary empty port will be bound to, and this
+        new port will be returned.
+    """
+    http_server = tornado.httpserver.HTTPServer(
+        application,
+        ssl_options=ssl_options
+    )
+    http_server.listen(port, address)
+    port = port
+    for i in http_server._sockets.values():
+        sn = i.getsockname()
+        if sn[0] == address:
+            port = sn[1]
+    return http_server, port
+
+
+# begin nocover
+def run(server):
+    tornado.ioloop.IOLoop.instance().start()
+    server.stop()
+
+
+class APILog(tornado.web.RequestHandler):
+    def get(self):
+        self.write(
+            dict(
+                d = self.application.get_log()
+            )
+        )
+
+
+class APILogClear(tornado.web.RequestHandler):
+    def post(self):
+        self.application.clear_log()
+        self.write("OK")
+
+
+class APIShutdown(tornado.web.RequestHandler):
+    def post(self):
+        tornado.ioloop.IOLoop.instance().stop()
+        self.write("OK")
+
+
+class APIInfo(tornado.web.RequestHandler):
+    def get(self):
+        self.write(
+            dict(
+                version = version.IVERSION
+            )
+        )
+
+
+class _Page(tornado.web.RequestHandler):
+    def render(self, name, **kwargs):
+        tornado.web.RequestHandler.render(self, name + ".html", **kwargs)
+
+
+class Index(_Page):
+    name = "index"
+    section = "main"
+    def get(self):
+        self.render(self.name, section=self.section, spec="")
+
+
+class Preview(_Page):
+    name = "preview"
+    section = "main"
+    SANITY = 1024*1024
+    def get(self):
+        spec = self.get_argument("spec", None)
+        args = dict(
+            spec = spec,
+            section = self.section,
+            syntaxerror = None,
+            error = None
+        )
+        try:
+            r = rparse.parse(self.application.settings, spec)
+        except rparse.ParseException, v:
+            args["syntaxerror"] = str(v)
+            args["marked"] = v.marked()
+            return self.render(self.name, **args)
+        if r.length() > self.SANITY:
+            error = "Refusing to preview a response of %s bytes. This is for your own good."%r.length()
+            args["error"] = error
+        else:
+            d = utils.DummyRequest()
+            r.serve(d)
+            args["output"] = d.getvalue()
+        self.render(self.name, **args)
+
+
+class Help(_Page):
+    name = "help"
+    section = "help"
+    def get(self):
+        self.render(self.name, section=self.section)
+
+
+class Log(_Page):
+    name = "log"
+    section = "log"
+    def get(self):
+        self.render(self.name, section=self.section, log=self.application.log)
+
+
+class OneLog(_Page):
+    name = "onelog"
+    section = "log"
+    def get(self, lid):
+        l = pprint.pformat(self.application.log_by_id(int(lid)))
+        self.render(self.name, section=self.section, alog=l, lid=lid)
+
+
+class ClearLog(_Page):
+    def post(self):
+        self.application.clear_logs()
+        self.redirect("/log")
diff --git a/libpathod/pathod.py b/libpathod/pathod.py
index 476d2676..f712582e 100644
--- a/libpathod/pathod.py
+++ b/libpathod/pathod.py
@@ -1,265 +1,14 @@
-import urllib, pprint
-import tornado.web, tornado.template, tornado.ioloop, tornado.httpserver
-import rparse, utils, version
+import netlib
 
+class PathodHandler(netlib.BaseHandler):
+    def handle(self):
+        print "Here"
 
-class Pathod(object):
-    def __init__(self, spec, application, request, **settings):
-        self.application, self.request, self.settings = application, request, settings
-        try:
-            self.response = rparse.parse(self.settings, spec)
-        except rparse.ParseException, v:
-            self.response = rparse.InternalResponse(
-                800,
-                "Error parsing response spec: %s\n"%v.msg + v.marked()
-            )
 
-    def _execute(self, transforms, *args, **kwargs):
-        d = self.response.serve(self.request)
-        d["request"] = dict(
-            path = self.request.path,
-            method = self.request.method,
-            headers = self.request.headers,
-            host = self.request.host,
-            protocol = self.request.protocol,
-            remote_address = self.request.connection.address,
-            full_url = self.request.full_url(),
-            query = self.request.query,
-            version = self.request.version,
-            uri = self.request.uri,
-        )
-        self.application.add_log(d)
-
-
-class RequestPathod(Pathod):
-    anchor = "/p/"
-    def __init__(self, application, request, **settings):
-        spec = urllib.unquote(request.uri)[len(self.anchor):]
-        Pathod.__init__(self, spec, application, request, **settings)
-
-
-class PathodApp(tornado.web.Application):
-    LOGBUF = 500
-    def __init__(self, **settings):
-        self.appsettings = settings
-        tornado.web.Application.__init__(
-            self,
-            [
-                (r"/", Index),
-                (r"/log", Log),
-                (r"/log/clear", ClearLog),
-                (r"/log/([0-9]+)", OneLog),
-                (r"/help", Help),
-                (r"/preview", Preview),
-                (r"/api/shutdown", APIShutdown),
-                (r"/api/info", APIInfo),
-                (r"/api/log", APILog),
-                (r"/api/log/clear", APILogClear),
-                (r"/p/.*", RequestPathod, settings),
-            ],
-            static_path = utils.data.path("static"),
-            template_path = utils.data.path("templates"),
-            debug=True
-        )
-        self.log = []
-        self.logid = 0
-
-    def add_anchor(self, pattern, spec):
-        """
-            Anchors are added to the beginning of the handlers.
-        """
-        # We assume we have only one host...
-        l = self.handlers[0][1]
-        class FixedPathod(Pathod):
-            def __init__(self, application, request, **settings):
-                Pathod.__init__(self, spec, application, request, **settings)
-        FixedPathod.spec = spec
-        FixedPathod.pattern = pattern
-        l.insert(0, tornado.web.URLSpec(pattern, FixedPathod, self.appsettings))
-
-    def get_anchors(self):
-        """
-            Anchors are added to the beginning of the handlers.
-        """
-        l = self.handlers[0][1]
-        a = []
-        for i in l:
-            if i.handler_class.__name__ == "FixedPathod":
-                a.append(
-                    (
-                        i.handler_class.pattern,
-                        i.handler_class.spec
-                    )
-                )
-        return a
-
-    def remove_anchor(self, pattern, spec):
-        """
-            Anchors are added to the beginning of the handlers.
-        """
-        l = self.handlers[0][1]
-        for i, h in enumerate(l):
-            if h.handler_class.__name__ == "FixedPathod":
-                if (h.handler_class.pattern, h.handler_class.spec) == (pattern, spec):
-                    del l[i]
-                    return
-
-    def add_log(self, d):
-        d["id"] = self.logid
-        self.log.insert(0, d)
-        if len(self.log) > self.LOGBUF:
-            self.log.pop()
-        self.logid += 1
-
-    def log_by_id(self, id):
-        for i in self.log:
-            if i["id"] == id:
-                return i
-
-    def clear_log(self):
-        self.log = []
-
-    def get_log(self):
-        return self.log
-
-
-def make_app(staticdir=None, anchors=()):
-    """
-        staticdir: A directory for static assets referenced in response patterns.
-        anchors: A sequence of strings of the form "pattern=pagespec"
-    """
-    settings = dict(
-        staticdir=staticdir
-    )
-    application = PathodApp(**settings)
-    for i in anchors:
-        rex, spec = utils.parse_anchor_spec(i, settings)
-        application.add_anchor(rex, spec)
-    return application
-
-
-def make_server(application, port, address, ssl_options):
-    """
-        Returns a (server, port) tuple.
-
-        The returned port will match the passed port, unless the passed port
-        was 0. In that case, an arbitrary empty port will be bound to, and this
-        new port will be returned.
-    """
-    http_server = tornado.httpserver.HTTPServer(
-        application,
-        ssl_options=ssl_options
-    )
-    http_server.listen(port, address)
-    port = port
-    for i in http_server._sockets.values():
-        sn = i.getsockname()
-        if sn[0] == address:
-            port = sn[1]
-    return http_server, port
-
-
-# begin nocover
-def run(server):
-    tornado.ioloop.IOLoop.instance().start()
-    server.stop()
-
-
-class APILog(tornado.web.RequestHandler):
-    def get(self):
-        self.write(
-            dict(
-                d = self.application.get_log()
-            )
-        )
-
-
-class APILogClear(tornado.web.RequestHandler):
-    def post(self):
-        self.application.clear_log()
-        self.write("OK")
-
-
-class APIShutdown(tornado.web.RequestHandler):
-    def post(self):
-        tornado.ioloop.IOLoop.instance().stop()
-        self.write("OK")
-
-
-class APIInfo(tornado.web.RequestHandler):
-    def get(self):
-        self.write(
-            dict(
-                version = version.IVERSION
-            )
-        )
-
-
-class _Page(tornado.web.RequestHandler):
-    def render(self, name, **kwargs):
-        tornado.web.RequestHandler.render(self, name + ".html", **kwargs)
-
-
-class Index(_Page):
-    name = "index"
-    section = "main"
-    def get(self):
-        self.render(self.name, section=self.section, spec="")
-
-
-class Preview(_Page):
-    name = "preview"
-    section = "main"
-    SANITY = 1024*1024
-    def get(self):
-        spec = self.get_argument("spec", None)
-        args = dict(
-            spec = spec,
-            section = self.section,
-            syntaxerror = None,
-            error = None
-        )
-        try:
-            r = rparse.parse(self.application.settings, spec)
-        except rparse.ParseException, v:
-            args["syntaxerror"] = str(v)
-            args["marked"] = v.marked()
-            return self.render(self.name, **args)
-        if r.length() > self.SANITY:
-            error = "Refusing to preview a response of %s bytes. This is for your own good."%r.length()
-            args["error"] = error
-        else:
-            d = utils.DummyRequest()
-            r.serve(d)
-            args["output"] = d.getvalue()
-        self.render(self.name, **args)
-
-
-class Help(_Page):
-    name = "help"
-    section = "help"
-    def get(self):
-        self.render(self.name, section=self.section)
-
-
-class Log(_Page):
-    name = "log"
-    section = "log"
-    def get(self):
-        self.render(self.name, section=self.section, log=self.application.log)
-
-
-class OneLog(_Page):
-    name = "onelog"
-    section = "log"
-    def get(self, lid):
-        l = pprint.pformat(self.application.log_by_id(int(lid)))
-        self.render(self.name, section=self.section, alog=l, lid=lid)
-
-
-class ClearLog(_Page):
-    def post(self):
-        self.application.clear_logs()
-        self.redirect("/log")
+class PathodServer(netlib.TCPServer):
+    def __init__(self, addr):
+        netlib.TCPServer.__init__(self, addr)
 
+    def handle_connection(self, request, client_address):
+        PathodHandler(request, client_address, self)
 
diff --git a/libpathod/test.py b/libpathod/test.py
index dcbb65d5..a66124a3 100644
--- a/libpathod/test.py
+++ b/libpathod/test.py
@@ -29,7 +29,6 @@ class PaThread(threading.Thread):
         self.q, self.app, self.ssl = q, app, ssl
         self.port = None
 
-# begin nocover
     def run(self):
         if self.ssl is True:
             ssloptions = dict(