Parse patterns eagerly on instantiation

7 files changed, 155 insertions, 108 deletions

diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py
index 2c6e094e..1a000a93 100644
--- a/libpathod/cmdline.py
+++ b/libpathod/cmdline.py
@@ -3,7 +3,8 @@ import argparse
 import os
 import os.path
 import sys
-from . import pathoc, pathod, version, utils
+import re
+from . import pathoc, pathod, version, utils, language
 from netlib import http_uastrings
 
 
@@ -78,7 +79,10 @@ def go_pathoc():
     group.add_argument(
         "--sslversion", dest="sslversion", type=int, default=4,
         choices=[1, 2, 3, 4],
-        help="Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default to SSLv23."
+        help="""
+            Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default
+            to SSLv23.
+        """
     )
 
     group = parser.add_argument_group(
@@ -149,9 +153,14 @@ def go_pathoc():
     for r in args.request:
         if os.path.exists(r):
             data = open(r).read()
-            reqs.append(data)
-        else:
-            reqs.append(r)
+            r = data
+        try:
+            req = language.parse_request(r)
+        except language.ParseException, v:
+            print >> sys.stderr, "Error parsing request spec: %s"%v.msg
+            print >> sys.stderr, v.marked()
+            sys.exit(1)
+        reqs.append(req)
     args.request = reqs
 
     pathoc.main(args)
@@ -267,7 +276,8 @@ def go_pathod():
     group.add_argument(
         "--sslversion", dest="sslversion", type=int, default=4,
         choices=[1, 2, 3, 4],
-        help="Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default to SSLv23."
+        help=""""Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default
+        to SSLv23."""
     )
 
     group = parser.add_argument_group(
@@ -327,13 +337,23 @@ def go_pathod():
     args.sizelimit = sizelimit
 
     anchors = []
-    for patt, spec in anchors:
+    for patt, spec in args.anchors:
         if os.path.exists(spec):
             data = open(spec).read()
-            anchors.append((patt, data))
-        else:
-            anchors.append((patt, spec))
+            spec = data
+
+        try:
+            req = language.parse_response(spec)
+        except language.ParseException, v:
+            print >> sys.stderr, "Error parsing anchor spec: %s"%v.msg
+            print >> sys.stderr, v.marked()
+            sys.exit(1)
+        try:
+            arex = re.compile(patt)
+        except re.error:
+            print >> sys.stderr, "Invalid regex in anchor: %s" % patt
+            sys.exit(1)
+        anchors.append((arex, req))
     args.anchors = anchors
 
     pathod.main(args)
-
diff --git a/libpathod/language.py b/libpathod/language.py
index 2969055b..b4b59167 100644
--- a/libpathod/language.py
+++ b/libpathod/language.py
@@ -1018,7 +1018,7 @@ def read_file(settings, s):
 
 def parse_response(s):
     """
-        May raise ParseException or FileAccessDenied
+        May raise ParseException
     """
     try:
         s = s.decode("ascii")
@@ -1032,7 +1032,7 @@ def parse_response(s):
 
 def parse_request(s):
     """
-        May raise ParseException or FileAccessDenied
+        May raise ParseException
     """
     try:
         s = s.decode("ascii")
diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py
index e534bba5..ae1e98cf 100644
--- a/libpathod/pathoc.py
+++ b/libpathod/pathoc.py
@@ -2,10 +2,14 @@ import sys
 import os
 from netlib import tcp, http, certutils
 import netlib.utils
-import language, utils
+
+import language
+import utils
 import OpenSSL.crypto
 
-class PathocError(Exception): pass
+
+class PathocError(Exception):
+    pass
 
 
 class SSLInfo:
@@ -24,7 +28,14 @@ class Response:
 
 
 class Pathoc(tcp.TCPClient):
-    def __init__(self, address, ssl=None, sni=None, sslversion=4, clientcert=None, ciphers=None):
+    def __init__(
+            self,
+            address,
+            ssl=None,
+            sni=None,
+            sslversion=4,
+            clientcert=None,
+            ciphers=None):
         tcp.TCPClient.__init__(self, address)
         self.settings = dict(
             staticdir = os.getcwd(),
@@ -37,9 +48,9 @@ class Pathoc(tcp.TCPClient):
 
     def http_connect(self, connect_to):
         self.wfile.write(
-                    'CONNECT %s:%s HTTP/1.1\r\n'%tuple(connect_to) +
-                    '\r\n'
-                    )
+            'CONNECT %s:%s HTTP/1.1\r\n'%tuple(connect_to) +
+            '\r\n'
+        )
         self.wfile.flush()
         l = self.rfile.readline()
         if not l:
@@ -61,17 +72,17 @@ class Pathoc(tcp.TCPClient):
         if self.ssl:
             try:
                 self.convert_to_ssl(
-                        sni=self.sni,
-                        cert=self.clientcert,
-                        method=self.sslversion,
-                        cipher_list = self.ciphers
-                    )
+                    sni=self.sni,
+                    cert=self.clientcert,
+                    method=self.sslversion,
+                    cipher_list = self.ciphers
+                )
             except tcp.NetLibError, v:
                 raise PathocError(str(v))
             self.sslinfo = SSLInfo(
-                        self.connection.get_peer_cert_chain(),
-                        self.get_current_cipher()
-                    )
+                self.connection.get_peer_cert_chain(),
+                self.get_current_cipher()
+            )
 
     def request(self, spec):
         """
@@ -88,7 +99,9 @@ class Pathoc(tcp.TCPClient):
         return Response(*ret)
 
     def _show_summary(self, fp, httpversion, code, msg, headers, content):
-        print >> fp, "<< %s %s: %s bytes"%(code, utils.xrepr(msg), len(content))
+        print >> fp, "<< %s %s: %s bytes"%(
+            code, utils.xrepr(msg), len(content)
+        )
 
     def _show(self, fp, header, data, hexdump):
         if hexdump:
@@ -99,7 +112,18 @@ class Pathoc(tcp.TCPClient):
             print >> fp, "%s (unprintables escaped):"%header
             print >> fp, netlib.utils.cleanBin(data)
 
-    def print_request(self, spec, showreq, showresp, explain, showssl, hexdump, ignorecodes, ignoretimeout, fp=sys.stdout):
+    def print_request(
+        self,
+        r,
+        showreq,
+        showresp,
+        explain,
+        showssl,
+        hexdump,
+        ignorecodes,
+        ignoretimeout,
+        fp=sys.stdout
+    ):
         """
             Performs a series of requests, and prints results to the specified
             file descriptor.
@@ -114,16 +138,6 @@ class Pathoc(tcp.TCPClient):
 
             Returns True if we have a non-ignored response.
         """
-        try:
-            r = language.parse_request(spec)
-        except language.ParseException, v:
-            print >> fp, "Error parsing request spec: %s"%v.msg
-            print >> fp, v.marked()
-            return
-        except language.FileAccessDenied, v:
-            print >> fp, "File access error: %s"%v
-            return
-
         if explain:
             r = r.freeze(self.settings, self.address.host)
 
@@ -133,7 +147,12 @@ class Pathoc(tcp.TCPClient):
         if showresp:
             self.rfile.start_log()
         try:
-            req = language.serve(r, self.wfile, self.settings, self.address.host)
+            req = language.serve(
+                r,
+                self.wfile,
+                self.settings,
+                self.address.host
+            )
             self.wfile.flush()
             resp = http.read_response(self.rfile, r.method.string(), None)
         except http.HttpError, v:
@@ -174,13 +193,15 @@ class Pathoc(tcp.TCPClient):
                         print >> fp, "%s=%s"%cn,
                     print >> fp
                     print >> fp, "\tVersion: %s"%i.get_version()
-                    print >> fp, "\tValidity: %s - %s"%(i.get_notBefore(),i.get_notAfter())
+                    print >> fp, "\tValidity: %s - %s"%(
+                        i.get_notBefore(), i.get_notAfter()
+                    )
                     print >> fp, "\tSerial: %s"%i.get_serial_number()
                     print >> fp, "\tAlgorithm: %s"%i.get_signature_algorithm()
                     pk = i.get_pubkey()
                     types = {
-                            OpenSSL.crypto.TYPE_RSA: "RSA",
-                            OpenSSL.crypto.TYPE_DSA: "DSA"
+                        OpenSSL.crypto.TYPE_RSA: "RSA",
+                        OpenSSL.crypto.TYPE_DSA: "DSA"
                     }
                     t = types.get(pk.type(), "Uknown")
                     print >> fp, "\tPubkey: %s bit %s"%(pk.bits(), t)
diff --git a/libpathod/pathod.py b/libpathod/pathod.py
index 92e5b2db..173773cf 100644
--- a/libpathod/pathod.py
+++ b/libpathod/pathod.py
@@ -1,13 +1,12 @@
 import urllib
 import threading
-import re
 import logging
 import os
 import sys
 from netlib import tcp, http, wsgi, certutils
 import netlib.utils
 
-import version, app, language, utils
+from . import version, app, language, utils
 
 
 DEFAULT_CERT_DOMAIN = "pathod.net"
@@ -71,7 +70,12 @@ class PathodHandler(tcp.BaseHandler):
         if self.server.explain and not isinstance(crafted, language.PathodErrorResponse):
             crafted = crafted.freeze(self.server.request_settings, None)
             self.info(">> Spec: %s" % crafted.spec())
-        response_log = language.serve(crafted, self.wfile, self.server.request_settings, None)
+        response_log = language.serve(
+            crafted,
+            self.wfile,
+            self.server.request_settings,
+            None
+        )
         if response_log["disconnect"]:
             return False, response_log
         return True, response_log
@@ -169,8 +173,7 @@ class PathodHandler(tcp.BaseHandler):
         for i in self.server.anchors:
             if i[0].match(path):
                 self.info("crafting anchor: %s" % path)
-                aresp = language.parse_response(i[1])
-                again, retlog["response"] = self.serve_crafted(aresp)
+                again, retlog["response"] = self.serve_crafted(i[1])
                 return again, retlog
 
         if not self.server.nocraft and path.startswith(self.server.craftanchor):
@@ -189,7 +192,10 @@ class PathodHandler(tcp.BaseHandler):
         elif self.server.noweb:
             crafted = language.make_error_response("Access Denied")
             language.serve(crafted, self.wfile, self.server.request_settings)
-            return False, dict(type="error", msg="Access denied: web interface disabled")
+            return False, dict(
+                type="error",
+                msg="Access denied: web interface disabled"
+            )
         else:
             self.info("app: %s %s" % (method, path))
             req = wsgi.Request("http", method, path, headers, content)
@@ -259,19 +265,34 @@ class Pathod(tcp.TCPServer):
     LOGBUF = 500
 
     def __init__(
-            self, addr, confdir=CONFDIR, ssl=False, ssloptions=None,
-            craftanchor="/p/", staticdir=None, anchors=None,
-            sizelimit=None, noweb=False, nocraft=False, noapi=False,
-            nohang=False, timeout=None, logreq=False, logresp=False,
-            explain=False, hexdump=False
+        self,
+        addr,
+        confdir=CONFDIR,
+        ssl=False,
+        ssloptions=None,
+        craftanchor="/p/",
+        staticdir=None,
+        anchors=(),
+        sizelimit=None,
+        noweb=False,
+        nocraft=False,
+        noapi=False,
+        nohang=False,
+        timeout=None,
+        logreq=False,
+        logresp=False,
+        explain=False,
+        hexdump=False
     ):
         """
             addr: (address, port) tuple. If port is 0, a free port will be
             automatically chosen.
             ssloptions: an SSLOptions object.
-            craftanchor: string specifying the path under which to anchor response generation.
+            craftanchor: string specifying the path under which to anchor
+            response generation.
             staticdir: path to a directory of static resources, or None.
-            anchors: A list of (regex, spec) tuples, or None.
+            anchors: List of (regex object, language.Request object) tuples, or
+            None.
             sizelimit: Limit size of served data.
             nocraft: Disable response crafting.
             noapi: Disable the API.
@@ -283,26 +304,17 @@ class Pathod(tcp.TCPServer):
         self.staticdir = staticdir
         self.craftanchor = craftanchor
         self.sizelimit = sizelimit
-        self.noweb, self.nocraft, self.noapi, self.nohang = noweb, nocraft, noapi, nohang
-        self.timeout, self.logreq, self.logresp, self.hexdump = timeout, logreq, logresp, hexdump
+        self.noweb, self.nocraft = noweb, nocraft
+        self.noapi, self.nohang = noapi, nohang
+        self.timeout, self.logreq = timeout, logreq
+        self.logresp, self.hexdump = logresp, hexdump
         self.explain = explain
 
         self.app = app.make_app(noapi)
         self.app.config["pathod"] = self
         self.log = []
         self.logid = 0
-        self.anchors = []
-        if anchors:
-            for i in anchors:
-                try:
-                    arex = re.compile(i[0])
-                except re.error:
-                    raise PathodError("Invalid regex in anchor: %s" % i[0])
-                try:
-                    language.parse_response(i[1])
-                except language.ParseException, v:
-                    raise PathodError("Invalid page spec in anchor: '%s', %s" % (i[1], str(v)))
-                self.anchors.append((arex, i[1]))
+        self.anchors = anchors
 
     def check_policy(self, req, settings):
         """
diff --git a/test/test_pathoc.py b/test/test_pathoc.py
index 5172d85f..88479b6c 100644
--- a/test/test_pathoc.py
+++ b/test/test_pathoc.py
@@ -1,6 +1,8 @@
 import json
 import cStringIO
-from libpathod import pathoc, test, version, pathod
+import re
+
+from libpathod import pathoc, test, version, pathod, language
 import tutils
 
 
@@ -18,7 +20,9 @@ class _TestDaemon:
             ssl=self.ssl,
             ssloptions=self.ssloptions,
             staticdir=tutils.test_data.path("data"),
-            anchors=[("/anchor/.*", "202")]
+            anchors=[
+                (re.compile("/anchor/.*"), language.parse_response("202"))
+            ]
         )
 
     @classmethod
@@ -37,9 +41,18 @@ class _TestDaemon:
         r = c.request("get:/api/info")
         assert tuple(json.loads(r.content)["version"]) == version.IVERSION
 
-    def tval(self, requests, showreq=False, showresp=False, explain=False,
-                   showssl=False, hexdump=False, timeout=None, ignorecodes=None,
-                   ignoretimeout=None):
+    def tval(
+        self,
+        requests,
+        showreq=False,
+        showresp=False,
+        explain=False,
+        showssl=False,
+        hexdump=False,
+        timeout=None,
+        ignorecodes=None,
+        ignoretimeout=None
+    ):
         c = pathoc.Pathoc(("127.0.0.1", self.d.port), ssl=self.ssl)
         c.connect()
         if timeout:
@@ -47,7 +60,7 @@ class _TestDaemon:
         s = cStringIO.StringIO()
         for i in requests:
             c.print_request(
-                i,
+                language.parse_request(i),
                 showreq = showreq,
                 showresp = showresp,
                 explain = explain,
@@ -125,17 +138,14 @@ class TestDaemon(_TestDaemon):
         assert "HTTP/" in v
 
     def test_explain(self):
-        reqs = [ "get:/p/200:b@100" ]
-        assert not "b@100" in self.tval(reqs, explain=True)
+        reqs = ["get:/p/200:b@100"]
+        assert "b@100" not in self.tval(reqs, explain=True)
 
     def test_showreq(self):
-        reqs = [ "get:/api/info:p0,0", "get:/api/info:p0,0" ]
+        reqs = ["get:/api/info:p0,0", "get:/api/info:p0,0"]
         assert self.tval(reqs, showreq=True).count("unprintables escaped") == 2
         assert self.tval(reqs, showreq=True, hexdump=True).count("hex dump") == 2
 
-    def test_parse_err(self):
-        assert "Error parsing" in self.tval(["foo"])
-
     def test_conn_err(self):
         assert "Invalid server response" in self.tval(["get:'/p/200:d2'"])
 
@@ -152,6 +162,3 @@ class TestDaemon(_TestDaemon):
             "HTTP/1.1 200 OK\r\n"
         )
         c.http_connect(to)
-
-
-
diff --git a/test/test_pathod.py b/test/test_pathod.py
index 0172678c..158f3bda 100644
--- a/test/test_pathod.py
+++ b/test/test_pathod.py
@@ -1,28 +1,9 @@
 from libpathod import pathod, version
-from netlib import tcp, http, certutils
+from netlib import tcp, http
 import tutils
 
 
 class TestPathod:
-    def test_instantiation(self):
-        p = pathod.Pathod(
-            ("127.0.0.1", 0),
-            anchors = [(".*", "200:da")]
-        )
-        assert p.anchors
-        tutils.raises(
-            "invalid regex",
-            pathod.Pathod,
-            ("127.0.0.1", 0),
-            anchors=[("*", "200:da")]
-        )
-        tutils.raises(
-            "invalid page spec",
-            pathod.Pathod,
-            ("127.0.0.1", 0),
-            anchors=[("foo", "bar")]
-        )
-
     def test_logging(self):
         p = pathod.Pathod(("127.0.0.1", 0))
         assert len(p.get_log()) == 0
@@ -39,6 +20,7 @@ class TestPathod:
 
 class TestNoWeb(tutils.DaemonTests):
     noweb = True
+
     def test_noweb(self):
         assert self.get("200:da").status_code == 200
         assert self.getpath("/").status_code == 800
@@ -46,6 +28,7 @@ class TestNoWeb(tutils.DaemonTests):
 
 class TestTimeout(tutils.DaemonTests):
     timeout = 0.01
+
     def test_noweb(self):
         # FIXME: Add float values to spec language, reduce test timeout to
         # increase test performance
@@ -55,6 +38,7 @@ class TestTimeout(tutils.DaemonTests):
 
 class TestNoApi(tutils.DaemonTests):
     noapi = True
+
     def test_noapi(self):
         assert self.getpath("/log").status_code == 404
         r = self.getpath("/")
@@ -238,4 +222,3 @@ class TestDaemonSSL(CommonTests):
         r = self.pathoc(r"get:/p/202")
         assert r.status_code == 202
         assert self.d.last_log()["cipher"][1] > 0
-
diff --git a/test/tutils.py b/test/tutils.py
index 94c1ff9d..5876e5e6 100644
--- a/test/tutils.py
+++ b/test/tutils.py
@@ -1,10 +1,12 @@
 import tempfile
 import os
+import re
 import shutil
 from contextlib import contextmanager
-from libpathod import utils, test, pathoc, pathod
+from libpathod import utils, test, pathoc, pathod, language
 import requests
 
+
 class DaemonTests:
     noweb = False
     noapi = False
@@ -22,7 +24,9 @@ class DaemonTests:
         so = pathod.SSLOptions(**opts)
         self.d = test.Daemon(
             staticdir=test_data.path("data"),
-            anchors=[("/anchor/.*", "202:da")],
+            anchors=[
+                (re.compile("/anchor/.*"), language.parse_response("202:da"))
+            ],
             ssl = self.ssl,
             ssloptions = so,
             sizelimit=1*1024*1024,