Whitespace, interface refcactoring, sketch websockets language

7 files changed, 145 insertions, 57 deletions

diff --git a/libpathod/app.py b/libpathod/app.py
index d23d26a0..858c1d86 100644
--- a/libpathod/app.py
+++ b/libpathod/app.py
@@ -51,7 +51,9 @@ def make_app(noapi):
     @app.route('/download')
     @app.route('/download.html')
     def download():
-        return render("download.html", True, section="download", version=version.VERSION)
+        return render(
+            "download.html", True, section="download", version=version.VERSION
+        )
 
     @app.route('/about')
     @app.route('/about.html')
@@ -60,7 +62,9 @@ def make_app(noapi):
 
     @app.route('/docs/pathod')
     def docs_pathod():
-        return render("docs_pathod.html", True, section="docs", subsection="pathod")
+        return render(
+            "docs_pathod.html", True, section="docs", subsection="pathod"
+        )
 
     @app.route('/docs/language')
     def docs_language():
@@ -72,21 +76,32 @@ def make_app(noapi):
 
     @app.route('/docs/pathoc')
     def docs_pathoc():
-        return render("docs_pathoc.html", True, section="docs", subsection="pathoc")
+        return render(
+            "docs_pathoc.html", True, section="docs", subsection="pathoc"
+        )
 
     @app.route('/docs/libpathod')
     def docs_libpathod():
-        return render("docs_libpathod.html", True, section="docs", subsection="libpathod")
+        return render(
+            "docs_libpathod.html", True, section="docs", subsection="libpathod"
+        )
 
     @app.route('/docs/test')
     def docs_test():
-        return render("docs_test.html", True, section="docs", subsection="test")
+        return render(
+            "docs_test.html", True, section="docs", subsection="test"
+        )
 
     @app.route('/log')
     def log():
         if app.config["pathod"].noapi:
             abort(404)
-        return render("log.html", False, section="log", log=app.config["pathod"].get_log())
+        return render(
+            "log.html",
+            False,
+            section="log",
+            log=app.config["pathod"].get_log()
+        )
 
     @app.route('/log/<int:lid>')
     def onelog(lid):
@@ -127,14 +142,22 @@ def make_app(noapi):
         s = cStringIO.StringIO()
         safe = r.preview_safe()
 
-        c = app.config["pathod"].check_policy(safe, app.config["pathod"].request_settings)
+        c = app.config["pathod"].check_policy(
+            safe,
+            app.config["pathod"].request_settings
+        )
         if c:
             args["error"] = c
             return render(template, False, **args)
         if is_request:
-            language.serve(safe, s, app.config["pathod"].request_settings, "example.com")
+            language.serve(
+                safe,
+                s,
+                app.config["pathod"].request_settings,
+                request_host = "example.com"
+            )
         else:
-            language.serve(safe, s, app.config["pathod"].request_settings, None)
+            language.serve(safe, s, app.config["pathod"].request_settings)
 
         args["output"] = utils.escape_unprintables(s.getvalue())
         return render(template, False, **args)
diff --git a/libpathod/language.py b/libpathod/language.py
index 5c53453d..b7b95ed8 100644
--- a/libpathod/language.py
+++ b/libpathod/language.py
@@ -34,7 +34,7 @@ class ParseException(Exception):
         self.col = col
 
     def marked(self):
-        return "%s\n%s"%(self.s, " "*(self.col - 1) + "^")
+        return "%s\n%s"%(self.s, " " * (self.col - 1) + "^")
 
     def __str__(self):
         return "%s at char %s"%(self.msg, self.col)
@@ -46,9 +46,9 @@ def send_chunk(fp, val, blocksize, start, end):
     """
     for i in range(start, end, blocksize):
         fp.write(
-            val[i:min(i+blocksize, end)]
+            val[i:min(i + blocksize, end)]
         )
-    return end-start
+    return end - start
 
 
 def write_values(fp, vals, actions, sofar=0, blocksize=BLOCKSIZE):
@@ -74,7 +74,7 @@ def write_values(fp, vals, actions, sofar=0, blocksize=BLOCKSIZE):
                     v,
                     blocksize,
                     offset,
-                    a[0]-sofar-offset
+                    a[0] - sofar - offset
                 )
                 if a[1] == "pause":
                     time.sleep(a[2])
@@ -97,7 +97,7 @@ def write_values(fp, vals, actions, sofar=0, blocksize=BLOCKSIZE):
         return True
 
 
-def serve(msg, fp, settings, request_host=None):
+def serve(msg, fp, settings, **kwargs):
     """
         fp: The file pointer to write to.
 
@@ -107,7 +107,7 @@ def serve(msg, fp, settings, request_host=None):
 
         Calling this function may modify the object.
     """
-    msg = msg.resolve(settings, request_host)
+    msg = msg.resolve(settings, **kwargs)
     started = time.time()
 
     vals = msg.values(settings)
@@ -596,6 +596,7 @@ class Path(_Component):
 
 class Method(_Component):
     methods = [
+        "ws",
         "get",
         "head",
         "post",
@@ -845,31 +846,6 @@ class _Message(object):
                 l += len(i.value.get_generator(settings))
         return l
 
-    def resolve(self, settings, request_host):
-        tokens = self.tokens[:]
-        if not self.raw:
-            if not utils.get_header("Content-Length", self.headers):
-                if not self.body:
-                    length = 0
-                else:
-                    length = len(self.body.value.get_generator(settings))
-                tokens.append(
-                    Header(
-                        ValueLiteral("Content-Length"),
-                        ValueLiteral(str(length)),
-                    )
-                )
-            if request_host:
-                if not utils.get_header("Host", self.headers):
-                    tokens.append(
-                        Header(
-                            ValueLiteral("Host"),
-                            ValueLiteral(request_host)
-                        )
-                    )
-        intermediate = self.__class__(tokens)
-        return self.__class__([i.resolve(intermediate, settings) for i in tokens])
-
     @abc.abstractmethod
     def preamble(self, settings): # pragma: no cover
         pass
@@ -907,8 +883,8 @@ class _Message(object):
             vals.append(self.body.value.get_generator(settings))
         return vals
 
-    def freeze(self, settings, request_host=None):
-        r = self.resolve(settings, request_host=None)
+    def freeze(self, settings, **kwargs):
+        r = self.resolve(settings, **kwargs)
         return self.__class__([i.freeze(settings) for i in r.tokens])
 
     def __repr__(self):
@@ -957,6 +933,25 @@ class Response(_Message):
             )
         return l
 
+    def resolve(self, settings):
+        tokens = self.tokens[:]
+        if not self.raw:
+            if not utils.get_header("Content-Length", self.headers):
+                if not self.body:
+                    length = 0
+                else:
+                    length = len(self.body.value.get_generator(settings))
+                tokens.append(
+                    Header(
+                        ValueLiteral("Content-Length"),
+                        ValueLiteral(str(length)),
+                    )
+                )
+        intermediate = self.__class__(tokens)
+        return self.__class__(
+            [i.resolve(intermediate, settings) for i in tokens]
+        )
+
     @classmethod
     def expr(klass):
         parts = [i.expr() for i in klass.comps]
@@ -1009,6 +1004,32 @@ class Request(_Message):
         v.append(self.version)
         return v
 
+    def resolve(self, settings, **kwargs):
+        tokens = self.tokens[:]
+        if not self.raw:
+            if not utils.get_header("Content-Length", self.headers):
+                if self.body:
+                    length = len(self.body.value.get_generator(settings))
+                    tokens.append(
+                        Header(
+                            ValueLiteral("Content-Length"),
+                            ValueLiteral(str(length)),
+                        )
+                    )
+            request_host = kwargs.get("request_host")
+            if request_host:
+                if not utils.get_header("Host", self.headers):
+                    tokens.append(
+                        Header(
+                            ValueLiteral("Host"),
+                            ValueLiteral(request_host)
+                        )
+                    )
+        intermediate = self.__class__(tokens)
+        return self.__class__(
+            [i.resolve(intermediate, settings) for i in tokens]
+        )
+
     @classmethod
     def expr(klass):
         parts = [i.expr() for i in klass.comps]
@@ -1027,6 +1048,32 @@ class Request(_Message):
         return ":".join([i.spec() for i in self.tokens])
 
 
+class WebsocketFrame(_Message):
+    comps = (
+        Body,
+        PauseAt,
+        DisconnectAt,
+        InjectAt
+    )
+    logattrs = ["body"]
+
+    @classmethod
+    def expr(klass):
+        parts = [i.expr() for i in klass.comps]
+        atom = pp.MatchFirst(parts)
+        resp = pp.And(
+            [
+                pp.Literal("ws"),
+                Sep,
+                pp.ZeroOrMore(Sep + atom)
+            ]
+        )
+        return resp
+
+    def spec(self):
+        return ":".join([i.spec() for i in self.tokens])
+
+
 class PathodErrorResponse(Response):
     pass
 
diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py
index 616550fa..db58394d 100644
--- a/libpathod/pathoc.py
+++ b/libpathod/pathoc.py
@@ -92,11 +92,11 @@ class Pathoc(tcp.TCPClient):
             showresp = False,
             explain = False,
             hexdump = False,
-            ignorecodes = False,
+            ignorecodes = (),
             ignoretimeout = False,
             showsummary = False,
             fp = sys.stderr
-        ):
+    ):
         """
             spec: A request specification
             showreq: Print requests
@@ -138,13 +138,15 @@ class Pathoc(tcp.TCPClient):
             raise PathocError("Proxy CONNECT failed")
         parsed = http.parse_response_line(l)
         if not parsed[1] == 200:
-            raise PathocError("Proxy CONNECT failed: %s - %s"%(parsed[1], parsed[2]))
+            raise PathocError(
+                "Proxy CONNECT failed: %s - %s"%(parsed[1], parsed[2])
+            )
         http.read_headers(self.rfile)
 
     def connect(self, connect_to=None, showssl=False, fp=sys.stdout):
         """
-            connect_to: A (host, port) tuple, which will be connected to with an
-            HTTP CONNECT request.
+            connect_to: A (host, port) tuple, which will be connected to with
+            an HTTP CONNECT request.
         """
         tcp.TCPClient.connect(self)
         if connect_to:
@@ -203,10 +205,12 @@ class Pathoc(tcp.TCPClient):
                 r,
                 self.wfile,
                 self.settings,
-                self.address.host
+                requets_host = self.address.host
             )
             self.wfile.flush()
-            resp = list(http.read_response(self.rfile, r.method.string(), None))
+            resp = list(
+                http.read_response(self.rfile, r.method.string(), None)
+            )
             resp.append(self.sslinfo)
             resp = Response(*resp)
         except http.HttpError, v:
@@ -225,7 +229,7 @@ class Pathoc(tcp.TCPClient):
             raise
         finally:
             if req:
-                if self.ignorecodes and resp and resp.status_code in self.ignorecodes:
+                if resp and resp.status_code in self.ignorecodes:
                     resp = None
                 else:
                     if self.explain:
@@ -233,7 +237,9 @@ class Pathoc(tcp.TCPClient):
 
                     if self.showreq:
                         self._show(
-                            self.fp, ">> Request", self.wfile.get_log(), self.hexdump
+                            self.fp, ">> Request",
+                            self.wfile.get_log(),
+                            self.hexdump
                         )
 
                     if self.showsummary and resp:
diff --git a/libpathod/pathod.py b/libpathod/pathod.py
index 07354aa8..1c23baae 100644
--- a/libpathod/pathod.py
+++ b/libpathod/pathod.py
@@ -77,13 +77,12 @@ class PathodHandler(tcp.BaseHandler):
             return False, log
 
         if self.server.explain and not isinstance(crafted, language.PathodErrorResponse):
-            crafted = crafted.freeze(self.server.request_settings, None)
+            crafted = crafted.freeze(self.server.request_settings)
             self.info(">> Spec: %s" % crafted.spec())
         response_log = language.serve(
             crafted,
             self.wfile,
-            self.server.request_settings,
-            None
+            self.server.request_settings
         )
         if response_log["disconnect"]:
             return False, response_log
diff --git a/libpathod/templates/docs_lang.html b/libpathod/templates/docs_lang.html
index 116dab87..4ed7f151 100644
--- a/libpathod/templates/docs_lang.html
+++ b/libpathod/templates/docs_lang.html
@@ -103,6 +103,17 @@
 
         <table class="table table-bordered">
             <tbody >
+
+                <tr>
+                    <td> method </td>
+                    <td>
+                        A <a href="#valuespec">VALUE</a> specifying the HTTP
+                        method to use. Standard methods do not need to be
+                        quoted. The special method <b>ws</b> creates a valid
+                        websocket upgrade request.
+                    </td>
+                </tr>
+
                 <tr>
                     <td> b<a href="#valuespec">VALUE</a> </td>
                     <td>
diff --git a/test/test_language.py b/test/test_language.py
index 25f4eec4..cd7f703e 100644
--- a/test/test_language.py
+++ b/test/test_language.py
@@ -124,7 +124,9 @@ class TestValueFile:
             assert v.get_generator(dict(staticdir=t))
 
             v = language.Value.parseString("<path2")[0]
-            tutils.raises(language.FileAccessDenied, v.get_generator, dict(staticdir=t))
+            tutils.raises(
+                language.FileAccessDenied, v.get_generator, dict(staticdir=t)
+            )
             tutils.raises("access disabled", v.get_generator, dict())
 
             v = language.Value.parseString("</outside")[0]
@@ -554,7 +556,7 @@ class TestRequest:
     def test_render(self):
         s = cStringIO.StringIO()
         r = parse_request("GET:'/foo'")
-        assert language.serve(r, s, {}, "foo.com")
+        assert language.serve(r, s, {}, request_host = "foo.com")
 
     def test_multiline(self):
         l = """
diff --git a/test/test_pathoc.py b/test/test_pathoc.py
index 90f798c9..52075231 100644
--- a/test/test_pathoc.py
+++ b/test/test_pathoc.py
@@ -51,7 +51,7 @@ class _TestDaemon:
         showssl=False,
         hexdump=False,
         timeout=None,
-        ignorecodes=None,
+        ignorecodes=(),
         ignoretimeout=None,
         showsummary=True
     ):