Merge branch 'pathod-lang-http'

9 files changed, 115 insertions, 108 deletions

diff --git a/.travis.yml b/.travis.yml
index b77091b1..f6731c0b 100644
--- a/.travis.yml
+++ b/.travis.yml
@@ -22,9 +22,9 @@ matrix:
       git:
         depth: 9999999
     - python: 3.5
-      env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_base.py"
+      env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_base.py test/pathod/test_language_http.py"
     - python: 3.5
-      env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_base.py" NO_ALPN=1
+      env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_base.py test/pathod/test_language_http.py" NO_ALPN=1
     - python: 2.7
       env: DOCS=1
       script: 'cd docs && make html'
diff --git a/pathod/language/base.py b/pathod/language/base.py
index 1369a3c7..25f3fd1a 100644
--- a/pathod/language/base.py
+++ b/pathod/language/base.py
@@ -261,7 +261,7 @@ class _Component(Token):
 
     """
         A value component of the primary specification of an message.
-        Components produce byte values desribe the bytes of the message.
+        Components produce byte values describing the bytes of the message.
     """
 
     def values(self, settings):  # pragma: no cover
@@ -272,9 +272,9 @@ class _Component(Token):
 
     def string(self, settings=None):
         """
-            A string representation of the object.
+            A bytestring representation of the object.
         """
-        return "".join(i[:] for i in self.values(settings or {}))
+        return b"".join(i[:] for i in self.values(settings or {}))
 
 
 class KeyValue(_Component):
@@ -391,7 +391,7 @@ class Integer(_Component):
                 "Integer value must be between %s and %s." % self.bounds,
                 0, 0
             )
-        self.value = str(value)
+        self.value = str(value).encode()
 
     @classmethod
     def expr(cls):
@@ -401,10 +401,10 @@ class Integer(_Component):
         return e.setParseAction(lambda x: cls(*x))
 
     def values(self, settings):
-        return self.value
+        return [self.value]
 
     def spec(self):
-        return "%s%s" % (self.preamble, self.value)
+        return "%s%s" % (self.preamble, self.value.decode())
 
     def freeze(self, settings_):
         return self
@@ -555,7 +555,7 @@ class NestedMessage(Token):
         try:
             self.parsed = self.nest_type(
                 self.nest_type.expr().parseString(
-                    value.val,
+                    value.val.decode(),
                     parseAll=True
                 )
             )
@@ -578,4 +578,4 @@ class NestedMessage(Token):
 
     def freeze(self, settings):
         f = self.parsed.freeze(settings).spec()
-        return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f)))
+        return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f.encode())))
diff --git a/pathod/language/http.py b/pathod/language/http.py
index b2308d5e..4cc7db5f 100644
--- a/pathod/language/http.py
+++ b/pathod/language/http.py
@@ -57,7 +57,7 @@ class _HeaderMixin(object):
     unique_name = None
 
     def format_header(self, key, value):
-        return [key, ": ", value, "\r\n"]
+        return [key, b": ", value, b"\r\n"]
 
     def values(self, settings):
         return self.format_header(
@@ -88,7 +88,7 @@ class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue):
     def values(self, settings):
         value = self.value.val
         if self.option_used:
-            value = user_agents.get_by_shortcut(value.lower())[2]
+            value = user_agents.get_by_shortcut(value.lower().decode())[2].encode()
 
         return self.format_header(
             self.key.get_generator(settings),
@@ -109,7 +109,7 @@ def get_header(val, headers):
 
 
 class _HTTPMessage(message.Message):
-    version = "HTTP/1.1"
+    version = b"HTTP/1.1"
 
     @property
     def actions(self):
@@ -133,10 +133,10 @@ class _HTTPMessage(message.Message):
 
     def values(self, settings):
         vals = self.preamble(settings)
-        vals.append("\r\n")
+        vals.append(b"\r\n")
         for h in self.headers:
             vals.extend(h.values(settings))
-        vals.append("\r\n")
+        vals.append(b"\r\n")
         if self.body:
             vals.extend(self.body.values(settings))
         return vals
@@ -171,18 +171,18 @@ class Response(_HTTPMessage):
         return self.tok(Reason)
 
     def preamble(self, settings):
-        l = [self.version, " "]
+        l = [self.version, b" "]
         l.extend(self.status_code.values(settings))
         status_code = int(self.status_code.value)
-        l.append(" ")
+        l.append(b" ")
         if self.reason:
             l.extend(self.reason.values(settings))
         else:
             l.append(
                 status_codes.RESPONSES.get(
                     status_code,
-                    "Unknown code"
-                )
+                    b"Unknown code"
+                ).encode()
             )
         return l
 
@@ -205,8 +205,8 @@ class Response(_HTTPMessage):
                 if not get_header(i[0], self.headers):
                     tokens.append(
                         Header(
-                            base.TokValueLiteral(i[0]),
-                            base.TokValueLiteral(i[1]))
+                            base.TokValueLiteral(i[0].decode()),
+                            base.TokValueLiteral(i[1].decode()))
                     )
         if not self.raw:
             if not get_header("Content-Length", self.headers):
@@ -294,11 +294,11 @@ class Request(_HTTPMessage):
 
     def preamble(self, settings):
         v = self.method.values(settings)
-        v.append(" ")
+        v.append(b" ")
         v.extend(self.path.values(settings))
         if self.nested_response:
             v.append(self.nested_response.parsed.spec())
-        v.append(" ")
+        v.append(b" ")
         v.append(self.version)
         return v
 
@@ -314,8 +314,8 @@ class Request(_HTTPMessage):
                 if not get_header(i[0], self.headers):
                     tokens.append(
                         Header(
-                            base.TokValueLiteral(i[0]),
-                            base.TokValueLiteral(i[1])
+                            base.TokValueLiteral(i[0].decode()),
+                            base.TokValueLiteral(i[1].decode())
                         )
                     )
         if not self.raw:
diff --git a/pathod/language/message.py b/pathod/language/message.py
index 33124856..fea4f4de 100644
--- a/pathod/language/message.py
+++ b/pathod/language/message.py
@@ -1,5 +1,6 @@
 import abc
 from . import actions, exceptions
+from netlib import strutils
 
 LOG_TRUNCATE = 1024
 
@@ -49,7 +50,7 @@ class Message(object):
 
     def preview_safe(self):
         """
-            Return a copy of this message that issafe for previews.
+            Return a copy of this message that is safe for previews.
         """
         tokens = [i for i in self.tokens if not isinstance(i, actions.PauseAt)]
         return self.__class__(tokens)
@@ -80,10 +81,10 @@ class Message(object):
             # We truncate at 1k.
             if hasattr(v, "values"):
                 v = [x[:LOG_TRUNCATE] for x in v.values(settings)]
-                v = "".join(v).encode("string_escape")
+                v = strutils.bytes_to_escaped_str(b"".join(v))
             elif hasattr(v, "__len__"):
                 v = v[:LOG_TRUNCATE]
-                v = v.encode("string_escape")
+                v = strutils.bytes_to_escaped_str(v)
             ret[i] = v
         ret["spec"] = self.spec()
         return ret
diff --git a/setup.py b/setup.py
index 2e7f0a0b..dd34465c 100644
--- a/setup.py
+++ b/setup.py
@@ -75,7 +75,7 @@ setup(
         "passlib>=1.6.5, <1.7",
         "pyasn1>=0.1.9, <0.2",
         "pyOpenSSL>=16.0, <17.0",
-        "pyparsing>=2.0, <2.1",  # 2.1.1 breaks our binaries, see https://sourceforge.net/p/pyparsing/bugs/93/
+        "pyparsing>=2.1.3, <2.2",
         "pyperclip>=1.5.22, <1.6",
         "requests>=2.9.1, <2.10",
         "six>=1.10, <1.11",
diff --git a/test/pathod/test_language_base.py b/test/pathod/test_language_base.py
index 075dc2b8..7c7d8cf9 100644
--- a/test/pathod/test_language_base.py
+++ b/test/pathod/test_language_base.py
@@ -55,8 +55,15 @@ class TestTokValueLiteral:
         v = base.TokValueLiteral("f\x00oo")
         assert v.spec() == repr(v) == r"'f\x00oo'"
 
-        v = base.TokValueLiteral("\"")
-        assert v.spec() == repr(v) == '\'"\''
+        v = base.TokValueLiteral('"')
+        assert v.spec() == repr(v) == """  '"'  """.strip()
+
+        # While pyparsing has a escChar argument for QuotedString,
+        # escChar only performs scapes single-character escapes and does not work for e.g. r"\x02".
+        # Thus, we cannot use that option, which means we cannot have single quotes in strings.
+        # To fix this, we represent single quotes as r"\x07".
+        v = base.TokValueLiteral("'")
+        assert v.spec() == r"'\x27'"
 
     def roundtrip(self, spec):
         e = base.TokValueLiteral.expr()
@@ -311,7 +318,7 @@ def test_options_or_value():
 def test_integer():
     e = base.Integer.expr()
     v = e.parseString("200")[0]
-    assert v.string() == "200"
+    assert v.string() == b"200"
     assert v.spec() == "200"
 
     assert v.freeze({}).value == v.value
diff --git a/test/pathod/test_language_http.py b/test/pathod/test_language_http.py
index d1870a63..18059e3a 100644
--- a/test/pathod/test_language_http.py
+++ b/test/pathod/test_language_http.py
@@ -1,15 +1,15 @@
-from six.moves import cStringIO as StringIO
+from six import BytesIO
 from pathod import language
 from pathod.language import http, base
 import tutils
 
 
 def parse_request(s):
-    return language.parse_pathoc(s).next()
+    return next(language.parse_pathoc(s))
 
 
 def test_make_error_response():
-    d = StringIO()
+    d = BytesIO()
     s = http.make_error_response("foo")
     language.serve(s, d, {})
 
@@ -24,17 +24,17 @@ class TestRequest:
 
     def test_simple(self):
         r = parse_request('GET:"/foo"')
-        assert r.method.string() == "GET"
-        assert r.path.string() == "/foo"
+        assert r.method.string() == b"GET"
+        assert r.path.string() == b"/foo"
         r = parse_request('GET:/foo')
-        assert r.path.string() == "/foo"
+        assert r.path.string() == b"/foo"
         r = parse_request('GET:@1k')
         assert len(r.path.string()) == 1024
 
     def test_multiple(self):
         r = list(language.parse_pathoc("GET:/ PUT:/"))
-        assert r[0].method.string() == "GET"
-        assert r[1].method.string() == "PUT"
+        assert r[0].method.string() == b"GET"
+        assert r[1].method.string() == b"PUT"
         assert len(r) == 2
 
         l = """
@@ -54,8 +54,8 @@ class TestRequest:
         """
         r = list(language.parse_pathoc(l))
         assert len(r) == 2
-        assert r[0].method.string() == "GET"
-        assert r[1].method.string() == "PUT"
+        assert r[0].method.string() == b"GET"
+        assert r[1].method.string() == b"PUT"
 
         l = """
             get:"http://localhost:9999/p/200":ir,@1
@@ -63,8 +63,8 @@ class TestRequest:
         """
         r = list(language.parse_pathoc(l))
         assert len(r) == 2
-        assert r[0].method.string() == "GET"
-        assert r[1].method.string() == "GET"
+        assert r[0].method.string() == b"GET"
+        assert r[1].method.string() == b"GET"
 
     def test_nested_response(self):
         l = "get:/p:s'200'"
@@ -75,7 +75,7 @@ class TestRequest:
         assert r[0].values({})
 
     def test_render(self):
-        s = StringIO()
+        s = BytesIO()
         r = parse_request("GET:'/foo'")
         assert language.serve(
             r,
@@ -90,8 +90,8 @@ class TestRequest:
             ir,@1
         """
         r = parse_request(l)
-        assert r.method.string() == "GET"
-        assert r.path.string() == "/foo"
+        assert r.method.string() == b"GET"
+        assert r.path.string() == b"/foo"
         assert r.actions
 
         l = """
@@ -106,8 +106,8 @@ class TestRequest:
             ir,@1
         """
         r = parse_request(l)
-        assert r.method.string() == "GET"
-        assert r.path.string().endswith("bar")
+        assert r.method.string() == b"GET"
+        assert r.path.string().endswith(b"bar")
         assert r.actions
 
     def test_spec(self):
@@ -128,66 +128,66 @@ class TestRequest:
     def test_websocket(self):
         r = parse_request('ws:/path/')
         res = r.resolve(language.Settings())
-        assert res.method.string().lower() == "get"
-        assert res.tok(http.Path).value.val == "/path/"
-        assert res.tok(http.Method).value.val.lower() == "get"
-        assert http.get_header("Upgrade", res.headers).value.val == "websocket"
+        assert res.method.string().lower() == b"get"
+        assert res.tok(http.Path).value.val == b"/path/"
+        assert res.tok(http.Method).value.val.lower() == b"get"
+        assert http.get_header(b"Upgrade", res.headers).value.val == b"websocket"
 
         r = parse_request('ws:put:/path/')
         res = r.resolve(language.Settings())
-        assert r.method.string().lower() == "put"
-        assert res.tok(http.Path).value.val == "/path/"
-        assert res.tok(http.Method).value.val.lower() == "put"
-        assert http.get_header("Upgrade", res.headers).value.val == "websocket"
+        assert r.method.string().lower() == b"put"
+        assert res.tok(http.Path).value.val == b"/path/"
+        assert res.tok(http.Method).value.val.lower() == b"put"
+        assert http.get_header(b"Upgrade", res.headers).value.val == b"websocket"
 
 
 class TestResponse:
 
     def dummy_response(self):
-        return language.parse_pathod("400'msg'").next()
+        return next(language.parse_pathod("400'msg'"))
 
     def test_response(self):
-        r = language.parse_pathod("400:m'msg'").next()
-        assert r.status_code.string() == "400"
-        assert r.reason.string() == "msg"
+        r = next(language.parse_pathod("400:m'msg'"))
+        assert r.status_code.string() == b"400"
+        assert r.reason.string() == b"msg"
 
-        r = language.parse_pathod("400:m'msg':b@100b").next()
-        assert r.reason.string() == "msg"
+        r = next(language.parse_pathod("400:m'msg':b@100b"))
+        assert r.reason.string() == b"msg"
         assert r.body.values({})
         assert str(r)
 
-        r = language.parse_pathod("200").next()
-        assert r.status_code.string() == "200"
+        r = next(language.parse_pathod("200"))
+        assert r.status_code.string() == b"200"
         assert not r.reason
-        assert "OK" in [i[:] for i in r.preamble({})]
+        assert b"OK" in [i[:] for i in r.preamble({})]
 
     def test_render(self):
-        s = StringIO()
-        r = language.parse_pathod("400:m'msg'").next()
+        s = BytesIO()
+        r = next(language.parse_pathod("400:m'msg'"))
         assert language.serve(r, s, {})
 
-        r = language.parse_pathod("400:p0,100:dr").next()
+        r = next(language.parse_pathod("400:p0,100:dr"))
         assert "p0" in r.spec()
         s = r.preview_safe()
         assert "p0" not in s.spec()
 
     def test_raw(self):
-        s = StringIO()
-        r = language.parse_pathod("400:b'foo'").next()
+        s = BytesIO()
+        r = next(language.parse_pathod("400:b'foo'"))
         language.serve(r, s, {})
         v = s.getvalue()
-        assert "Content-Length" in v
+        assert b"Content-Length" in v
 
-        s = StringIO()
-        r = language.parse_pathod("400:b'foo':r").next()
+        s = BytesIO()
+        r = next(language.parse_pathod("400:b'foo':r"))
         language.serve(r, s, {})
         v = s.getvalue()
-        assert "Content-Length" not in v
+        assert b"Content-Length" not in v
 
     def test_length(self):
         def testlen(x):
-            s = StringIO()
-            x = x.next()
+            s = BytesIO()
+            x = next(x)
             language.serve(x, s, language.Settings())
             assert x.length(language.Settings()) == len(s.getvalue())
         testlen(language.parse_pathod("400:m'msg':r"))
@@ -196,8 +196,8 @@ class TestResponse:
 
     def test_maximum_length(self):
         def testlen(x):
-            x = x.next()
-            s = StringIO()
+            x = next(x)
+            s = BytesIO()
             m = x.maximum_length({})
             language.serve(x, s, {})
             assert m >= len(s.getvalue())
@@ -225,19 +225,19 @@ class TestResponse:
         tutils.raises("ascii", language.parse_pathod, "foo:b\xf0")
 
     def test_parse_header(self):
-        r = language.parse_pathod('400:h"foo"="bar"').next()
-        assert http.get_header("foo", r.headers)
+        r = next(language.parse_pathod('400:h"foo"="bar"'))
+        assert http.get_header(b"foo", r.headers)
 
     def test_parse_pause_before(self):
-        r = language.parse_pathod("400:p0,10").next()
+        r = next(language.parse_pathod("400:p0,10"))
         assert r.actions[0].spec() == "p0,10"
 
     def test_parse_pause_after(self):
-        r = language.parse_pathod("400:pa,10").next()
+        r = next(language.parse_pathod("400:pa,10"))
         assert r.actions[0].spec() == "pa,10"
 
     def test_parse_pause_random(self):
-        r = language.parse_pathod("400:pr,10").next()
+        r = next(language.parse_pathod("400:pr,10"))
         assert r.actions[0].spec() == "pr,10"
 
     def test_parse_stress(self):
@@ -245,29 +245,29 @@ class TestResponse:
         # returns an int and a python 2.7 int on windows has 32bit precision.
         # Therefore, we should keep the body length < 2147483647 bytes in our
         # tests.
-        r = language.parse_pathod("400:b@1g").next()
+        r = next(language.parse_pathod("400:b@1g"))
         assert r.length({})
 
     def test_spec(self):
         def rt(s):
-            s = language.parse_pathod(s).next().spec()
-            assert language.parse_pathod(s).next().spec() == s
+            s = next(language.parse_pathod(s)).spec()
+            assert next(language.parse_pathod(s)).spec() == s
         rt("400:b@100g")
         rt("400")
         rt("400:da")
 
     def test_websockets(self):
-        r = language.parse_pathod("ws").next()
+        r = next(language.parse_pathod("ws"))
         tutils.raises("no websocket key", r.resolve, language.Settings())
-        res = r.resolve(language.Settings(websocket_key="foo"))
-        assert res.status_code.string() == "101"
+        res = r.resolve(language.Settings(websocket_key=b"foo"))
+        assert res.status_code.string() == b"101"
 
 
 def test_ctype_shortcut():
     e = http.ShortcutContentType.expr()
     v = e.parseString("c'foo'")[0]
-    assert v.key.val == "Content-Type"
-    assert v.value.val == "foo"
+    assert v.key.val == b"Content-Type"
+    assert v.value.val == b"foo"
 
     s = v.spec()
     assert s == e.parseString(s)[0].spec()
@@ -282,8 +282,8 @@ def test_ctype_shortcut():
 def test_location_shortcut():
     e = http.ShortcutLocation.expr()
     v = e.parseString("l'foo'")[0]
-    assert v.key.val == "Location"
-    assert v.value.val == "foo"
+    assert v.key.val == b"Location"
+    assert v.value.val == b"foo"
 
     s = v.spec()
     assert s == e.parseString(s)[0].spec()
@@ -296,23 +296,23 @@ def test_location_shortcut():
 
 
 def test_shortcuts():
-    assert language.parse_pathod(
-        "400:c'foo'").next().headers[0].key.val == "Content-Type"
-    assert language.parse_pathod(
-        "400:l'foo'").next().headers[0].key.val == "Location"
+    assert next(language.parse_pathod(
+        "400:c'foo'")).headers[0].key.val == b"Content-Type"
+    assert next(language.parse_pathod(
+        "400:l'foo'")).headers[0].key.val == b"Location"
 
-    assert "Android" in tutils.render(parse_request("get:/:ua"))
-    assert "User-Agent" in tutils.render(parse_request("get:/:ua"))
+    assert b"Android" in tutils.render(parse_request("get:/:ua"))
+    assert b"User-Agent" in tutils.render(parse_request("get:/:ua"))
 
 
 def test_user_agent():
     e = http.ShortcutUserAgent.expr()
     v = e.parseString("ua")[0]
-    assert "Android" in v.string()
+    assert b"Android" in v.string()
 
     e = http.ShortcutUserAgent.expr()
     v = e.parseString("u'a'")[0]
-    assert "Android" not in v.string()
+    assert b"Android" not in v.string()
 
     v = e.parseString("u@100'")[0]
     assert len(str(v.freeze({}).value)) > 100
@@ -324,7 +324,7 @@ def test_user_agent():
 def test_nested_response():
     e = http.NestedResponse.expr()
     v = e.parseString("s'200'")[0]
-    assert v.value.val == "200"
+    assert v.value.val == b"200"
     tutils.raises(
         language.ParseException,
         e.parseString,
@@ -340,9 +340,7 @@ def test_nested_response():
 def test_nested_response_freeze():
     e = http.NestedResponse(
         base.TokValueLiteral(
-            "200:b'foo':i10,'\\x27'".encode(
-                "string_escape"
-            )
+            r"200:b\'foo\':i10,\'\\x27\'"
         )
     )
     assert e.freeze({})
diff --git a/test/pathod/tutils.py b/test/pathod/tutils.py
index 56cd2002..bf5e3165 100644
--- a/test/pathod/tutils.py
+++ b/test/pathod/tutils.py
@@ -3,6 +3,7 @@ import re
 import shutil
 import requests
 from six.moves import cStringIO as StringIO
+from six import BytesIO
 import urllib
 
 from netlib import tcp
@@ -147,6 +148,6 @@ test_data = utils.Data(__name__)
 
 def render(r, settings=language.Settings()):
     r = r.resolve(settings)
-    s = StringIO()
+    s = BytesIO()
     assert language.serve(r, s, settings)
     return s.getvalue()
diff --git a/tox.ini b/tox.ini
index f94dfb49..a1e6a816 100644
--- a/tox.ini
+++ b/tox.ini
@@ -8,7 +8,7 @@ deps = -rrequirements.txt
 commands = py.test -n 8 --timeout 60 ./test
 
 [testenv:py35]
-commands = py.test -n 8 --timeout 60 test/netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py
+commands = py.test -n 8 --timeout 60 test/netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py test/pathod/test_language_http.py
 
 [testenv:lint]
 deps = flake8