diff options
| -rw-r--r-- | .travis.yml | 4 | ||||
| -rw-r--r-- | pathod/language/base.py | 16 | ||||
| -rw-r--r-- | pathod/language/http.py | 30 | ||||
| -rw-r--r-- | pathod/language/message.py | 7 | ||||
| -rw-r--r-- | setup.py | 2 | ||||
| -rw-r--r-- | test/pathod/test_language_base.py | 13 | ||||
| -rw-r--r-- | test/pathod/test_language_http.py | 146 | ||||
| -rw-r--r-- | test/pathod/tutils.py | 3 | ||||
| -rw-r--r-- | tox.ini | 2 |
9 files changed, 115 insertions, 108 deletions
diff --git a/.travis.yml b/.travis.yml index b77091b1..f6731c0b 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,9 @@ matrix: git: depth: 9999999 - python: 3.5 - env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_base.py" + env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_base.py test/pathod/test_language_http.py" - python: 3.5 - env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_base.py" NO_ALPN=1 + env: SCOPE="netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_base.py test/pathod/test_language_http.py" NO_ALPN=1 - python: 2.7 env: DOCS=1 script: 'cd docs && make html' diff --git a/pathod/language/base.py b/pathod/language/base.py index 1369a3c7..25f3fd1a 100644 --- a/pathod/language/base.py +++ b/pathod/language/base.py @@ -261,7 +261,7 @@ class _Component(Token): """ A value component of the primary specification of an message. - Components produce byte values desribe the bytes of the message. + Components produce byte values describing the bytes of the message. """ def values(self, settings): # pragma: no cover @@ -272,9 +272,9 @@ class _Component(Token): def string(self, settings=None): """ - A string representation of the object. + A bytestring representation of the object. """ - return "".join(i[:] for i in self.values(settings or {})) + return b"".join(i[:] for i in self.values(settings or {})) class KeyValue(_Component): @@ -391,7 +391,7 @@ class Integer(_Component): "Integer value must be between %s and %s." % self.bounds, 0, 0 ) - self.value = str(value) + self.value = str(value).encode() @classmethod def expr(cls): @@ -401,10 +401,10 @@ class Integer(_Component): return e.setParseAction(lambda x: cls(*x)) def values(self, settings): - return self.value + return [self.value] def spec(self): - return "%s%s" % (self.preamble, self.value) + return "%s%s" % (self.preamble, self.value.decode()) def freeze(self, settings_): return self @@ -555,7 +555,7 @@ class NestedMessage(Token): try: self.parsed = self.nest_type( self.nest_type.expr().parseString( - value.val, + value.val.decode(), parseAll=True ) ) @@ -578,4 +578,4 @@ class NestedMessage(Token): def freeze(self, settings): f = self.parsed.freeze(settings).spec() - return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f))) + return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f.encode()))) diff --git a/pathod/language/http.py b/pathod/language/http.py index b2308d5e..4cc7db5f 100644 --- a/pathod/language/http.py +++ b/pathod/language/http.py @@ -57,7 +57,7 @@ class _HeaderMixin(object): unique_name = None def format_header(self, key, value): - return [key, ": ", value, "\r\n"] + return [key, b": ", value, b"\r\n"] def values(self, settings): return self.format_header( @@ -88,7 +88,7 @@ class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue): def values(self, settings): value = self.value.val if self.option_used: - value = user_agents.get_by_shortcut(value.lower())[2] + value = user_agents.get_by_shortcut(value.lower().decode())[2].encode() return self.format_header( self.key.get_generator(settings), @@ -109,7 +109,7 @@ def get_header(val, headers): class _HTTPMessage(message.Message): - version = "HTTP/1.1" + version = b"HTTP/1.1" @property def actions(self): @@ -133,10 +133,10 @@ class _HTTPMessage(message.Message): def values(self, settings): vals = self.preamble(settings) - vals.append("\r\n") + vals.append(b"\r\n") for h in self.headers: vals.extend(h.values(settings)) - vals.append("\r\n") + vals.append(b"\r\n") if self.body: vals.extend(self.body.values(settings)) return vals @@ -171,18 +171,18 @@ class Response(_HTTPMessage): return self.tok(Reason) def preamble(self, settings): - l = [self.version, " "] + l = [self.version, b" "] l.extend(self.status_code.values(settings)) status_code = int(self.status_code.value) - l.append(" ") + l.append(b" ") if self.reason: l.extend(self.reason.values(settings)) else: l.append( status_codes.RESPONSES.get( status_code, - "Unknown code" - ) + b"Unknown code" + ).encode() ) return l @@ -205,8 +205,8 @@ class Response(_HTTPMessage): if not get_header(i[0], self.headers): tokens.append( Header( - base.TokValueLiteral(i[0]), - base.TokValueLiteral(i[1])) + base.TokValueLiteral(i[0].decode()), + base.TokValueLiteral(i[1].decode())) ) if not self.raw: if not get_header("Content-Length", self.headers): @@ -294,11 +294,11 @@ class Request(_HTTPMessage): def preamble(self, settings): v = self.method.values(settings) - v.append(" ") + v.append(b" ") v.extend(self.path.values(settings)) if self.nested_response: v.append(self.nested_response.parsed.spec()) - v.append(" ") + v.append(b" ") v.append(self.version) return v @@ -314,8 +314,8 @@ class Request(_HTTPMessage): if not get_header(i[0], self.headers): tokens.append( Header( - base.TokValueLiteral(i[0]), - base.TokValueLiteral(i[1]) + base.TokValueLiteral(i[0].decode()), + base.TokValueLiteral(i[1].decode()) ) ) if not self.raw: diff --git a/pathod/language/message.py b/pathod/language/message.py index 33124856..fea4f4de 100644 --- a/pathod/language/message.py +++ b/pathod/language/message.py @@ -1,5 +1,6 @@ import abc from . import actions, exceptions +from netlib import strutils LOG_TRUNCATE = 1024 @@ -49,7 +50,7 @@ class Message(object): def preview_safe(self): """ - Return a copy of this message that issafe for previews. + Return a copy of this message that is safe for previews. """ tokens = [i for i in self.tokens if not isinstance(i, actions.PauseAt)] return self.__class__(tokens) @@ -80,10 +81,10 @@ class Message(object): # We truncate at 1k. if hasattr(v, "values"): v = [x[:LOG_TRUNCATE] for x in v.values(settings)] - v = "".join(v).encode("string_escape") + v = strutils.bytes_to_escaped_str(b"".join(v)) elif hasattr(v, "__len__"): v = v[:LOG_TRUNCATE] - v = v.encode("string_escape") + v = strutils.bytes_to_escaped_str(v) ret[i] = v ret["spec"] = self.spec() return ret @@ -75,7 +75,7 @@ setup( "passlib>=1.6.5, <1.7", "pyasn1>=0.1.9, <0.2", "pyOpenSSL>=16.0, <17.0", - "pyparsing>=2.0, <2.1", # 2.1.1 breaks our binaries, see https://sourceforge.net/p/pyparsing/bugs/93/ + "pyparsing>=2.1.3, <2.2", "pyperclip>=1.5.22, <1.6", "requests>=2.9.1, <2.10", "six>=1.10, <1.11", diff --git a/test/pathod/test_language_base.py b/test/pathod/test_language_base.py index 075dc2b8..7c7d8cf9 100644 --- a/test/pathod/test_language_base.py +++ b/test/pathod/test_language_base.py @@ -55,8 +55,15 @@ class TestTokValueLiteral: v = base.TokValueLiteral("f\x00oo") assert v.spec() == repr(v) == r"'f\x00oo'" - v = base.TokValueLiteral("\"") - assert v.spec() == repr(v) == '\'"\'' + v = base.TokValueLiteral('"') + assert v.spec() == repr(v) == """ '"' """.strip() + + # While pyparsing has a escChar argument for QuotedString, + # escChar only performs scapes single-character escapes and does not work for e.g. r"\x02". + # Thus, we cannot use that option, which means we cannot have single quotes in strings. + # To fix this, we represent single quotes as r"\x07". + v = base.TokValueLiteral("'") + assert v.spec() == r"'\x27'" def roundtrip(self, spec): e = base.TokValueLiteral.expr() @@ -311,7 +318,7 @@ def test_options_or_value(): def test_integer(): e = base.Integer.expr() v = e.parseString("200")[0] - assert v.string() == "200" + assert v.string() == b"200" assert v.spec() == "200" assert v.freeze({}).value == v.value diff --git a/test/pathod/test_language_http.py b/test/pathod/test_language_http.py index d1870a63..18059e3a 100644 --- a/test/pathod/test_language_http.py +++ b/test/pathod/test_language_http.py @@ -1,15 +1,15 @@ -from six.moves import cStringIO as StringIO +from six import BytesIO from pathod import language from pathod.language import http, base import tutils def parse_request(s): - return language.parse_pathoc(s).next() + return next(language.parse_pathoc(s)) def test_make_error_response(): - d = StringIO() + d = BytesIO() s = http.make_error_response("foo") language.serve(s, d, {}) @@ -24,17 +24,17 @@ class TestRequest: def test_simple(self): r = parse_request('GET:"/foo"') - assert r.method.string() == "GET" - assert r.path.string() == "/foo" + assert r.method.string() == b"GET" + assert r.path.string() == b"/foo" r = parse_request('GET:/foo') - assert r.path.string() == "/foo" + assert r.path.string() == b"/foo" r = parse_request('GET:@1k') assert len(r.path.string()) == 1024 def test_multiple(self): r = list(language.parse_pathoc("GET:/ PUT:/")) - assert r[0].method.string() == "GET" - assert r[1].method.string() == "PUT" + assert r[0].method.string() == b"GET" + assert r[1].method.string() == b"PUT" assert len(r) == 2 l = """ @@ -54,8 +54,8 @@ class TestRequest: """ r = list(language.parse_pathoc(l)) assert len(r) == 2 - assert r[0].method.string() == "GET" - assert r[1].method.string() == "PUT" + assert r[0].method.string() == b"GET" + assert r[1].method.string() == b"PUT" l = """ get:"http://localhost:9999/p/200":ir,@1 @@ -63,8 +63,8 @@ class TestRequest: """ r = list(language.parse_pathoc(l)) assert len(r) == 2 - assert r[0].method.string() == "GET" - assert r[1].method.string() == "GET" + assert r[0].method.string() == b"GET" + assert r[1].method.string() == b"GET" def test_nested_response(self): l = "get:/p:s'200'" @@ -75,7 +75,7 @@ class TestRequest: assert r[0].values({}) def test_render(self): - s = StringIO() + s = BytesIO() r = parse_request("GET:'/foo'") assert language.serve( r, @@ -90,8 +90,8 @@ class TestRequest: ir,@1 """ r = parse_request(l) - assert r.method.string() == "GET" - assert r.path.string() == "/foo" + assert r.method.string() == b"GET" + assert r.path.string() == b"/foo" assert r.actions l = """ @@ -106,8 +106,8 @@ class TestRequest: ir,@1 """ r = parse_request(l) - assert r.method.string() == "GET" - assert r.path.string().endswith("bar") + assert r.method.string() == b"GET" + assert r.path.string().endswith(b"bar") assert r.actions def test_spec(self): @@ -128,66 +128,66 @@ class TestRequest: def test_websocket(self): r = parse_request('ws:/path/') res = r.resolve(language.Settings()) - assert res.method.string().lower() == "get" - assert res.tok(http.Path).value.val == "/path/" - assert res.tok(http.Method).value.val.lower() == "get" - assert http.get_header("Upgrade", res.headers).value.val == "websocket" + assert res.method.string().lower() == b"get" + assert res.tok(http.Path).value.val == b"/path/" + assert res.tok(http.Method).value.val.lower() == b"get" + assert http.get_header(b"Upgrade", res.headers).value.val == b"websocket" r = parse_request('ws:put:/path/') res = r.resolve(language.Settings()) - assert r.method.string().lower() == "put" - assert res.tok(http.Path).value.val == "/path/" - assert res.tok(http.Method).value.val.lower() == "put" - assert http.get_header("Upgrade", res.headers).value.val == "websocket" + assert r.method.string().lower() == b"put" + assert res.tok(http.Path).value.val == b"/path/" + assert res.tok(http.Method).value.val.lower() == b"put" + assert http.get_header(b"Upgrade", res.headers).value.val == b"websocket" class TestResponse: def dummy_response(self): - return language.parse_pathod("400'msg'").next() + return next(language.parse_pathod("400'msg'")) def test_response(self): - r = language.parse_pathod("400:m'msg'").next() - assert r.status_code.string() == "400" - assert r.reason.string() == "msg" + r = next(language.parse_pathod("400:m'msg'")) + assert r.status_code.string() == b"400" + assert r.reason.string() == b"msg" - r = language.parse_pathod("400:m'msg':b@100b").next() - assert r.reason.string() == "msg" + r = next(language.parse_pathod("400:m'msg':b@100b")) + assert r.reason.string() == b"msg" assert r.body.values({}) assert str(r) - r = language.parse_pathod("200").next() - assert r.status_code.string() == "200" + r = next(language.parse_pathod("200")) + assert r.status_code.string() == b"200" assert not r.reason - assert "OK" in [i[:] for i in r.preamble({})] + assert b"OK" in [i[:] for i in r.preamble({})] def test_render(self): - s = StringIO() - r = language.parse_pathod("400:m'msg'").next() + s = BytesIO() + r = next(language.parse_pathod("400:m'msg'")) assert language.serve(r, s, {}) - r = language.parse_pathod("400:p0,100:dr").next() + r = next(language.parse_pathod("400:p0,100:dr")) assert "p0" in r.spec() s = r.preview_safe() assert "p0" not in s.spec() def test_raw(self): - s = StringIO() - r = language.parse_pathod("400:b'foo'").next() + s = BytesIO() + r = next(language.parse_pathod("400:b'foo'")) language.serve(r, s, {}) v = s.getvalue() - assert "Content-Length" in v + assert b"Content-Length" in v - s = StringIO() - r = language.parse_pathod("400:b'foo':r").next() + s = BytesIO() + r = next(language.parse_pathod("400:b'foo':r")) language.serve(r, s, {}) v = s.getvalue() - assert "Content-Length" not in v + assert b"Content-Length" not in v def test_length(self): def testlen(x): - s = StringIO() - x = x.next() + s = BytesIO() + x = next(x) language.serve(x, s, language.Settings()) assert x.length(language.Settings()) == len(s.getvalue()) testlen(language.parse_pathod("400:m'msg':r")) @@ -196,8 +196,8 @@ class TestResponse: def test_maximum_length(self): def testlen(x): - x = x.next() - s = StringIO() + x = next(x) + s = BytesIO() m = x.maximum_length({}) language.serve(x, s, {}) assert m >= len(s.getvalue()) @@ -225,19 +225,19 @@ class TestResponse: tutils.raises("ascii", language.parse_pathod, "foo:b\xf0") def test_parse_header(self): - r = language.parse_pathod('400:h"foo"="bar"').next() - assert http.get_header("foo", r.headers) + r = next(language.parse_pathod('400:h"foo"="bar"')) + assert http.get_header(b"foo", r.headers) def test_parse_pause_before(self): - r = language.parse_pathod("400:p0,10").next() + r = next(language.parse_pathod("400:p0,10")) assert r.actions[0].spec() == "p0,10" def test_parse_pause_after(self): - r = language.parse_pathod("400:pa,10").next() + r = next(language.parse_pathod("400:pa,10")) assert r.actions[0].spec() == "pa,10" def test_parse_pause_random(self): - r = language.parse_pathod("400:pr,10").next() + r = next(language.parse_pathod("400:pr,10")) assert r.actions[0].spec() == "pr,10" def test_parse_stress(self): @@ -245,29 +245,29 @@ class TestResponse: # returns an int and a python 2.7 int on windows has 32bit precision. # Therefore, we should keep the body length < 2147483647 bytes in our # tests. - r = language.parse_pathod("400:b@1g").next() + r = next(language.parse_pathod("400:b@1g")) assert r.length({}) def test_spec(self): def rt(s): - s = language.parse_pathod(s).next().spec() - assert language.parse_pathod(s).next().spec() == s + s = next(language.parse_pathod(s)).spec() + assert next(language.parse_pathod(s)).spec() == s rt("400:b@100g") rt("400") rt("400:da") def test_websockets(self): - r = language.parse_pathod("ws").next() + r = next(language.parse_pathod("ws")) tutils.raises("no websocket key", r.resolve, language.Settings()) - res = r.resolve(language.Settings(websocket_key="foo")) - assert res.status_code.string() == "101" + res = r.resolve(language.Settings(websocket_key=b"foo")) + assert res.status_code.string() == b"101" def test_ctype_shortcut(): e = http.ShortcutContentType.expr() v = e.parseString("c'foo'")[0] - assert v.key.val == "Content-Type" - assert v.value.val == "foo" + assert v.key.val == b"Content-Type" + assert v.value.val == b"foo" s = v.spec() assert s == e.parseString(s)[0].spec() @@ -282,8 +282,8 @@ def test_ctype_shortcut(): def test_location_shortcut(): e = http.ShortcutLocation.expr() v = e.parseString("l'foo'")[0] - assert v.key.val == "Location" - assert v.value.val == "foo" + assert v.key.val == b"Location" + assert v.value.val == b"foo" s = v.spec() assert s == e.parseString(s)[0].spec() @@ -296,23 +296,23 @@ def test_location_shortcut(): def test_shortcuts(): - assert language.parse_pathod( - "400:c'foo'").next().headers[0].key.val == "Content-Type" - assert language.parse_pathod( - "400:l'foo'").next().headers[0].key.val == "Location" + assert next(language.parse_pathod( + "400:c'foo'")).headers[0].key.val == b"Content-Type" + assert next(language.parse_pathod( + "400:l'foo'")).headers[0].key.val == b"Location" - assert "Android" in tutils.render(parse_request("get:/:ua")) - assert "User-Agent" in tutils.render(parse_request("get:/:ua")) + assert b"Android" in tutils.render(parse_request("get:/:ua")) + assert b"User-Agent" in tutils.render(parse_request("get:/:ua")) def test_user_agent(): e = http.ShortcutUserAgent.expr() v = e.parseString("ua")[0] - assert "Android" in v.string() + assert b"Android" in v.string() e = http.ShortcutUserAgent.expr() v = e.parseString("u'a'")[0] - assert "Android" not in v.string() + assert b"Android" not in v.string() v = e.parseString("u@100'")[0] assert len(str(v.freeze({}).value)) > 100 @@ -324,7 +324,7 @@ def test_user_agent(): def test_nested_response(): e = http.NestedResponse.expr() v = e.parseString("s'200'")[0] - assert v.value.val == "200" + assert v.value.val == b"200" tutils.raises( language.ParseException, e.parseString, @@ -340,9 +340,7 @@ def test_nested_response(): def test_nested_response_freeze(): e = http.NestedResponse( base.TokValueLiteral( - "200:b'foo':i10,'\\x27'".encode( - "string_escape" - ) + r"200:b\'foo\':i10,\'\\x27\'" ) ) assert e.freeze({}) diff --git a/test/pathod/tutils.py b/test/pathod/tutils.py index 56cd2002..bf5e3165 100644 --- a/test/pathod/tutils.py +++ b/test/pathod/tutils.py @@ -3,6 +3,7 @@ import re import shutil import requests from six.moves import cStringIO as StringIO +from six import BytesIO import urllib from netlib import tcp @@ -147,6 +148,6 @@ test_data = utils.Data(__name__) def render(r, settings=language.Settings()): r = r.resolve(settings) - s = StringIO() + s = BytesIO() assert language.serve(r, s, settings) return s.getvalue() @@ -8,7 +8,7 @@ deps = -rrequirements.txt commands = py.test -n 8 --timeout 60 ./test [testenv:py35] -commands = py.test -n 8 --timeout 60 test/netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py +commands = py.test -n 8 --timeout 60 test/netlib test/mitmproxy/script test/pathod/test_utils.py test/pathod/test_log.py test/pathod/test_language_generators.py test/pathod/test_language_writer.py test/pathod/test_language_base.py test/pathod/test_language_http.py [testenv:lint] deps = flake8 |