From f3932b27dafbf040c60556de7c5739148ffd67a6 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sat, 7 May 2016 23:44:39 +0530 Subject: Py3: Import and Other misc. stuff --- pathod/language/__init__.py | 5 +++-- pathod/language/base.py | 5 ++++- pathod/log.py | 4 +++- pathod/pathoc.py | 6 ++---- pathod/pathod.py | 6 +----- 5 files changed, 13 insertions(+), 13 deletions(-) diff --git a/pathod/language/__init__.py b/pathod/language/__init__.py index 32199e08..e27452dd 100644 --- a/pathod/language/__init__.py +++ b/pathod/language/__init__.py @@ -1,3 +1,5 @@ +from __future__ import absolute_import + import itertools import time @@ -5,8 +7,7 @@ import pyparsing as pp from . import http, http2, websockets, writer, exceptions -from exceptions import * -from base import Settings +from .base import Settings assert Settings # prevent pyflakes from messing with this diff --git a/pathod/language/base.py b/pathod/language/base.py index a4302998..c53e6b20 100644 --- a/pathod/language/base.py +++ b/pathod/language/base.py @@ -3,9 +3,12 @@ import os import abc import pyparsing as pp +from six.moves import reduce + from .. import utils from . import generators, exceptions + class Settings(object): def __init__( @@ -105,7 +108,7 @@ class Token(object): class _TokValueLiteral(Token): def __init__(self, val): - self.val = val.decode("string_escape") + self.val = val def get_generator(self, settings_): return self.val diff --git a/pathod/log.py b/pathod/log.py index f203542f..3f6aaea0 100644 --- a/pathod/log.py +++ b/pathod/log.py @@ -1,5 +1,7 @@ import datetime +import six + import netlib.utils import netlib.tcp import netlib.http @@ -53,7 +55,7 @@ class LogCtx(object): ] ) if exc_value: - raise exc_type, exc_value, traceback + six.reraise(exc_type, exc_value, traceback) def suppress(self): self.suppressed = True diff --git a/pathod/pathoc.py b/pathod/pathoc.py index a49ed351..8706868b 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -13,14 +13,12 @@ import threading import OpenSSL.crypto import six -from netlib import tcp, http, certutils, websockets, socks +from netlib import tcp, certutils, websockets, socks from netlib.exceptions import HttpException, TcpDisconnect, TcpTimeout, TlsException, TcpException, \ NetlibException from netlib.http import http1, http2 -import language.http -import language.websockets -from . import utils, log +from . import utils, log, language import logging from netlib.tutils import treq diff --git a/pathod/pathod.py b/pathod/pathod.py index 017ce072..af5f9e6a 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -6,15 +6,11 @@ import sys import threading import urllib -from netlib import tcp, http, certutils, websockets +from netlib import tcp, certutils, websockets from netlib.exceptions import HttpException, HttpReadDisconnect, TcpTimeout, TcpDisconnect, \ TlsException from . import version, app, language, utils, log, protocols -import language.http -import language.actions -import language.exceptions -import language.websockets DEFAULT_CERT_DOMAIN = "pathod.net" -- cgit v1.2.3 From 88e42bab6d95eb0d1d1224c4f69caafdc03a69aa Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sat, 7 May 2016 23:45:03 +0530 Subject: Py3: inner_repr and escape_unprintables --- pathod/utils.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) diff --git a/pathod/utils.py b/pathod/utils.py index d1e2dd00..2f9de83d 100644 --- a/pathod/utils.py +++ b/pathod/utils.py @@ -2,6 +2,8 @@ import os import sys import netlib.utils +import six + SIZE_UNITS = dict( b=1024 ** 0, @@ -58,7 +60,7 @@ def inner_repr(s): Returns the inner portion of a string or unicode repr (i.e. without the quotes) """ - if isinstance(s, unicode): + if six.PY2 and isinstance(s, unicode): return repr(s)[2:-1] else: return repr(s)[1:-1] @@ -70,7 +72,10 @@ def escape_unprintables(s): """ s = s.replace("\r\n", "PATHOD_MARKER_RN") s = s.replace("\n", "PATHOD_MARKER_N") - s = inner_repr(s) + if six.PY2: + s = inner_repr(s) + else: + s = s.encode('unicode_escape').decode('ascii') s = s.replace("PATHOD_MARKER_RN", "\n") s = s.replace("PATHOD_MARKER_N", "\n") return s -- cgit v1.2.3 From e5cebb81fb46f21ba77cbc2a1b3a58aad5e77a66 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sun, 8 May 2016 02:55:57 +0530 Subject: Removed wrong import --- pathod/language/__init__.py | 1 + 1 file changed, 1 insertion(+) diff --git a/pathod/language/__init__.py b/pathod/language/__init__.py index e27452dd..10da93ba 100644 --- a/pathod/language/__init__.py +++ b/pathod/language/__init__.py @@ -7,6 +7,7 @@ import pyparsing as pp from . import http, http2, websockets, writer, exceptions +from .exceptions import * from .base import Settings assert Settings # prevent pyflakes from messing with this -- cgit v1.2.3 From 22e4bc1938b74b0d0bfda7cab41b54070bcd7fb9 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sun, 8 May 2016 16:20:27 +0530 Subject: Py3: Handle bytes case in inner_repr --- pathod/utils.py | 3 ++- test/pathod/test_utils.py | 1 + 2 files changed, 3 insertions(+), 1 deletion(-) diff --git a/pathod/utils.py b/pathod/utils.py index 2f9de83d..b14535ca 100644 --- a/pathod/utils.py +++ b/pathod/utils.py @@ -60,7 +60,8 @@ def inner_repr(s): Returns the inner portion of a string or unicode repr (i.e. without the quotes) """ - if six.PY2 and isinstance(s, unicode): + if (six.PY2 and isinstance(s, unicode)) or \ + (six.PY3 and isinstance(s, bytes)): return repr(s)[2:-1] else: return repr(s)[1:-1] diff --git a/test/pathod/test_utils.py b/test/pathod/test_utils.py index 4dcedf6e..4e891ad9 100644 --- a/test/pathod/test_utils.py +++ b/test/pathod/test_utils.py @@ -30,6 +30,7 @@ def test_data_path(): def test_inner_repr(): assert utils.inner_repr("\x66") == "\x66" assert utils.inner_repr(u"foo") == "foo" + assert utils.inner_repr(b"foo") == "foo" def test_escape_unprintables(): -- cgit v1.2.3 From 5c62fabc84d0e2e7cbb5fcde494bbfa11dd593b2 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Wed, 11 May 2016 02:07:57 +0530 Subject: Use BytesIO in pathod app --- pathod/app.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/pathod/app.py b/pathod/app.py index aa00ed69..7e9860b9 100644 --- a/pathod/app.py +++ b/pathod/app.py @@ -1,6 +1,6 @@ import logging import pprint -from six.moves import cStringIO as StringIO +import io import copy from flask import Flask, jsonify, render_template, request, abort, make_response from . import version, language, utils @@ -145,7 +145,7 @@ def make_app(noapi, debug): args["marked"] = v.marked() return render(template, False, **args) - s = StringIO() + s = io.BytesIO() settings = copy.copy(app.config["pathod"].settings) settings.request_host = EXAMPLE_HOST -- cgit v1.2.3 From daaa672d3974dfda9bfdad9c89b27e5e6237992c Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Wed, 11 May 2016 02:09:42 +0530 Subject: Remove Py3 specific check --- pathod/utils.py | 3 +-- 1 file changed, 1 insertion(+), 2 deletions(-) diff --git a/pathod/utils.py b/pathod/utils.py index b14535ca..0cc77ab2 100644 --- a/pathod/utils.py +++ b/pathod/utils.py @@ -60,8 +60,7 @@ def inner_repr(s): Returns the inner portion of a string or unicode repr (i.e. without the quotes) """ - if (six.PY2 and isinstance(s, unicode)) or \ - (six.PY3 and isinstance(s, bytes)): + if (six.PY2 and isinstance(s, unicode)) or isinstance(s, bytes): return repr(s)[2:-1] else: return repr(s)[1:-1] -- cgit v1.2.3 From 1699592f092b5f616ef1aa33b611ec6dabe7b255 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Sun, 15 May 2016 23:27:07 +0530 Subject: Use escaped_str functions for TokValueLiteral --- pathod/language/base.py | 13 +++++++------ 1 file changed, 7 insertions(+), 6 deletions(-) diff --git a/pathod/language/base.py b/pathod/language/base.py index c53e6b20..bc6da6ab 100644 --- a/pathod/language/base.py +++ b/pathod/language/base.py @@ -4,6 +4,7 @@ import abc import pyparsing as pp from six.moves import reduce +from netlib.utils import escaped_str_to_bytes, bytes_to_escaped_str from .. import utils from . import generators, exceptions @@ -108,7 +109,7 @@ class Token(object): class _TokValueLiteral(Token): def __init__(self, val): - self.val = val + self.val = escaped_str_to_bytes(str(val)) def get_generator(self, settings_): return self.val @@ -133,7 +134,7 @@ class TokValueLiteral(_TokValueLiteral): return v def spec(self): - inner = self.val.encode("string_escape") + inner = bytes_to_escaped_str(self.val) inner = inner.replace(r"\'", r"\x27") return "'" + inner + "'" @@ -146,7 +147,7 @@ class TokValueNakedLiteral(_TokValueLiteral): return e.setParseAction(lambda x: cls(*x)) def spec(self): - return self.val.encode("string_escape") + return bytes_to_escaped_str(self.val) class TokValueGenerate(Token): @@ -164,7 +165,7 @@ class TokValueGenerate(Token): def freeze(self, settings): g = self.get_generator(settings) - return TokValueLiteral(g[:].encode("string_escape")) + return TokValueLiteral(bytes_to_escaped_str(g[:])) @classmethod def expr(cls): @@ -224,7 +225,7 @@ class TokValueFile(Token): return generators.FileGenerator(s) def spec(self): - return "<'%s'" % self.path.encode("string_escape") + return "<'%s'" % bytes_to_escaped_str(self.path) TokValue = pp.MatchFirst( @@ -576,4 +577,4 @@ class NestedMessage(Token): def freeze(self, settings): f = self.parsed.freeze(settings).spec() - return self.__class__(TokValueLiteral(f.encode("string_escape"))) + return self.__class__(TokValueLiteral(bytes_to_escaped_str(f))) -- cgit v1.2.3 From 1bbb178b6a0ebd534b6b9f78299118496de6b92a Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Wed, 25 May 2016 16:42:46 +0530 Subject: Remove inner_repr, fixup escape_unprintables --- pathod/utils.py | 22 ++++------------------ test/pathod/test_utils.py | 12 ++++-------- 2 files changed, 8 insertions(+), 26 deletions(-) diff --git a/pathod/utils.py b/pathod/utils.py index 0cc77ab2..8c6d6290 100644 --- a/pathod/utils.py +++ b/pathod/utils.py @@ -2,7 +2,7 @@ import os import sys import netlib.utils -import six +from netlib.utils import bytes_to_escaped_str SIZE_UNITS = dict( @@ -55,27 +55,13 @@ def xrepr(s): return repr(s)[1:-1] -def inner_repr(s): - """ - Returns the inner portion of a string or unicode repr (i.e. without the - quotes) - """ - if (six.PY2 and isinstance(s, unicode)) or isinstance(s, bytes): - return repr(s)[2:-1] - else: - return repr(s)[1:-1] - - def escape_unprintables(s): """ Like inner_repr, but preserves line breaks. """ - s = s.replace("\r\n", "PATHOD_MARKER_RN") - s = s.replace("\n", "PATHOD_MARKER_N") - if six.PY2: - s = inner_repr(s) - else: - s = s.encode('unicode_escape').decode('ascii') + s = s.replace(b"\r\n", b"PATHOD_MARKER_RN") + s = s.replace(b"\n", b"PATHOD_MARKER_N") + s = bytes_to_escaped_str(s) s = s.replace("PATHOD_MARKER_RN", "\n") s = s.replace("PATHOD_MARKER_N", "\n") return s diff --git a/test/pathod/test_utils.py b/test/pathod/test_utils.py index 4e891ad9..32ba3bdf 100644 --- a/test/pathod/test_utils.py +++ b/test/pathod/test_utils.py @@ -1,6 +1,8 @@ from pathod import utils import tutils +import six + def test_membool(): m = utils.MemBool() @@ -27,14 +29,8 @@ def test_data_path(): tutils.raises(ValueError, utils.data.path, "nonexistent") -def test_inner_repr(): - assert utils.inner_repr("\x66") == "\x66" - assert utils.inner_repr(u"foo") == "foo" - assert utils.inner_repr(b"foo") == "foo" - - def test_escape_unprintables(): s = "".join([chr(i) for i in range(255)]) - e = utils.escape_unprintables(s) + e = utils.escape_unprintables(six.b(s)) assert e.encode('ascii') - assert not "PATHOD_MARKER" in e + assert "PATHOD_MARKER" not in e -- cgit v1.2.3 From 4ec56808ddfc6e6c03ce916ca35b0d7439165650 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Wed, 25 May 2016 17:18:02 +0530 Subject: remove str() --- pathod/language/base.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/pathod/language/base.py b/pathod/language/base.py index bc6da6ab..54ca6492 100644 --- a/pathod/language/base.py +++ b/pathod/language/base.py @@ -109,7 +109,7 @@ class Token(object): class _TokValueLiteral(Token): def __init__(self, val): - self.val = escaped_str_to_bytes(str(val)) + self.val = escaped_str_to_bytes(val) def get_generator(self, settings_): return self.val -- cgit v1.2.3 From bc6cd13356a5181baafc9b01385a21991695ade3 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Thu, 26 May 2016 22:19:58 +0530 Subject: Go Python 3 by default with the bytes conversion --- test/pathod/test_utils.py | 6 ++++-- 1 file changed, 4 insertions(+), 2 deletions(-) diff --git a/test/pathod/test_utils.py b/test/pathod/test_utils.py index 32ba3bdf..8026a576 100644 --- a/test/pathod/test_utils.py +++ b/test/pathod/test_utils.py @@ -30,7 +30,9 @@ def test_data_path(): def test_escape_unprintables(): - s = "".join([chr(i) for i in range(255)]) - e = utils.escape_unprintables(six.b(s)) + s = bytes(range(256)) + if six.PY2: + s = "".join([chr(i) for i in range(255)]) + e = utils.escape_unprintables(s) assert e.encode('ascii') assert "PATHOD_MARKER" not in e -- cgit v1.2.3 From 92317bc81d72f243095a1bfa192f568637d9bc32 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Thu, 26 May 2016 23:39:16 +0530 Subject: Enable travis Py3 testing for test_utils --- .travis.yml | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) diff --git a/.travis.yml b/.travis.yml index 7d3fbee8..4a01174a 100644 --- a/.travis.yml +++ b/.travis.yml @@ -22,9 +22,9 @@ matrix: git: depth: 9999999 - python: 3.5 - env: SCOPE="netlib ./test/mitmproxy/script" + env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py" - python: 3.5 - env: SCOPE="netlib ./test/mitmproxy/script" NO_ALPN=1 + env: SCOPE="netlib ./test/mitmproxy/script ./test/pathod/test_utils.py" NO_ALPN=1 - python: 2.7 env: DOCS=1 script: 'cd docs && make html' -- cgit v1.2.3