From 29bcdc82509662d1c016dfd42122331678586d5b Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 2 Jun 2016 16:08:17 +1200 Subject: Fix lock over pathod locks There were basically a nop before... o_O --- pathod/pathod.py | 18 ++++++++++-------- 1 file changed, 10 insertions(+), 8 deletions(-) (limited to 'pathod') diff --git a/pathod/pathod.py b/pathod/pathod.py index 7795df0e..0449c0c1 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -353,6 +353,8 @@ class Pathod(tcp.TCPServer): staticdir=self.staticdir ) + self.loglock = threading.Lock() + def check_policy(self, req, settings): """ A policy check that verifies the request size is within limits. @@ -403,8 +405,7 @@ class Pathod(tcp.TCPServer): def add_log(self, d): if not self.noapi: - lock = threading.Lock() - with lock: + with self.loglock: d["id"] = self.logid self.log.insert(0, d) if len(self.log) > self.LOGBUF: @@ -413,17 +414,18 @@ class Pathod(tcp.TCPServer): return d["id"] def clear_log(self): - lock = threading.Lock() - with lock: + with self.loglock: self.log = [] def log_by_id(self, identifier): - for i in self.log: - if i["id"] == identifier: - return i + with self.loglock: + for i in self.log: + if i["id"] == identifier: + return i def get_log(self): - return self.log + with self.loglock: + return self.log def main(args): # pragma: no cover -- cgit v1.2.3 From 92decf96dda3704c3a303399e0e7a0920e130b95 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 2 Jun 2016 17:46:37 +1200 Subject: Add expect_log to the pathod test truss, and use it for last_log --- pathod/test.py | 16 +++++++++++++++- 1 file changed, 15 insertions(+), 1 deletion(-) (limited to 'pathod') diff --git a/pathod/test.py b/pathod/test.py index 23b7a5b6..7e597a72 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -1,5 +1,7 @@ from six.moves import cStringIO as StringIO import threading +import time + from six.moves import queue import requests @@ -49,11 +51,23 @@ class Daemon: def text_log(self): return self.logfp.getvalue() + def expect_log(self, n, timeout=1): + l = [] + start = time.time() + while True: + l = self.log() + print l + if time.time() - start >= timeout: + return None + if len(l) >= n: + break + return l + def last_log(self): """ Returns the last logged request, or None. """ - l = self.log() + l = self.expect_log(1) if not l: return None return l[0] -- cgit v1.2.3 From 5e123844865715085cb3ab06f264c0c3032458c4 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 2 Jun 2016 17:54:34 +1200 Subject: Remove debug print --- pathod/test.py | 1 - 1 file changed, 1 deletion(-) (limited to 'pathod') diff --git a/pathod/test.py b/pathod/test.py index 7e597a72..c6e2caae 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -56,7 +56,6 @@ class Daemon: start = time.time() while True: l = self.log() - print l if time.time() - start >= timeout: return None if len(l) >= n: -- cgit v1.2.3 From 40156ce123962a6d0e431761833a506ec5aeebb9 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 2 Jun 2016 17:55:11 +1200 Subject: Bump error timeout --- pathod/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pathod') diff --git a/pathod/test.py b/pathod/test.py index c6e2caae..797b4970 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -51,7 +51,7 @@ class Daemon: def text_log(self): return self.logfp.getvalue() - def expect_log(self, n, timeout=1): + def expect_log(self, n, timeout=5): l = [] start = time.time() while True: -- cgit v1.2.3 From 254614e9f77e108d186ff3f7e89ec78012af65a1 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 2 Jun 2016 18:10:36 +1200 Subject: Since we have locks over the logs, use direct access rather than API requests to get to them --- pathod/test.py | 8 +++----- 1 file changed, 3 insertions(+), 5 deletions(-) (limited to 'pathod') diff --git a/pathod/test.py b/pathod/test.py index 797b4970..0804951e 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -75,16 +75,13 @@ class Daemon: """ Return the log buffer as a list of dictionaries. """ - resp = requests.get("%s/api/log" % self.urlbase, verify=False) - return resp.json()["log"] + return self.thread.server.get_log() def clear_log(self): """ Clear the log. """ - self.logfp.truncate(0) - resp = requests.get("%s/api/clear_log" % self.urlbase, verify=False) - return resp.ok + return self.thread.server.clear_log() def shutdown(self): """ @@ -101,6 +98,7 @@ class _PaThread(threading.Thread): self.name = "PathodThread" self.iface, self.q, self.ssl = iface, q, ssl self.daemonargs = daemonargs + self.server = None def run(self): self.server = pathod.Pathod( -- cgit v1.2.3 From a7522d9308a9592b0c79709f3d04cdd816ee1a57 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 2 Jun 2016 18:27:51 +1200 Subject: pathod.test shouldn't use the API at all --- pathod/test.py | 11 ----------- 1 file changed, 11 deletions(-) (limited to 'pathod') diff --git a/pathod/test.py b/pathod/test.py index 0804951e..28ade101 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -4,12 +4,8 @@ import time from six.moves import queue -import requests -import requests.packages.urllib3 from . import pathod -requests.packages.urllib3.disable_warnings() - class Daemon: IFACE = "127.0.0.1" @@ -41,13 +37,6 @@ class Daemon: """ return "%s/p/%s" % (self.urlbase, spec) - def info(self): - """ - Return some basic info about the remote daemon. - """ - resp = requests.get("%s/api/info" % self.urlbase, verify=False) - return resp.json() - def text_log(self): return self.logfp.getvalue() -- cgit v1.2.3 From a04d7fd166498b54316571a4e71a58837cf11df8 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Thu, 2 Jun 2016 12:59:14 +0530 Subject: Py3: Return bytes from RandomGenerator and use __getitem__ rather than __getslice__ --- pathod/language/generators.py | 37 ++++++++++++++++++++++--------------- 1 file changed, 22 insertions(+), 15 deletions(-) (limited to 'pathod') diff --git a/pathod/language/generators.py b/pathod/language/generators.py index a17e7052..fdba1ab2 100644 --- a/pathod/language/generators.py +++ b/pathod/language/generators.py @@ -3,16 +3,16 @@ import random import mmap DATATYPES = dict( - ascii_letters=string.ascii_letters, - ascii_lowercase=string.ascii_lowercase, - ascii_uppercase=string.ascii_uppercase, - digits=string.digits, - hexdigits=string.hexdigits, - octdigits=string.octdigits, - punctuation=string.punctuation, - whitespace=string.whitespace, - ascii=string.printable, - bytes="".join(chr(i) for i in range(256)) + ascii_letters=string.ascii_letters.encode(), + ascii_lowercase=string.ascii_lowercase.encode(), + ascii_uppercase=string.ascii_uppercase.encode(), + digits=string.digits.encode(), + hexdigits=string.hexdigits.encode(), + octdigits=string.octdigits.encode(), + punctuation=string.punctuation.encode(), + whitespace=string.whitespace.encode(), + ascii=string.printable.encode(), + bytes=bytes(bytearray(range(256))) ) @@ -45,6 +45,15 @@ class TransformGenerator(object): return "'transform(%s)'" % self.gen +def rand_byte(chars): + """ + Return a random character as byte from a charset. + """ + # bytearray has consistent behaviour on both Python 2 and 3 + # while bytes does not + return bytes(bytearray([random.choice(chars)])) + + class RandomGenerator(object): def __init__(self, dtype, length): @@ -55,12 +64,10 @@ class RandomGenerator(object): return self.length def __getitem__(self, x): - return random.choice(DATATYPES[self.dtype]) - - def __getslice__(self, a, b): - b = min(b, self.length) chars = DATATYPES[self.dtype] - return "".join(random.choice(chars) for x in range(a, b)) + if isinstance(x, slice): + return b"".join(rand_byte(chars) for _ in range(*x.indices(self.length))) + return rand_byte(chars) def __repr__(self): return "%s random from %s" % (self.length, self.dtype) -- cgit v1.2.3 From b510616c69135ff2e8f18da3a5fd9a19ddcdfc63 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Thu, 2 Jun 2016 13:00:44 +0530 Subject: Py3: Return bytes from FileGenerator and use __getitem__ instead of __getslice__ --- pathod/language/generators.py | 10 +++++----- 1 file changed, 5 insertions(+), 5 deletions(-) (limited to 'pathod') diff --git a/pathod/language/generators.py b/pathod/language/generators.py index fdba1ab2..bbad3d18 100644 --- a/pathod/language/generators.py +++ b/pathod/language/generators.py @@ -77,17 +77,17 @@ class FileGenerator(object): def __init__(self, path): self.path = path - self.fp = file(path, "rb") + self.fp = open(path, "rb") self.map = mmap.mmap(self.fp.fileno(), 0, access=mmap.ACCESS_READ) def __len__(self): return len(self.map) def __getitem__(self, x): - return self.map.__getitem__(x) - - def __getslice__(self, a, b): - return self.map.__getslice__(a, b) + if isinstance(x, slice): + return self.map.__getitem__(x) + # A slice of length 1 returns a byte object (not an integer) + return self.map.__getitem__(slice(x, x+1 or self.map.size())) def __repr__(self): return "<%s" % self.path -- cgit v1.2.3 From ec5d931ee3f4760b250de000ed005c73d17af462 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Thu, 2 Jun 2016 13:02:17 +0530 Subject: Remove redundant __getslice__ from TransformGenerator --- pathod/language/generators.py | 4 ---- 1 file changed, 4 deletions(-) (limited to 'pathod') diff --git a/pathod/language/generators.py b/pathod/language/generators.py index bbad3d18..e736e043 100644 --- a/pathod/language/generators.py +++ b/pathod/language/generators.py @@ -37,10 +37,6 @@ class TransformGenerator(object): d = self.gen.__getitem__(x) return self.transform(x, d) - def __getslice__(self, a, b): - d = self.gen.__getslice__(a, b) - return self.transform(a, d) - def __repr__(self): return "'transform(%s)'" % self.gen -- cgit v1.2.3 From 69de78515d7d19c72d27439df509cc9294da587f Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Thu, 2 Jun 2016 19:45:18 +1200 Subject: Make last_log actually return last log --- pathod/test.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pathod') diff --git a/pathod/test.py b/pathod/test.py index 28ade101..32b37731 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -58,7 +58,7 @@ class Daemon: l = self.expect_log(1) if not l: return None - return l[0] + return l[-1] def log(self): """ -- cgit v1.2.3 From 75ce2498e8c98ec3fd6583985990a56eeb03f2b8 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Thu, 2 Jun 2016 13:34:18 +0530 Subject: Simplify rand_byte by creating a special case for Python 2 --- pathod/language/generators.py | 6 +++++- 1 file changed, 5 insertions(+), 1 deletion(-) (limited to 'pathod') diff --git a/pathod/language/generators.py b/pathod/language/generators.py index e736e043..20bb7ae1 100644 --- a/pathod/language/generators.py +++ b/pathod/language/generators.py @@ -2,6 +2,8 @@ import string import random import mmap +import six + DATATYPES = dict( ascii_letters=string.ascii_letters.encode(), ascii_lowercase=string.ascii_lowercase.encode(), @@ -47,7 +49,9 @@ def rand_byte(chars): """ # bytearray has consistent behaviour on both Python 2 and 3 # while bytes does not - return bytes(bytearray([random.choice(chars)])) + if six.PY2: + return random.choice(chars) + return bytes([random.choice(chars)]) class RandomGenerator(object): -- cgit v1.2.3 From 5e385405ca38aec56575be688d4906036f3bb650 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Thu, 2 Jun 2016 09:54:19 -0700 Subject: please flake8 --- pathod/language/generators.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pathod') diff --git a/pathod/language/generators.py b/pathod/language/generators.py index 20bb7ae1..01f709e2 100644 --- a/pathod/language/generators.py +++ b/pathod/language/generators.py @@ -87,7 +87,7 @@ class FileGenerator(object): if isinstance(x, slice): return self.map.__getitem__(x) # A slice of length 1 returns a byte object (not an integer) - return self.map.__getitem__(slice(x, x+1 or self.map.size())) + return self.map.__getitem__(slice(x, x + 1 or self.map.size())) def __repr__(self): return "<%s" % self.path -- cgit v1.2.3 From 8e60a9bca9a14a9b3393b82927c56cdb13e1e8c8 Mon Sep 17 00:00:00 2001 From: Shadab Zafar Date: Fri, 3 Jun 2016 02:00:58 +0530 Subject: Handle the slice object case in TransformGenerator.__getitem__ --- pathod/language/generators.py | 2 ++ 1 file changed, 2 insertions(+) (limited to 'pathod') diff --git a/pathod/language/generators.py b/pathod/language/generators.py index 01f709e2..9fff3082 100644 --- a/pathod/language/generators.py +++ b/pathod/language/generators.py @@ -37,6 +37,8 @@ class TransformGenerator(object): def __getitem__(self, x): d = self.gen.__getitem__(x) + if isinstance(x, slice): + return self.transform(x.start, d) return self.transform(x, d) def __repr__(self): -- cgit v1.2.3 From c5076f5e019b73ec2f6efea5a4bafed90423df8f Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 3 Jun 2016 11:47:07 +1200 Subject: Implement a service connection handler counter, use it in Pathod test suite Lots of failures, but that's a good thing. --- pathod/test.py | 15 +++++++++++++++ 1 file changed, 15 insertions(+) (limited to 'pathod') diff --git a/pathod/test.py b/pathod/test.py index 32b37731..b10e9229 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -7,6 +7,10 @@ from six.moves import queue from . import pathod +class TimeoutError(Exception): + pass + + class Daemon: IFACE = "127.0.0.1" @@ -40,6 +44,17 @@ class Daemon: def text_log(self): return self.logfp.getvalue() + def wait_for_silence(self, timeout=5): + start = time.time() + while 1: + if time.time() - start >= timeout: + raise TimeoutError( + "%s service threads still alive" % + self.thread.server.counter.count + ) + if self.thread.server.counter.count == 0: + return + def expect_log(self, n, timeout=5): l = [] start = time.time() -- cgit v1.2.3 From 36383a6146d605879bf8c2834370b09f42bfec06 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 3 Jun 2016 11:57:42 +1200 Subject: Pathod websocket service threads should not be immortal --- pathod/protocols/websockets.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'pathod') diff --git a/pathod/protocols/websockets.py b/pathod/protocols/websockets.py index 134d27bc..2b60e618 100644 --- a/pathod/protocols/websockets.py +++ b/pathod/protocols/websockets.py @@ -18,7 +18,7 @@ class WebsocketsProtocol: frm = websockets.Frame.from_file(self.pathod_handler.rfile) except NetlibException as e: lg("Error reading websocket frame: %s" % e) - break + return None, None ended = time.time() lg(frm.human_readable()) retlog = dict( -- cgit v1.2.3 From e60860e65d06d2b4452b7ea94902d79eed11d78c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 3 Jun 2016 12:06:36 +1200 Subject: Make tcp.Client.connect return a context manager that closes the connection --- pathod/pathoc.py | 3 ++- 1 file changed, 2 insertions(+), 1 deletion(-) (limited to 'pathod') diff --git a/pathod/pathoc.py b/pathod/pathoc.py index 2b7d053c..5cfb4591 100644 --- a/pathod/pathoc.py +++ b/pathod/pathoc.py @@ -286,7 +286,7 @@ class Pathoc(tcp.TCPClient): if self.use_http2 and not self.ssl: raise NotImplementedError("HTTP2 without SSL is not supported.") - tcp.TCPClient.connect(self) + ret = tcp.TCPClient.connect(self) if connect_to: self.http_connect(connect_to) @@ -324,6 +324,7 @@ class Pathoc(tcp.TCPClient): if self.timeout: self.settimeout(self.timeout) + return ret def stop(self): if self.ws_framereader: -- cgit v1.2.3 From 6943d7e3977a53df83267c1bbe58f60b9867a2a6 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Fri, 3 Jun 2016 13:57:12 +1200 Subject: More explicit name for the tcp.Server handler counter --- pathod/test.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'pathod') diff --git a/pathod/test.py b/pathod/test.py index b10e9229..11462729 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -50,9 +50,9 @@ class Daemon: if time.time() - start >= timeout: raise TimeoutError( "%s service threads still alive" % - self.thread.server.counter.count + self.thread.server.handler_counter.count ) - if self.thread.server.counter.count == 0: + if self.thread.server.handler_counter.count == 0: return def expect_log(self, n, timeout=5): -- cgit v1.2.3