diff options
| -rw-r--r-- | libpathod/language/actions.py | 119 | ||||
| -rw-r--r-- | libpathod/language/base.py | 236 | ||||
| -rw-r--r-- | libpathod/language/http.py | 23 | ||||
| -rw-r--r-- | libpathod/language/message.py | 81 | ||||
| -rw-r--r-- | libpathod/language/websockets.py | 13 | ||||
| -rw-r--r-- | libpathod/pathod.py | 3 | ||||
| -rw-r--r-- | test/test_language_actions.py | 127 | ||||
| -rw-r--r-- | test/test_language_base.py | 123 |
8 files changed, 372 insertions, 353 deletions
diff --git a/libpathod/language/actions.py b/libpathod/language/actions.py new file mode 100644 index 00000000..10feabf8 --- /dev/null +++ b/libpathod/language/actions.py @@ -0,0 +1,119 @@ +import abc +import copy +import random + +import contrib.pyparsing as pp + +from . import base + + +class _Action(base.Token): + """ + An action that operates on the raw data stream of the message. All + actions have one thing in common: an offset that specifies where the + action should take place. + """ + def __init__(self, offset): + self.offset = offset + + def resolve(self, settings, msg): + """ + Resolves offset specifications to a numeric offset. Returns a copy + of the action object. + """ + c = copy.copy(self) + l = msg.length(settings) + if c.offset == "r": + c.offset = random.randrange(l) + elif c.offset == "a": + c.offset = l + 1 + return c + + def __cmp__(self, other): + return cmp(self.offset, other.offset) + + def __repr__(self): + return self.spec() + + @abc.abstractmethod + def spec(self): # pragma: no cover + pass + + @abc.abstractmethod + def intermediate(self, settings): # pragma: no cover + pass + + +class PauseAt(_Action): + def __init__(self, offset, seconds): + _Action.__init__(self, offset) + self.seconds = seconds + + @classmethod + def expr(klass): + e = pp.Literal("p").suppress() + e += base.Offset + e += pp.Literal(",").suppress() + e += pp.MatchFirst( + [ + base.v_integer, + pp.Literal("f") + ] + ) + return e.setParseAction(lambda x: klass(*x)) + + def spec(self): + return "p%s,%s"%(self.offset, self.seconds) + + def intermediate(self, settings): + return (self.offset, "pause", self.seconds) + + def freeze(self, settings): + return self + + +class DisconnectAt(_Action): + def __init__(self, offset): + _Action.__init__(self, offset) + + @classmethod + def expr(klass): + e = pp.Literal("d").suppress() + e += base.Offset + return e.setParseAction(lambda x: klass(*x)) + + def spec(self): + return "d%s"%self.offset + + def intermediate(self, settings): + return (self.offset, "disconnect") + + def freeze(self, settings): + return self + + +class InjectAt(_Action): + def __init__(self, offset, value): + _Action.__init__(self, offset) + self.value = value + + @classmethod + def expr(klass): + e = pp.Literal("i").suppress() + e += base.Offset + e += pp.Literal(",").suppress() + e += base.Value + return e.setParseAction(lambda x: klass(*x)) + + def spec(self): + return "i%s,%s"%(self.offset, self.value.spec()) + + def intermediate(self, settings): + return ( + self.offset, + "inject", + self.value.get_generator(settings) + ) + + def freeze(self, settings): + return InjectAt(self.offset, self.value.freeze(settings)) diff --git a/libpathod/language/base.py b/libpathod/language/base.py index 4c337a9b..f91add75 100644 --- a/libpathod/language/base.py +++ b/libpathod/language/base.py @@ -1,14 +1,11 @@ import operator -import random import os -import copy import abc import contrib.pyparsing as pp from .. import utils from . import generators, exceptions -TRUNCATE = 1024 Sep = pp.Optional(pp.Literal(":")).suppress() @@ -43,7 +40,7 @@ v_naked_literal = pp.MatchFirst( ) -class _Token(object): +class Token(object): """ A specification token. Tokens are immutable. """ @@ -74,7 +71,7 @@ class _Token(object): return self.spec() -class _ValueLiteral(_Token): +class _ValueLiteral(Token): def __init__(self, val): self.val = val.decode("string_escape") @@ -111,7 +108,7 @@ class ValueNakedLiteral(_ValueLiteral): return self.val.encode("string_escape") -class ValueGenerate(_Token): +class ValueGenerate(Token): def __init__(self, usize, unit, datatype): if not unit: unit = "b" @@ -154,7 +151,7 @@ class ValueGenerate(_Token): return s -class ValueFile(_Token): +class ValueFile(Token): def __init__(self, path): self.path = str(path) @@ -215,9 +212,9 @@ Offset = pp.MatchFirst( ) -class _Component(_Token): +class _Component(Token): """ - A value component of the primary specification of an HTTP message. + A value component of the primary specification of an message. """ @abc.abstractmethod def values(self, settings): # pragma: no cover @@ -258,7 +255,7 @@ class KeyValue(_Component): ) -class PathodSpec(_Token): +class PathodSpec(Token): def __init__(self, value): self.value = value try: @@ -291,32 +288,6 @@ class PathodSpec(_Token): return PathodSpec(ValueLiteral(f.encode("string_escape"))) -class SimpleValue(_Component): - """ - A simple value - i.e. one without a preface. - """ - def __init__(self, value): - if isinstance(value, basestring): - value = ValueLiteral(value) - self.value = value - - @classmethod - def expr(klass): - e = Value | NakedValue - return e.setParseAction(lambda x: klass(*x)) - - def values(self, settings): - return [ - self.value.get_generator(settings), - ] - - def spec(self): - return "%s"%(self.value.spec()) - - def freeze(self, settings): - return self.__class__(self.value.freeze(settings)) - - class CaselessLiteral(_Component): """ A caseless token that can take only one value. @@ -422,194 +393,27 @@ class PreValue(_Component): return self.__class__(self.value.freeze(settings)) -class _Action(_Token): +class SimpleValue(_Component): """ - An action that operates on the raw data stream of the message. All - actions have one thing in common: an offset that specifies where the - action should take place. + A simple value - i.e. one without a preface. """ - def __init__(self, offset): - self.offset = offset - - def resolve(self, settings, msg): - """ - Resolves offset specifications to a numeric offset. Returns a copy - of the action object. - """ - c = copy.copy(self) - l = msg.length(settings) - if c.offset == "r": - c.offset = random.randrange(l) - elif c.offset == "a": - c.offset = l + 1 - return c - - def __cmp__(self, other): - return cmp(self.offset, other.offset) - - def __repr__(self): - return self.spec() - - @abc.abstractmethod - def spec(self): # pragma: no cover - pass - - @abc.abstractmethod - def intermediate(self, settings): # pragma: no cover - pass - - -class PauseAt(_Action): - def __init__(self, offset, seconds): - _Action.__init__(self, offset) - self.seconds = seconds - - @classmethod - def expr(klass): - e = pp.Literal("p").suppress() - e += Offset - e += pp.Literal(",").suppress() - e += pp.MatchFirst( - [ - v_integer, - pp.Literal("f") - ] - ) - return e.setParseAction(lambda x: klass(*x)) - - def spec(self): - return "p%s,%s"%(self.offset, self.seconds) - - def intermediate(self, settings): - return (self.offset, "pause", self.seconds) - - def freeze(self, settings): - return self - - -class DisconnectAt(_Action): - def __init__(self, offset): - _Action.__init__(self, offset) - - @classmethod - def expr(klass): - e = pp.Literal("d").suppress() - e += Offset - return e.setParseAction(lambda x: klass(*x)) - - def spec(self): - return "d%s"%self.offset - - def intermediate(self, settings): - return (self.offset, "disconnect") - - def freeze(self, settings): - return self - - -class InjectAt(_Action): - def __init__(self, offset, value): - _Action.__init__(self, offset) + def __init__(self, value): + if isinstance(value, basestring): + value = ValueLiteral(value) self.value = value @classmethod def expr(klass): - e = pp.Literal("i").suppress() - e += Offset - e += pp.Literal(",").suppress() - e += Value + e = Value | NakedValue return e.setParseAction(lambda x: klass(*x)) - def spec(self): - return "i%s,%s"%(self.offset, self.value.spec()) - - def intermediate(self, settings): - return ( - self.offset, - "inject", - self.value.get_generator(settings) - ) - - def freeze(self, settings): - return InjectAt(self.offset, self.value.freeze(settings)) - - -class _Message(object): - __metaclass__ = abc.ABCMeta - logattrs = [] - - def __init__(self, tokens): - self.tokens = tokens - - def toks(self, klass): - """ - Fetch all tokens that are instances of klass - """ - return [i for i in self.tokens if isinstance(i, klass)] - - def tok(self, klass): - """ - Fetch first token that is an instance of klass - """ - l = self.toks(klass) - if l: - return l[0] - - @property - def actions(self): - return self.toks(_Action) - - def length(self, settings): - """ - Calculate the length of the base message without any applied - actions. - """ - return sum(len(x) for x in self.values(settings)) - - def preview_safe(self): - """ - Return a copy of this message that issafe for previews. - """ - tokens = [i for i in self.tokens if not isinstance(i, PauseAt)] - return self.__class__(tokens) - - def maximum_length(self, settings): - """ - Calculate the maximum length of the base message with all applied - actions. - """ - l = self.length(settings) - for i in self.actions: - if isinstance(i, InjectAt): - l += len(i.value.get_generator(settings)) - return l - - @classmethod - def expr(klass): # pragma: no cover - pass + def values(self, settings): + return [ + self.value.get_generator(settings), + ] - def log(self, settings): - """ - A dictionary that should be logged if this message is served. - """ - ret = {} - for i in self.logattrs: - v = getattr(self, i) - # Careful not to log any VALUE specs without sanitizing them first. - # We truncate at 1k. - if hasattr(v, "values"): - v = [x[:TRUNCATE] for x in v.values(settings)] - v = "".join(v).encode("string_escape") - elif hasattr(v, "__len__"): - v = v[:TRUNCATE] - v = v.encode("string_escape") - ret[i] = v - ret["spec"] = self.spec() - return ret + def spec(self): + return "%s"%(self.value.spec()) def freeze(self, settings): - r = self.resolve(settings) - return self.__class__([i.freeze(settings) for i in r.tokens]) - - def __repr__(self): - return self.spec() + return self.__class__(self.value.freeze(settings)) diff --git a/libpathod/language/http.py b/libpathod/language/http.py index a759aeb1..c1c2ae96 100644 --- a/libpathod/language/http.py +++ b/libpathod/language/http.py @@ -5,7 +5,7 @@ import contrib.pyparsing as pp import netlib.websockets from netlib import http_status, http_uastrings -from . import base, generators, exceptions +from . import base, generators, exceptions, actions, message class WS(base.CaselessLiteral): @@ -100,8 +100,11 @@ def get_header(val, headers): return None -class _HTTPMessage(base._Message): +class _HTTPMessage(message.Message): version = "HTTP/1.1" + @property + def actions(self): + return self.toks(actions._Action) @property def raw(self): @@ -134,13 +137,14 @@ class Response(_HTTPMessage): comps = ( Body, Header, - base.PauseAt, - base.DisconnectAt, - base.InjectAt, ShortcutContentType, ShortcutLocation, Raw, - Reason + Reason, + + actions.PauseAt, + actions.DisconnectAt, + actions.InjectAt, ) logattrs = ["code", "reason", "version", "body"] @@ -241,13 +245,14 @@ class Request(_HTTPMessage): comps = ( Body, Header, - base.PauseAt, - base.DisconnectAt, - base.InjectAt, ShortcutContentType, ShortcutUserAgent, Raw, base.PathodSpec, + + actions.PauseAt, + actions.DisconnectAt, + actions.InjectAt, ) logattrs = ["method", "path", "body"] diff --git a/libpathod/language/message.py b/libpathod/language/message.py new file mode 100644 index 00000000..b5ef7045 --- /dev/null +++ b/libpathod/language/message.py @@ -0,0 +1,81 @@ +import abc +from . import actions + +LOG_TRUNCATE = 1024 + + +class Message(object): + __metaclass__ = abc.ABCMeta + logattrs = [] + + def __init__(self, tokens): + self.tokens = tokens + + def toks(self, klass): + """ + Fetch all tokens that are instances of klass + """ + return [i for i in self.tokens if isinstance(i, klass)] + + def tok(self, klass): + """ + Fetch first token that is an instance of klass + """ + l = self.toks(klass) + if l: + return l[0] + + def length(self, settings): + """ + Calculate the length of the base message without any applied + actions. + """ + return sum(len(x) for x in self.values(settings)) + + def preview_safe(self): + """ + Return a copy of this message that issafe for previews. + """ + tokens = [i for i in self.tokens if not isinstance(i, actions.PauseAt)] + return self.__class__(tokens) + + def maximum_length(self, settings): + """ + Calculate the maximum length of the base message with all applied + actions. + """ + l = self.length(settings) + for i in self.actions: + if isinstance(i, actions.InjectAt): + l += len(i.value.get_generator(settings)) + return l + + @classmethod + def expr(klass): # pragma: no cover + pass + + def log(self, settings): + """ + A dictionary that should be logged if this message is served. + """ + ret = {} + for i in self.logattrs: + v = getattr(self, i) + # Careful not to log any VALUE specs without sanitizing them first. + # We truncate at 1k. + if hasattr(v, "values"): + v = [x[:LOG_TRUNCATE] for x in v.values(settings)] + v = "".join(v).encode("string_escape") + elif hasattr(v, "__len__"): + v = v[:LOG_TRUNCATE] + v = v.encode("string_escape") + ret[i] = v + ret["spec"] = self.spec() + return ret + + def freeze(self, settings): + r = self.resolve(settings) + return self.__class__([i.freeze(settings) for i in r.tokens]) + + def __repr__(self): + return self.spec() diff --git a/libpathod/language/websockets.py b/libpathod/language/websockets.py index b666b2fe..3cc4adb0 100644 --- a/libpathod/language/websockets.py +++ b/libpathod/language/websockets.py @@ -1,7 +1,7 @@ import netlib.websockets import contrib.pyparsing as pp -from . import base, generators +from . import base, generators, actions, message """ wf:ctext:b'foo' @@ -21,14 +21,17 @@ class Body(base.PreValue): preamble = "b" -class WebsocketFrame(base._Message): +class WebsocketFrame(message.Message): comps = ( Body, - base.PauseAt, - base.DisconnectAt, - base.InjectAt + actions.PauseAt, + actions.DisconnectAt, + actions.InjectAt ) logattrs = ["body"] + @property + def actions(self): + return self.toks(actions._Action) @property def body(self): diff --git a/libpathod/pathod.py b/libpathod/pathod.py index 4e856f10..d69d2298 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -9,6 +9,7 @@ import netlib.utils from . import version, app, language, utils import language.http +import language.actions DEFAULT_CERT_DOMAIN = "pathod.net" @@ -365,7 +366,7 @@ class Pathod(tcp.TCPServer): return "File access denied.", None if self.sizelimit and l > self.sizelimit: return "Response too large.", None - pauses = [isinstance(i, language.base.PauseAt) for i in req.actions] + pauses = [isinstance(i, language.actions.PauseAt) for i in req.actions] if self.nohang and any(pauses): return "Pauses have been disabled.", None return None, req diff --git a/test/test_language_actions.py b/test/test_language_actions.py new file mode 100644 index 00000000..7676fb72 --- /dev/null +++ b/test/test_language_actions.py @@ -0,0 +1,127 @@ +import cStringIO + +from libpathod.language import actions +from libpathod import language + + +def parse_request(s): + return language.parse_requests(s)[0] + + +class TestDisconnects: + def test_parse_response(self): + a = language.parse_response("400:d0").actions[0] + assert a.spec() == "d0" + a = language.parse_response("400:dr").actions[0] + assert a.spec() == "dr" + + def test_at(self): + e = actions.DisconnectAt.expr() + v = e.parseString("d0")[0] + assert isinstance(v, actions.DisconnectAt) + assert v.offset == 0 + + v = e.parseString("d100")[0] + assert v.offset == 100 + + e = actions.DisconnectAt.expr() + v = e.parseString("dr")[0] + assert v.offset == "r" + + def test_spec(self): + assert actions.DisconnectAt("r").spec() == "dr" + assert actions.DisconnectAt(10).spec() == "d10" + + +class TestInject: + def test_parse_response(self): + a = language.parse_response("400:ir,@100").actions[0] + assert a.offset == "r" + assert a.value.datatype == "bytes" + assert a.value.usize == 100 + + a = language.parse_response("400:ia,@100").actions[0] + assert a.offset == "a" + + def test_at(self): + e = actions.InjectAt.expr() + v = e.parseString("i0,'foo'")[0] + assert v.value.val == "foo" + assert v.offset == 0 + assert isinstance(v, actions.InjectAt) + + v = e.parseString("ir,'foo'")[0] + assert v.offset == "r" + + def test_serve(self): + s = cStringIO.StringIO() + r = language.parse_response("400:i0,'foo'") + assert language.serve(r, s, {}) + + def test_spec(self): + e = actions.InjectAt.expr() + v = e.parseString("i0,'foo'")[0] + assert v.spec() == 'i0,"foo"' + + def test_spec(self): + e = actions.InjectAt.expr() + v = e.parseString("i0,@100")[0] + v2 = v.freeze({}) + v3 = v2.freeze({}) + assert v2.value.val == v3.value.val + + +class TestPauses: + def test_parse_response(self): + e = actions.PauseAt.expr() + v = e.parseString("p10,10")[0] + assert v.seconds == 10 + assert v.offset == 10 + + v = e.parseString("p10,f")[0] + assert v.seconds == "f" + + v = e.parseString("pr,f")[0] + assert v.offset == "r" + + v = e.parseString("pa,f")[0] + assert v.offset == "a" + + def test_request(self): + r = language.parse_response('400:p10,10') + assert r.actions[0].spec() == "p10,10" + + def test_spec(self): + assert actions.PauseAt("r", 5).spec() == "pr,5" + assert actions.PauseAt(0, 5).spec() == "p0,5" + assert actions.PauseAt(0, "f").spec() == "p0,f" + + def test_freeze(self): + l = actions.PauseAt("r", 5) + assert l.freeze({}).spec() == l.spec() + + +class Test_Action: + def test_cmp(self): + a = actions.DisconnectAt(0) + b = actions.DisconnectAt(1) + c = actions.DisconnectAt(0) + assert a < b + assert a == c + l = [b, a] + l.sort() + assert l[0].offset == 0 + + def test_resolve(self): + r = parse_request('GET:"/foo"') + e = actions.DisconnectAt("r") + ret = e.resolve({}, r) + assert isinstance(ret.offset, int) + + def test_repr(self): + e = actions.DisconnectAt("r") + assert repr(e) + + def test_freeze(self): + l = actions.DisconnectAt(5) + assert l.freeze({}).spec() == l.spec() diff --git a/test/test_language_base.py b/test/test_language_base.py index 48afd675..deb33317 100644 --- a/test/test_language_base.py +++ b/test/test_language_base.py @@ -1,7 +1,6 @@ import os -import cStringIO from libpathod import language -from libpathod.language import base, http, websockets, writer, exceptions +from libpathod.language import base, exceptions import tutils @@ -259,126 +258,6 @@ class TestKeyValue: assert v2.value.val == v3.value.val - -class Test_Action: - def test_cmp(self): - a = base.DisconnectAt(0) - b = base.DisconnectAt(1) - c = base.DisconnectAt(0) - assert a < b - assert a == c - l = [b, a] - l.sort() - assert l[0].offset == 0 - - def test_resolve(self): - r = parse_request('GET:"/foo"') - e = base.DisconnectAt("r") - ret = e.resolve({}, r) - assert isinstance(ret.offset, int) - - def test_repr(self): - e = base.DisconnectAt("r") - assert repr(e) - - def test_freeze(self): - l = base.DisconnectAt(5) - assert l.freeze({}).spec() == l.spec() - - -class TestDisconnects: - def test_parse_response(self): - a = language.parse_response("400:d0").actions[0] - assert a.spec() == "d0" - a = language.parse_response("400:dr").actions[0] - assert a.spec() == "dr" - - def test_at(self): - e = base.DisconnectAt.expr() - v = e.parseString("d0")[0] - assert isinstance(v, base.DisconnectAt) - assert v.offset == 0 - - v = e.parseString("d100")[0] - assert v.offset == 100 - - e = base.DisconnectAt.expr() - v = e.parseString("dr")[0] - assert v.offset == "r" - - def test_spec(self): - assert base.DisconnectAt("r").spec() == "dr" - assert base.DisconnectAt(10).spec() == "d10" - - -class TestInject: - def test_parse_response(self): - a = language.parse_response("400:ir,@100").actions[0] - assert a.offset == "r" - assert a.value.datatype == "bytes" - assert a.value.usize == 100 - - a = language.parse_response("400:ia,@100").actions[0] - assert a.offset == "a" - - def test_at(self): - e = base.InjectAt.expr() - v = e.parseString("i0,'foo'")[0] - assert v.value.val == "foo" - assert v.offset == 0 - assert isinstance(v, base.InjectAt) - - v = e.parseString("ir,'foo'")[0] - assert v.offset == "r" - - def test_serve(self): - s = cStringIO.StringIO() - r = language.parse_response("400:i0,'foo'") - assert language.serve(r, s, {}) - - def test_spec(self): - e = base.InjectAt.expr() - v = e.parseString("i0,'foo'")[0] - assert v.spec() == 'i0,"foo"' - - def test_spec(self): - e = base.InjectAt.expr() - v = e.parseString("i0,@100")[0] - v2 = v.freeze({}) - v3 = v2.freeze({}) - assert v2.value.val == v3.value.val - - -class TestPauses: - def test_parse_response(self): - e = base.PauseAt.expr() - v = e.parseString("p10,10")[0] - assert v.seconds == 10 - assert v.offset == 10 - - v = e.parseString("p10,f")[0] - assert v.seconds == "f" - - v = e.parseString("pr,f")[0] - assert v.offset == "r" - - v = e.parseString("pa,f")[0] - assert v.offset == "a" - - def test_request(self): - r = language.parse_response('400:p10,10') - assert r.actions[0].spec() == "p10,10" - - def test_spec(self): - assert base.PauseAt("r", 5).spec() == "pr,5" - assert base.PauseAt(0, 5).spec() == "p0,5" - assert base.PauseAt(0, "f").spec() == "p0,f" - - def test_freeze(self): - l = base.PauseAt("r", 5) - assert l.freeze({}).spec() == l.spec() - - def test_options_or_value(): class TT(base.OptionsOrValue): options = [ |