From 5aadf92767614b7bd8e2ef677085410ac359e5e8 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 25 Oct 2014 08:18:39 +1300 Subject: Nicer way to specify patterns read for file - just use a path --- libpathod/language.py | 8 ++------ 1 file changed, 2 insertions(+), 6 deletions(-) (limited to 'libpathod/language.py') diff --git a/libpathod/language.py b/libpathod/language.py index 286a1a8e..002c8205 100644 --- a/libpathod/language.py +++ b/libpathod/language.py @@ -961,7 +961,7 @@ def read_file(settings, s): return file(s, "rb").read() -def parse_response(settings, s): +def parse_response(s): """ May raise ParseException or FileAccessDenied """ @@ -969,15 +969,13 @@ def parse_response(settings, s): s = s.decode("ascii") except UnicodeError: raise ParseException("Spec must be valid ASCII.", 0, 0) - if s.startswith(FILESTART): - s = read_file(settings, s) try: return Response(Response.expr().parseString(s, parseAll=True)) except pp.ParseException, v: raise ParseException(v.msg, v.line, v.col) -def parse_request(settings, s): +def parse_request(s): """ May raise ParseException or FileAccessDenied """ @@ -985,8 +983,6 @@ def parse_request(settings, s): s = s.decode("ascii") except UnicodeError: raise ParseException("Spec must be valid ASCII.", 0, 0) - if s.startswith(FILESTART): - s = read_file(settings, s) try: return Request(Request.expr().parseString(s, parseAll=True)) except pp.ParseException, v: -- cgit v1.2.3 From d6ee5327112182202513ff6ce62d95df1567fdb6 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 25 Oct 2014 14:24:05 +1300 Subject: Whitespace and formatting --- libpathod/language.py | 137 +++++++++++++++++++++++++++++++++++--------------- 1 file changed, 96 insertions(+), 41 deletions(-) (limited to 'libpathod/language.py') diff --git a/libpathod/language.py b/libpathod/language.py index 002c8205..2969055b 100644 --- a/libpathod/language.py +++ b/libpathod/language.py @@ -1,4 +1,10 @@ -import operator, string, random, mmap, os, time, copy +import operator +import string +import random +import mmap +import os +import time +import copy import abc from email.utils import formatdate import contrib.pyparsing as pp @@ -9,7 +15,9 @@ import utils BLOCKSIZE = 1024 TRUNCATE = 1024 -class FileAccessDenied(Exception): pass + +class FileAccessDenied(Exception): + pass class ParseException(Exception): @@ -20,7 +28,7 @@ class ParseException(Exception): self.col = col def marked(self): - return "%s\n%s"%(self.s, " "*(self.col-1) + "^") + return "%s\n%s"%(self.s, " "*(self.col - 1) + "^") def __str__(self): return "%s at char %s"%(self.msg, self.col) @@ -40,7 +48,9 @@ def send_chunk(fp, val, blocksize, start, end): def write_values(fp, vals, actions, sofar=0, skip=0, blocksize=BLOCKSIZE): """ vals: A list of values, which may be strings or Value objects. - actions: A list of (offset, action, arg) tuples. Action may be "pause" or "disconnect". + + actions: A list of (offset, action, arg) tuples. Action may be "pause" + or "disconnect". Both vals and actions are in reverse order, with the first items last. @@ -53,7 +63,13 @@ def write_values(fp, vals, actions, sofar=0, skip=0, blocksize=BLOCKSIZE): offset = 0 while actions and actions[-1][0] < (sofar + len(v)): a = actions.pop() - offset += send_chunk(fp, v, blocksize, offset, a[0]-sofar-offset) + offset += send_chunk( + fp, + v, + blocksize, + offset, + a[0]-sofar-offset + ) if a[1] == "pause": time.sleep(a[2]) elif a[1] == "disconnect": @@ -128,8 +144,18 @@ v_integer = pp.Regex(r"\d+")\ v_literal = pp.MatchFirst( [ - pp.QuotedString("\"", escChar="\\", unquoteResults=True, multiline=True), - pp.QuotedString("'", escChar="\\", unquoteResults=True, multiline=True), + pp.QuotedString( + "\"", + escChar="\\", + unquoteResults=True, + multiline=True + ), + pp.QuotedString( + "'", + escChar="\\", + unquoteResults=True, + multiline=True + ), ] ) @@ -202,6 +228,7 @@ class _Token(object): A specification token. Tokens are immutable. """ __metaclass__ = abc.ABCMeta + @abc.abstractmethod def expr(klass): # pragma: no cover """ @@ -278,7 +305,10 @@ class ValueGenerate(_Token): def expr(klass): e = pp.Literal("@").suppress() + v_integer - u = reduce(operator.or_, [pp.Literal(i) for i in utils.SIZE_UNITS.keys()]) + u = reduce( + operator.or_, + [pp.Literal(i) for i in utils.SIZE_UNITS.keys()] + ) e = e + pp.Optional(u, default=None) s = pp.Literal(",").suppress() @@ -318,7 +348,9 @@ class ValueFile(_Token): s = os.path.expanduser(self.path) s = os.path.normpath(os.path.abspath(os.path.join(sd, s))) if not uf and not s.startswith(sd): - raise FileAccessDenied("File access outside of configured directory") + raise FileAccessDenied( + "File access outside of configured directory" + ) if not os.path.isfile(s): raise FileAccessDenied("File not readable") return FileGenerator(s) @@ -347,12 +379,12 @@ NakedValue = pp.MatchFirst( Offset = pp.MatchFirst( - [ - v_integer, - pp.Literal("r"), - pp.Literal("a") - ] - ) + [ + v_integer, + pp.Literal("r"), + pp.Literal("a") + ] +) class Raw(_Token): @@ -392,11 +424,11 @@ class _Header(_Component): def values(self, settings): return [ - self.key.get_generator(settings), - ": ", - self.value.get_generator(settings), - "\r\n", - ] + self.key.get_generator(settings), + ": ", + self.value.get_generator(settings), + "\r\n", + ] class Header(_Header): @@ -459,7 +491,10 @@ class ShortcutUserAgent(_Header): @classmethod def expr(klass): e = pp.Literal("u").suppress() - u = reduce(operator.or_, [pp.Literal(i[1]) for i in http_uastrings.UASTRINGS]) + u = reduce( + operator.or_, + [pp.Literal(i[1]) for i in http_uastrings.UASTRINGS] + ) e += u | Value return e.setParseAction(lambda x: klass(*x)) @@ -470,7 +505,6 @@ class ShortcutUserAgent(_Header): return ShortcutUserAgent(self.value.freeze(settings)) - class Body(_Component): def __init__(self, value): self.value = value @@ -483,8 +517,8 @@ class Body(_Component): def values(self, settings): return [ - self.value.get_generator(settings), - ] + self.value.get_generator(settings), + ] def spec(self): return "b%s"%(self.value.spec()) @@ -506,8 +540,8 @@ class Path(_Component): def values(self, settings): return [ - self.value.get_generator(settings), - ] + self.value.get_generator(settings), + ] def spec(self): return "%s"%(self.value.spec()) @@ -527,6 +561,7 @@ class Method(_Component): "trace", "connect", ] + def __init__(self, value): # If it's a string, we were passed one of the methods, so we upper-case # it to be canonical. The user can specify a different case by using a @@ -645,11 +680,11 @@ class PauseAt(_Action): e += Offset e += pp.Literal(",").suppress() e += pp.MatchFirst( - [ - v_integer, - pp.Literal("f") - ] - ) + [ + v_integer, + pp.Literal("f") + ] + ) return e.setParseAction(lambda x: klass(*x)) def spec(self): @@ -700,10 +735,10 @@ class InjectAt(_Action): def intermediate(self, settings): return ( - self.offset, - "inject", - self.value.get_generator(settings) - ) + self.offset, + "inject", + self.value.get_generator(settings) + ) def freeze(self, settings): return InjectAt(self.offset, self.value.freeze(settings)) @@ -712,6 +747,7 @@ class InjectAt(_Action): class _Message(object): __metaclass__ = abc.ABCMeta version = "HTTP/1.1" + def __init__(self, tokens): self.tokens = tokens @@ -741,7 +777,8 @@ class _Message(object): def length(self, settings): """ - Calculate the length of the base message without any applied actions. + Calculate the length of the base message without any applied + actions. """ return sum(len(x) for x in self.values(settings)) @@ -754,7 +791,8 @@ class _Message(object): def maximum_length(self, settings): """ - Calculate the maximum length of the base message with all applied actions. + Calculate the maximum length of the base message with all applied + actions. """ l = self.length(settings) for i in self.actions: @@ -786,7 +824,13 @@ class _Message(object): tokens.append( Header( ValueLiteral("Date"), - ValueLiteral(formatdate(timeval=None, localtime=False, usegmt=True)) + ValueLiteral( + formatdate( + timeval=None, + localtime=False, + usegmt=True + ) + ) ) ) intermediate = self.__class__(tokens) @@ -807,7 +851,8 @@ class _Message(object): ret = {} for i in self.logattrs: v = getattr(self, i) - # Careful not to log any VALUE specs without sanitizing them first. We truncate at 1k. + # Careful not to log any VALUE specs without sanitizing them first. + # We truncate at 1k. if hasattr(v, "values"): v = [x[:TRUNCATE] for x in v.values(settings)] v = "".join(v).encode("string_escape") @@ -838,6 +883,7 @@ class _Message(object): Sep = pp.Optional(pp.Literal(":")).suppress() + class Response(_Message): comps = ( Body, @@ -851,6 +897,7 @@ class Response(_Message): Reason ) logattrs = ["code", "reason", "version", "body"] + @property def code(self): return self._get_token(Code) @@ -866,7 +913,14 @@ class Response(_Message): if self.reason: l.extend(self.reason.values(settings)) else: - l.append(LiteralGenerator(http_status.RESPONSES.get(int(self.code.code), "Unknown code"))) + l.append( + LiteralGenerator( + http_status.RESPONSES.get( + int(self.code.code), + "Unknown code" + ) + ) + ) return l @classmethod @@ -897,6 +951,7 @@ class Request(_Message): Raw ) logattrs = ["method", "path", "body"] + @property def method(self): return self._get_token(Method) @@ -944,7 +999,7 @@ def make_error_response(reason, body=None): ] return PathodErrorResponse(tokens) -FILESTART = "+" + def read_file(settings, s): uf = settings.get("unconstrained_file_access") sd = settings.get("staticdir") -- cgit v1.2.3 From 00d0ee5ad56d8243b1e9bfffec9a941e11359d2c Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 25 Oct 2014 15:30:54 +1300 Subject: Parse patterns eagerly on instantiation --- libpathod/language.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libpathod/language.py') diff --git a/libpathod/language.py b/libpathod/language.py index 2969055b..b4b59167 100644 --- a/libpathod/language.py +++ b/libpathod/language.py @@ -1018,7 +1018,7 @@ def read_file(settings, s): def parse_response(s): """ - May raise ParseException or FileAccessDenied + May raise ParseException """ try: s = s.decode("ascii") @@ -1032,7 +1032,7 @@ def parse_response(s): def parse_request(s): """ - May raise ParseException or FileAccessDenied + May raise ParseException """ try: s = s.decode("ascii") -- cgit v1.2.3 From 6d8431ab3e96568b3579a85e680371fd20c961aa Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 25 Oct 2014 16:20:23 +1300 Subject: Allow specification of multiple patterns from file and on command line --- libpathod/language.py | 9 +++++++-- 1 file changed, 7 insertions(+), 2 deletions(-) (limited to 'libpathod/language.py') diff --git a/libpathod/language.py b/libpathod/language.py index b4b59167..f40499f6 100644 --- a/libpathod/language.py +++ b/libpathod/language.py @@ -1030,7 +1030,7 @@ def parse_response(s): raise ParseException(v.msg, v.line, v.col) -def parse_request(s): +def parse_requests(s): """ May raise ParseException """ @@ -1039,6 +1039,11 @@ def parse_request(s): except UnicodeError: raise ParseException("Spec must be valid ASCII.", 0, 0) try: - return Request(Request.expr().parseString(s, parseAll=True)) + parts = pp.OneOrMore( + pp.Group( + Request.expr() + ) + ).parseString(s, parseAll=True) + return [Request(i) for i in parts] except pp.ParseException, v: raise ParseException(v.msg, v.line, v.col) -- cgit v1.2.3 From 609d6eab30aac0601106993eb8c5d9a953ba402b Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 25 Oct 2014 17:27:08 +1300 Subject: Make grammar less ambiguous for multi-pattern files --- libpathod/language.py | 4 ++-- 1 file changed, 2 insertions(+), 2 deletions(-) (limited to 'libpathod/language.py') diff --git a/libpathod/language.py b/libpathod/language.py index f40499f6..d8f201db 100644 --- a/libpathod/language.py +++ b/libpathod/language.py @@ -137,7 +137,7 @@ DATATYPES = dict( ) -v_integer = pp.Regex(r"\d+")\ +v_integer = pp.Word(pp.nums)\ .setName("integer")\ .setParseAction(lambda toks: int(toks[0])) @@ -308,7 +308,7 @@ class ValueGenerate(_Token): u = reduce( operator.or_, [pp.Literal(i) for i in utils.SIZE_UNITS.keys()] - ) + ).leaveWhitespace() e = e + pp.Optional(u, default=None) s = pp.Literal(",").suppress() -- cgit v1.2.3 From c00ae41486de06192865f8539da0f00985a16a90 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sat, 25 Oct 2014 19:50:48 +1300 Subject: Add a memoize argument to prevent playing the same pattern twice Also remove addition of Date header, which makes this non-deterministic --- libpathod/language.py | 14 -------------- 1 file changed, 14 deletions(-) (limited to 'libpathod/language.py') diff --git a/libpathod/language.py b/libpathod/language.py index d8f201db..56cbc18b 100644 --- a/libpathod/language.py +++ b/libpathod/language.py @@ -819,20 +819,6 @@ class _Message(object): ValueLiteral(request_host) ) ) - else: - if not utils.get_header("Date", self.headers): - tokens.append( - Header( - ValueLiteral("Date"), - ValueLiteral( - formatdate( - timeval=None, - localtime=False, - usegmt=True - ) - ) - ) - ) intermediate = self.__class__(tokens) return self.__class__([i.resolve(intermediate, settings) for i in tokens]) -- cgit v1.2.3 From fc1fc80469dca11ff0241c4b263e4b39e5506ddd Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 26 Oct 2014 10:50:32 +1300 Subject: Allow nesting of pathod response specs in pathoc specs This opens the door to really neat, repeatable, client-side driven fuzzing, especially of proxies. --- libpathod/language.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++-- 1 file changed, 46 insertions(+), 2 deletions(-) (limited to 'libpathod/language.py') diff --git a/libpathod/language.py b/libpathod/language.py index 56cbc18b..d8e87145 100644 --- a/libpathod/language.py +++ b/libpathod/language.py @@ -1,3 +1,4 @@ +from __future__ import print_function import operator import string import random @@ -6,7 +7,6 @@ import os import time import copy import abc -from email.utils import formatdate import contrib.pyparsing as pp from netlib import http_status, tcp, http_uastrings @@ -527,6 +527,43 @@ class Body(_Component): return Body(self.value.freeze(settings)) +class PathodSpec(_Token): + def __init__(self, value): + self.value = value + try: + self.parsed = Response( + Response.expr().parseString( + value.val, + parseAll=True + ) + ) + except pp.ParseException, v: + raise ParseException(v.msg, v.line, v.col) + + @classmethod + def expr(klass): + e = pp.Literal("s").suppress() + e = e + ValueLiteral.expr() + return e.setParseAction(lambda x: klass(*x)) + + def values(self, settings): + return [ + self.value.get_generator(settings), + ] + + def quote(self, s): + quotechar = s[0] + s = s[1:-1] + s = s.replace(quotechar, "\\" + quotechar) + return quotechar + s + quotechar + + def spec(self): + return "s%s"%(self.quote(self.value.spec())) + + def freeze(self, settings): + return PathodSpec(ValueLiteral(self.parsed.freeze(settings).spec())) + + class Path(_Component): def __init__(self, value): if isinstance(value, basestring): @@ -934,7 +971,8 @@ class Request(_Message): InjectAt, ShortcutContentType, ShortcutUserAgent, - Raw + Raw, + PathodSpec, ) logattrs = ["method", "path", "body"] @@ -946,10 +984,16 @@ class Request(_Message): def path(self): return self._get_token(Path) + @property + def pathodspec(self): + return self._get_token(PathodSpec) + def preamble(self, settings): v = self.method.values(settings) v.append(" ") v.extend(self.path.values(settings)) + if self.pathodspec: + v.append(self.pathodspec.parsed.spec()) v.append(" ") v.append(self.version) return v -- cgit v1.2.3 From 974bd9d0f9f3d231ff99496c55ae40355398cfc6 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 26 Oct 2014 12:56:28 +1300 Subject: Resolve a quoting ambiguity in nested response specs --- libpathod/language.py | 21 +++++++++++---------- 1 file changed, 11 insertions(+), 10 deletions(-) (limited to 'libpathod/language.py') diff --git a/libpathod/language.py b/libpathod/language.py index d8e87145..42fffcf8 100644 --- a/libpathod/language.py +++ b/libpathod/language.py @@ -1,4 +1,3 @@ -from __future__ import print_function import operator import string import random @@ -15,6 +14,12 @@ import utils BLOCKSIZE = 1024 TRUNCATE = 1024 +def quote(s): + quotechar = s[0] + s = s[1:-1] + s = s.replace(quotechar, "\\" + quotechar) + return quotechar + s + quotechar + class FileAccessDenied(Exception): pass @@ -272,7 +277,7 @@ class ValueLiteral(_ValueLiteral): return e.setParseAction(lambda x: klass(*x)) def spec(self): - return '"%s"'%self.val.encode("string_escape") + return quote("'%s'"%self.val.encode("string_escape")) class ValueNakedLiteral(_ValueLiteral): @@ -551,17 +556,13 @@ class PathodSpec(_Token): self.value.get_generator(settings), ] - def quote(self, s): - quotechar = s[0] - s = s[1:-1] - s = s.replace(quotechar, "\\" + quotechar) - return quotechar + s + quotechar - def spec(self): - return "s%s"%(self.quote(self.value.spec())) + return "s%s"%(self.value.spec()) def freeze(self, settings): - return PathodSpec(ValueLiteral(self.parsed.freeze(settings).spec())) + f = self.parsed.freeze(settings).spec() + print [f] + return PathodSpec(ValueLiteral(f)) class Path(_Component): -- cgit v1.2.3 From bd1f7ebb5c3cf3dfa613e194f4728bae1420b241 Mon Sep 17 00:00:00 2001 From: Aldo Cortesi Date: Sun, 26 Oct 2014 16:27:25 +1300 Subject: Improve netability of grammars --- libpathod/language.py | 22 ++++++++++++++++------ 1 file changed, 16 insertions(+), 6 deletions(-) (limited to 'libpathod/language.py') diff --git a/libpathod/language.py b/libpathod/language.py index 42fffcf8..e1f6820f 100644 --- a/libpathod/language.py +++ b/libpathod/language.py @@ -14,6 +14,11 @@ import utils BLOCKSIZE = 1024 TRUNCATE = 1024 + +def escape_backslash(s): + return s.replace("\\", "\\\\") + + def quote(s): quotechar = s[0] s = s[1:-1] @@ -186,7 +191,7 @@ class LiteralGenerator: return self.s.__getslice__(a, b) def __repr__(self): - return '"%s"'%self.s + return "'%s'"%self.s class RandomGenerator: @@ -274,10 +279,16 @@ class ValueLiteral(_ValueLiteral): @classmethod def expr(klass): e = v_literal.copy() - return e.setParseAction(lambda x: klass(*x)) + return e.setParseAction(klass.parseAction) + + @classmethod + def parseAction(klass, x): + v = klass(*x) + return v def spec(self): - return quote("'%s'"%self.val.encode("string_escape")) + ret = "'%s'"%self.val.encode("string_escape") + return ret class ValueNakedLiteral(_ValueLiteral): @@ -361,7 +372,7 @@ class ValueFile(_Token): return FileGenerator(s) def spec(self): - return '<"%s"'%self.path.encode("string_escape") + return "<'%s'"%self.path.encode("string_escape") Value = pp.MatchFirst( @@ -561,8 +572,7 @@ class PathodSpec(_Token): def freeze(self, settings): f = self.parsed.freeze(settings).spec() - print [f] - return PathodSpec(ValueLiteral(f)) + return PathodSpec(ValueLiteral(f.encode("string_escape"))) class Path(_Component): -- cgit v1.2.3