From 962a414327d93b604a59a4b8c8582d359745009d Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Fri, 24 Oct 2014 14:01:34 +1300
Subject: Fix tests, re-add scripts

---
 libpathod/main.py   | 10 ++++++++--
 libpathod/pathoc.py |  5 +++--
 libpathod/pathod.py | 10 ++++++++--
 libpathod/test.py   | 15 +++++++++++----
 4 files changed, 30 insertions(+), 10 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/main.py b/libpathod/main.py
index 6f53832d..ff407ff6 100644
--- a/libpathod/main.py
+++ b/libpathod/main.py
@@ -17,8 +17,14 @@ def pathoc():
             print "  ", i[1], i[0]
         sys.exit(0)
 
-    parser = argparse.ArgumentParser(description='A perverse HTTP client.', parents=[preparser])
-    parser.add_argument('--version', action='version', version="pathoc " + version.VERSION)
+    parser = argparse.ArgumentParser(
+        description='A perverse HTTP client.', parents=[preparser]
+    )
+    parser.add_argument(
+        '--version',
+        action='version',
+        version="pathoc " + version.VERSION
+    )
     parser.add_argument(
         "-c", dest="connect_to", type=str, default=False,
         metavar = "HOST:PORT",
diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py
index 938dbeba..ba831fc6 100644
--- a/libpathod/pathoc.py
+++ b/libpathod/pathoc.py
@@ -1,4 +1,5 @@
-import sys, os
+import sys
+import os
 from netlib import tcp, http, certutils
 import netlib.utils
 import language, utils
@@ -173,7 +174,7 @@ class Pathoc(tcp.TCPClient):
                         print >> fp, "%s=%s"%cn,
                     print >> fp
                     print >> fp, "\tVersion: %s"%i.get_version()
-                    print >> fp, "\tValidity: %s - %s"%(i.get_notBefore(),i.get_notAfter()) 
+                    print >> fp, "\tValidity: %s - %s"%(i.get_notBefore(),i.get_notAfter())
                     print >> fp, "\tSerial: %s"%i.get_serial_number()
                     print >> fp, "\tAlgorithm: %s"%i.get_signature_algorithm()
                     pk = i.get_pubkey()
diff --git a/libpathod/pathod.py b/libpathod/pathod.py
index 79fe7ed1..ecf1cc7e 100644
--- a/libpathod/pathod.py
+++ b/libpathod/pathod.py
@@ -1,6 +1,11 @@
-import urllib, threading, re, logging, os
+import urllib
+import threading
+import re
+import logging
+import os
 from netlib import tcp, http, wsgi, certutils
 import netlib.utils
+
 import version, app, language, utils
 
 
@@ -12,7 +17,8 @@ CA_CERT_NAME = "mitmproxy-ca.pem"
 logger = logging.getLogger('pathod')
 
 
-class PathodError(Exception): pass
+class PathodError(Exception):
+    pass
 
 
 class SSLOptions:
diff --git a/libpathod/test.py b/libpathod/test.py
index 57d40300..6f95a797 100644
--- a/libpathod/test.py
+++ b/libpathod/test.py
@@ -1,4 +1,5 @@
-import threading, Queue
+import threading
+import Queue
 import requests
 import requests.packages.urllib3
 import pathod
@@ -6,7 +7,6 @@ import pathod
 requests.packages.urllib3.disable_warnings()
 
 
-
 class Daemon:
     IFACE = "127.0.0.1"
     def __init__(self, ssl=None, **daemonargs):
@@ -14,7 +14,11 @@ class Daemon:
         self.thread = _PaThread(self.IFACE, self.q, ssl, daemonargs)
         self.thread.start()
         self.port = self.q.get(True, 5)
-        self.urlbase = "%s://%s:%s"%("https" if ssl else "http", self.IFACE, self.port)
+        self.urlbase = "%s://%s:%s"%(
+            "https" if ssl else "http",
+            self.IFACE,
+            self.port
+        )
 
     def __enter__(self):
         return self
@@ -80,6 +84,9 @@ class _PaThread(threading.Thread):
             ssl = self.ssl,
             **self.daemonargs
         )
-        self.name = "PathodThread (%s:%s)" % (self.server.address.host, self.server.address.port)
+        self.name = "PathodThread (%s:%s)" % (
+            self.server.address.host,
+            self.server.address.port
+        )
         self.q.put(self.server.address.port)
         self.server.serve_forever()
-- 
cgit v1.2.3


From cd481fdd14a96afeb61ce5d2b8d899f24af1051f Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Fri, 24 Oct 2014 17:12:54 +1300
Subject: Refactor and whitespace

---
 libpathod/cmdline.py | 314 +++++++++++++++++++++++++++++++++++++++
 libpathod/main.py    | 407 ---------------------------------------------------
 libpathod/pathoc.py  |  35 ++++-
 libpathod/pathod.py  |  89 +++++++++++
 4 files changed, 437 insertions(+), 408 deletions(-)
 create mode 100644 libpathod/cmdline.py
 delete mode 100644 libpathod/main.py

(limited to 'libpathod')

diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py
new file mode 100644
index 00000000..fa2c10cd
--- /dev/null
+++ b/libpathod/cmdline.py
@@ -0,0 +1,314 @@
+#!/usr/bin/env python
+import argparse
+import sys
+import os
+from . import pathoc, pathod, version
+from netlib import http_uastrings
+
+
+def go_pathoc():
+    preparser = argparse.ArgumentParser(add_help=False)
+    preparser.add_argument(
+        "--show-uas", dest="showua", action="store_true", default=False,
+        help="Print user agent shortcuts and exit."
+    )
+    pa = preparser.parse_known_args()[0]
+    if pa.showua:
+        print "User agent strings:"
+        for i in http_uastrings.UASTRINGS:
+            print "  ", i[1], i[0]
+        sys.exit(0)
+
+    parser = argparse.ArgumentParser(
+        description='A perverse HTTP client.', parents=[preparser]
+    )
+    parser.add_argument(
+        '--version',
+        action='version',
+        version="pathoc " + version.VERSION
+    )
+    parser.add_argument(
+        "-c", dest="connect_to", type=str, default=False,
+        metavar = "HOST:PORT",
+        help="Issue an HTTP CONNECT to connect to the specified host."
+    )
+    parser.add_argument(
+        "-n", dest='repeat', default=1, type=int, metavar="N",
+        help='Repeat requests N times'
+    )
+    parser.add_argument(
+        "-p", dest="port", type=int, default=None,
+        help="Port. Defaults to 80, or 443 if SSL is active"
+    )
+    parser.add_argument(
+        "-t", dest="timeout", type=int, default=None,
+        help="Connection timeout"
+    )
+    parser.add_argument(
+        'host', type=str,
+        help='Host to connect to'
+    )
+    parser.add_argument(
+        'request', type=str, nargs="+",
+        help='Request specification'
+    )
+
+    group = parser.add_argument_group(
+        'SSL',
+    )
+    group.add_argument(
+        "-s", dest="ssl", action="store_true", default=False,
+        help="Connect with SSL"
+    )
+    group.add_argument(
+        "-C", dest="clientcert", type=str, default=False,
+        help="Path to a file containing client certificate and private key"
+    )
+    group.add_argument(
+        "-i", dest="sni", type=str, default=False,
+        help="SSL Server Name Indication"
+    )
+    group.add_argument(
+        "--ciphers", dest="ciphers", type=str, default=False,
+        help="SSL cipher specification"
+    )
+    group.add_argument(
+        "--sslversion", dest="sslversion", type=int, default=4,
+        choices=[1, 2, 3, 4],
+        help="Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default to SSLv23."
+    )
+
+    group = parser.add_argument_group(
+        'Controlling Output',
+        """
+            Some of these options expand generated values for logging - if
+            you're generating large data, use them with caution.
+        """
+    )
+    group.add_argument(
+        "-I", dest="ignorecodes", type=str, default="",
+        help="Comma-separated list of response codes to ignore"
+    )
+    group.add_argument(
+        "-S", dest="showssl", action="store_true", default=False,
+        help="Show info on SSL connection"
+    )
+    group.add_argument(
+        "-e", dest="explain", action="store_true", default=False,
+        help="Explain requests"
+    )
+    group.add_argument(
+        "-o", dest="oneshot", action="store_true", default=False,
+        help="Oneshot - exit after first non-ignored response"
+    )
+    group.add_argument(
+        "-q", dest="showreq", action="store_true", default=False,
+        help="Print full request"
+    )
+    group.add_argument(
+        "-r", dest="showresp", action="store_true", default=False,
+        help="Print full response"
+    )
+    group.add_argument(
+        "-T", dest="ignoretimeout", action="store_true", default=False,
+        help="Ignore timeouts"
+    )
+    group.add_argument(
+        "-x", dest="hexdump", action="store_true", default=False,
+        help="Output in hexdump format"
+    )
+
+    args = parser.parse_args()
+
+    if args.port is None:
+        args.port = 443 if args.ssl else 80
+    else:
+        args.port = args.port
+
+    try:
+        args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i]
+    except ValueError:
+        parser.error("Invalid return code specification: %s"%args.ignorecodes)
+
+    if args.connect_to:
+        parts = args.connect_to.split(":")
+        if len(parts) != 2:
+            parser.error("Invalid CONNECT specification: %s"%args.connect_to)
+        try:
+            parts[1] = int(parts[1])
+        except ValueError:
+            parser.error("Invalid CONNECT specification: %s"%args.connect_to)
+        args.connect_to = parts
+    else:
+        args.connect_to = None
+    pathoc.main(args)
+
+
+def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
+    try:
+        pid = os.fork()
+        if pid > 0:
+            sys.exit(0)
+    except OSError, e:
+        sys.stderr.write("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror))
+        sys.exit(1)
+    os.chdir("/")
+    os.umask(0)
+    os.setsid()
+    try:
+        pid = os.fork()
+        if pid > 0:
+            sys.exit(0)
+    except OSError, e:
+        sys.stderr.write("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror))
+        sys.exit(1)
+    si = open(stdin, 'rb')
+    so = open(stdout, 'a+b')
+    se = open(stderr, 'a+b', 0)
+    os.dup2(si.fileno(), sys.stdin.fileno())
+    os.dup2(so.fileno(), sys.stdout.fileno())
+    os.dup2(se.fileno(), sys.stderr.fileno())
+
+
+def go_pathod():
+    parser = argparse.ArgumentParser(
+        description='A pathological HTTP/S daemon.'
+    )
+    parser.add_argument(
+        '--version',
+        action='version',
+        version="pathod " + version.VERSION
+    )
+    parser.add_argument(
+        "-p",
+        dest='port',
+        default=9999,
+        type=int,
+        help='Port. Specify 0 to pick an arbitrary empty port.'
+    )
+    parser.add_argument(
+        "-l",
+        dest='address',
+        default="127.0.0.1",
+        type=str,
+        help='Listening address.'
+    )
+    parser.add_argument(
+        "-a",
+        dest='anchors',
+        default=[],
+        type=str,
+        action="append",
+        metavar="ANCHOR",
+        help='Add an anchor. Specified as a string with the form pattern=pagespec'
+    )
+    parser.add_argument(
+        "-c", dest='craftanchor', default="/p/", type=str,
+        help='Anchorpoint for URL crafting commands.'
+    )
+    parser.add_argument(
+        "--confdir",
+        action="store", type = str, dest="confdir", default='~/.mitmproxy',
+        help = "Configuration directory. (~/.mitmproxy)"
+    )
+    parser.add_argument(
+        "-d", dest='staticdir', default=None, type=str,
+        help='Directory for static files.'
+    )
+    parser.add_argument(
+        "-D", dest='daemonize', default=False, action="store_true",
+        help='Daemonize.'
+    )
+    parser.add_argument(
+        "-t", dest="timeout", type=int, default=None,
+        help="Connection timeout"
+    )
+    parser.add_argument(
+        "--limit-size", dest='sizelimit', default=None, type=str,
+        help='Size limit of served responses. Understands size suffixes, i.e. 100k.'
+    )
+    parser.add_argument(
+        "--noapi", dest='noapi', default=False, action="store_true",
+        help='Disable API.'
+    )
+    parser.add_argument(
+        "--nohang", dest='nohang', default=False, action="store_true",
+        help='Disable pauses during crafted response generation.'
+    )
+    parser.add_argument(
+        "--noweb", dest='noweb', default=False, action="store_true",
+        help='Disable both web interface and API.'
+    )
+    parser.add_argument(
+        "--nocraft", dest='nocraft', default=False, action="store_true",
+        help='Disable response crafting. If anchors are specified, they still work.'
+    )
+
+    group = parser.add_argument_group(
+        'SSL',
+    )
+    group.add_argument(
+        "-s", dest='ssl', default=False, action="store_true",
+        help='Run in HTTPS mode.'
+    )
+    group.add_argument(
+        "--cn", dest="cn", type=str, default=None,
+        help="CN for generated SSL certs. Default: %s"%pathod.DEFAULT_CERT_DOMAIN
+    )
+    group.add_argument(
+        "-C", dest='ssl_not_after_connect', default=False, action="store_true",
+        help="Don't expect SSL after a CONNECT request."
+    )
+    group.add_argument(
+        "--cert", dest='ssl_certs', default=[], type=str,
+        metavar = "SPEC", action="append",
+        help = """
+        Add an SSL certificate. SPEC is of the form "[domain=]path". The domain
+        may include a wildcard, and is equal to "*" if not specified. The file
+        at path is a certificate in PEM format. If a private key is included in
+        the PEM, it is used, else the default key in the conf dir is used. Can
+        be passed multiple times.'
+        """
+    )
+    group.add_argument(
+        "--ciphers", dest="ciphers", type=str, default=False,
+        help="SSL cipher specification"
+    )
+    group.add_argument(
+        "--sslversion", dest="sslversion", type=int, default=4,
+        choices=[1, 2, 3, 4],
+        help="Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default to SSLv23."
+    )
+
+    group = parser.add_argument_group(
+        'Controlling Logging',
+        """
+            Some of these options expand generated values for logging - if
+            you're generating large data, use them with caution.
+        """
+    )
+    group.add_argument(
+        "-e", dest="explain", action="store_true", default=False,
+        help="Explain responses"
+    )
+    group.add_argument(
+        "-f", dest='logfile', default=None, type=str,
+        help='Log to file.'
+    )
+    group.add_argument(
+        "-q", dest="logreq", action="store_true", default=False,
+        help="Log full request"
+    )
+    group.add_argument(
+        "-r", dest="logresp", action="store_true", default=False,
+        help="Log full response"
+    )
+    group.add_argument(
+        "-x", dest="hexdump", action="store_true", default=False,
+        help="Log request/response in hexdump format"
+    )
+    args = parser.parse_args()
+    if args.daemonize:
+        daemonize()
+    pathod.main(parser, args)
+
diff --git a/libpathod/main.py b/libpathod/main.py
deleted file mode 100644
index ff407ff6..00000000
--- a/libpathod/main.py
+++ /dev/null
@@ -1,407 +0,0 @@
-#!/usr/bin/env python
-import argparse, sys, logging, logging.handlers, os
-from . import pathoc as _pathoc, pathod as _pathod, utils, version, language
-from netlib import tcp, http_uastrings
-
-
-def pathoc():
-    preparser = argparse.ArgumentParser(add_help=False)
-    preparser.add_argument(
-        "--show-uas", dest="showua", action="store_true", default=False,
-        help="Print user agent shortcuts and exit."
-    )
-    pa = preparser.parse_known_args()[0]
-    if pa.showua:
-        print "User agent strings:"
-        for i in http_uastrings.UASTRINGS:
-            print "  ", i[1], i[0]
-        sys.exit(0)
-
-    parser = argparse.ArgumentParser(
-        description='A perverse HTTP client.', parents=[preparser]
-    )
-    parser.add_argument(
-        '--version',
-        action='version',
-        version="pathoc " + version.VERSION
-    )
-    parser.add_argument(
-        "-c", dest="connect_to", type=str, default=False,
-        metavar = "HOST:PORT",
-        help="Issue an HTTP CONNECT to connect to the specified host."
-    )
-    parser.add_argument(
-        "-n", dest='repeat', default=1, type=int, metavar="N",
-        help='Repeat requests N times'
-    )
-    parser.add_argument(
-        "-p", dest="port", type=int, default=None,
-        help="Port. Defaults to 80, or 443 if SSL is active"
-    )
-    parser.add_argument(
-        "-t", dest="timeout", type=int, default=None,
-        help="Connection timeout"
-    )
-    parser.add_argument(
-        'host', type=str,
-        help='Host to connect to'
-    )
-    parser.add_argument(
-        'request', type=str, nargs="+",
-        help='Request specification'
-    )
-
-    group = parser.add_argument_group(
-        'SSL',
-    )
-    group.add_argument(
-        "-s", dest="ssl", action="store_true", default=False,
-        help="Connect with SSL"
-    )
-    group.add_argument(
-        "-C", dest="clientcert", type=str, default=False,
-        help="Path to a file containing client certificate and private key"
-    )
-    group.add_argument(
-        "-i", dest="sni", type=str, default=False,
-        help="SSL Server Name Indication"
-    )
-    group.add_argument(
-        "--ciphers", dest="ciphers", type=str, default=False,
-        help="SSL cipher specification"
-    )
-    group.add_argument(
-        "--sslversion", dest="sslversion", type=int, default=4,
-        choices=[1, 2, 3, 4],
-        help="Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default to SSLv23."
-    )
-
-    group = parser.add_argument_group(
-        'Controlling Output',
-        """
-            Some of these options expand generated values for logging - if
-            you're generating large data, use them with caution.
-        """
-    )
-    group.add_argument(
-        "-I", dest="ignorecodes", type=str, default="",
-        help="Comma-separated list of response codes to ignore"
-    )
-    group.add_argument(
-        "-S", dest="showssl", action="store_true", default=False,
-        help="Show info on SSL connection"
-    )
-    group.add_argument(
-        "-e", dest="explain", action="store_true", default=False,
-        help="Explain requests"
-    )
-    group.add_argument(
-        "-o", dest="oneshot", action="store_true", default=False,
-        help="Oneshot - exit after first non-ignored response"
-    )
-    group.add_argument(
-        "-q", dest="showreq", action="store_true", default=False,
-        help="Print full request"
-    )
-    group.add_argument(
-        "-r", dest="showresp", action="store_true", default=False,
-        help="Print full response"
-    )
-    group.add_argument(
-        "-T", dest="ignoretimeout", action="store_true", default=False,
-        help="Ignore timeouts"
-    )
-    group.add_argument(
-        "-x", dest="hexdump", action="store_true", default=False,
-        help="Output in hexdump format"
-    )
-
-    args = parser.parse_args()
-
-    if args.port is None:
-        port = 443 if args.ssl else 80
-    else:
-        port = args.port
-
-    try:
-        codes = [int(i) for i in args.ignorecodes.split(",") if i]
-    except ValueError:
-        parser.error("Invalid return code specification: %s"%args.ignorecodes)
-
-    if args.connect_to:
-        parts = args.connect_to.split(":")
-        if len(parts) != 2:
-            parser.error("Invalid CONNECT specification: %s"%args.connect_to)
-        try:
-            parts[1] = int(parts[1])
-        except ValueError:
-            parser.error("Invalid CONNECT specification: %s"%args.connect_to)
-        connect_to = parts
-    else:
-        connect_to = None
-
-    try:
-        for i in range(args.repeat):
-            p = _pathoc.Pathoc(
-                (args.host, port),
-                ssl=args.ssl,
-                sni=args.sni,
-                sslversion=args.sslversion,
-                clientcert=args.clientcert,
-                ciphers=args.ciphers
-            )
-            try:
-                p.connect(connect_to)
-            except (tcp.NetLibError, _pathoc.PathocError), v:
-                print >> sys.stderr, str(v)
-                sys.exit(1)
-            if args.timeout:
-                p.settimeout(args.timeout)
-            for spec in args.request:
-                ret = p.print_request(
-                    spec,
-                    showreq=args.showreq,
-                    showresp=args.showresp,
-                    explain=args.explain,
-                    showssl=args.showssl,
-                    hexdump=args.hexdump,
-                    ignorecodes=codes,
-                    ignoretimeout=args.ignoretimeout
-                )
-                sys.stdout.flush()
-                if ret and args.oneshot:
-                    sys.exit(0)
-    except KeyboardInterrupt:
-        pass
-
-
-def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
-    try:
-        pid = os.fork()
-        if pid > 0:
-            sys.exit(0)
-    except OSError, e:
-        sys.stderr.write("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror))
-        sys.exit(1)
-    os.chdir("/")
-    os.umask(0)
-    os.setsid()
-    try:
-        pid = os.fork()
-        if pid > 0:
-            sys.exit(0)
-    except OSError, e:
-        sys.stderr.write("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror))
-        sys.exit(1)
-    si = open(stdin, 'rb')
-    so = open(stdout, 'a+b')
-    se = open(stderr, 'a+b', 0)
-    os.dup2(si.fileno(), sys.stdin.fileno())
-    os.dup2(so.fileno(), sys.stdout.fileno())
-    os.dup2(se.fileno(), sys.stderr.fileno())
-
-
-def pathod_main(parser, args):
-    certs = []
-    for i in args.ssl_certs:
-        parts = i.split("=", 1)
-        if len(parts) == 1:
-            parts = ["*", parts[0]]
-        parts[1] = os.path.expanduser(parts[1])
-        if not os.path.exists(parts[1]):
-            parser.error("Certificate file does not exist: %s"%parts[1])
-        certs.append(parts)
-
-    ssloptions = _pathod.SSLOptions(
-        cn = args.cn,
-        confdir = args.confdir,
-        not_after_connect = args.ssl_not_after_connect,
-        ciphers = args.ciphers,
-        sslversion = utils.SSLVERSIONS[args.sslversion],
-        certs = certs
-    )
-
-    alst = []
-    for i in args.anchors:
-        parts = utils.parse_anchor_spec(i)
-        if not parts:
-            parser.error("Invalid anchor specification: %s"%i)
-        alst.append(parts)
-
-    root = logging.getLogger()
-    if root.handlers:
-        for handler in root.handlers:
-            root.removeHandler(handler)
-
-    log = logging.getLogger('pathod')
-    log.setLevel(logging.DEBUG)
-    fmt = logging.Formatter(
-        '%(asctime)s: %(message)s',
-        datefmt='%d-%m-%y %H:%M:%S',
-    )
-    if args.logfile:
-        fh = logging.handlers.WatchedFileHandler(args.logfile)
-        fh.setFormatter(fmt)
-        log.addHandler(fh)
-    if not args.daemonize:
-        sh = logging.StreamHandler()
-        sh.setFormatter(fmt)
-        log.addHandler(sh)
-
-    sizelimit = None
-    if args.sizelimit:
-        try:
-            sizelimit = utils.parse_size(args.sizelimit)
-        except ValueError, v:
-            parser.error(v)
-
-    try:
-        pd = _pathod.Pathod(
-            (args.address, args.port),
-            craftanchor = args.craftanchor,
-            ssl = args.ssl,
-            ssloptions = ssloptions,
-            staticdir = args.staticdir,
-            anchors = alst,
-            sizelimit = sizelimit,
-            noweb = args.noweb,
-            nocraft = args.nocraft,
-            noapi = args.noapi,
-            nohang = args.nohang,
-            timeout = args.timeout,
-            logreq = args.logreq,
-            logresp = args.logresp,
-            hexdump = args.hexdump,
-            explain = args.explain,
-        )
-    except _pathod.PathodError, v:
-        parser.error(str(v))
-    except language.FileAccessDenied, v:
-        parser.error("%s You probably want to a -d argument."%str(v))
-
-    try:
-        print "%s listening on %s:%s"%(version.NAMEVERSION, pd.address.host, pd.address.port)
-        pd.serve_forever()
-    except KeyboardInterrupt:
-        pass
-
-
-def pathod():
-    parser = argparse.ArgumentParser(description='A pathological HTTP/S daemon.')
-    parser.add_argument('--version', action='version', version="pathod " + version.VERSION)
-    parser.add_argument("-p", dest='port', default=9999, type=int, help='Port. Specify 0 to pick an arbitrary empty port.')
-    parser.add_argument("-l", dest='address', default="127.0.0.1", type=str, help='Listening address.')
-    parser.add_argument(
-        "-a", dest='anchors', default=[], type=str, action="append", metavar="ANCHOR",
-        help='Add an anchor. Specified as a string with the form pattern=pagespec'
-    )
-    parser.add_argument(
-        "-c", dest='craftanchor', default="/p/", type=str,
-        help='Anchorpoint for URL crafting commands.'
-    )
-    parser.add_argument(
-        "--confdir",
-        action="store", type = str, dest="confdir", default='~/.mitmproxy',
-        help = "Configuration directory. (~/.mitmproxy)"
-    )
-    parser.add_argument(
-        "-d", dest='staticdir', default=None, type=str,
-        help='Directory for static files.'
-    )
-    parser.add_argument(
-        "-D", dest='daemonize', default=False, action="store_true",
-        help='Daemonize.'
-    )
-    parser.add_argument(
-        "-t", dest="timeout", type=int, default=None,
-        help="Connection timeout"
-    )
-    parser.add_argument(
-        "--limit-size", dest='sizelimit', default=None, type=str,
-        help='Size limit of served responses. Understands size suffixes, i.e. 100k.'
-    )
-    parser.add_argument(
-        "--noapi", dest='noapi', default=False, action="store_true",
-        help='Disable API.'
-    )
-    parser.add_argument(
-        "--nohang", dest='nohang', default=False, action="store_true",
-        help='Disable pauses during crafted response generation.'
-    )
-    parser.add_argument(
-        "--noweb", dest='noweb', default=False, action="store_true",
-        help='Disable both web interface and API.'
-    )
-    parser.add_argument(
-        "--nocraft", dest='nocraft', default=False, action="store_true",
-        help='Disable response crafting. If anchors are specified, they still work.'
-    )
-
-    group = parser.add_argument_group(
-        'SSL',
-    )
-    group.add_argument(
-        "-s", dest='ssl', default=False, action="store_true",
-        help='Run in HTTPS mode.'
-    )
-    group.add_argument(
-        "--cn", dest="cn", type=str, default=None,
-        help="CN for generated SSL certs. Default: %s"%_pathod.DEFAULT_CERT_DOMAIN
-    )
-    group.add_argument(
-        "-C", dest='ssl_not_after_connect', default=False, action="store_true",
-        help="Don't expect SSL after a CONNECT request."
-    )
-    group.add_argument(
-        "--cert", dest='ssl_certs', default=[], type=str,
-        metavar = "SPEC", action="append",
-        help='Add an SSL certificate. SPEC is of the form "[domain=]path". '\
-             'The domain may include a wildcard, and is equal to "*" if not specified. '\
-             'The file at path is a certificate in PEM format. If a private key is included in the PEM, '\
-             'it is used, else the default key in the conf dir is used. Can be passed multiple times.'
-    )
-    group.add_argument(
-        "--ciphers", dest="ciphers", type=str, default=False,
-        help="SSL cipher specification"
-    )
-    group.add_argument(
-        "--sslversion", dest="sslversion", type=int, default=4,
-        choices=[1, 2, 3, 4],
-        help="Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default to SSLv23."
-    )
-
-    group = parser.add_argument_group(
-        'Controlling Logging',
-        """
-            Some of these options expand generated values for logging - if
-            you're generating large data, use them with caution.
-        """
-    )
-    group.add_argument(
-        "-e", dest="explain", action="store_true", default=False,
-        help="Explain responses"
-    )
-    group.add_argument(
-        "-f", dest='logfile', default=None, type=str,
-        help='Log to file.'
-    )
-    group.add_argument(
-        "-q", dest="logreq", action="store_true", default=False,
-        help="Log full request"
-    )
-    group.add_argument(
-        "-r", dest="logresp", action="store_true", default=False,
-        help="Log full response"
-    )
-    group.add_argument(
-        "-x", dest="hexdump", action="store_true", default=False,
-        help="Log request/response in hexdump format"
-    )
-    args = parser.parse_args()
-    if args.daemonize:
-        daemonize()
-    pathod_main(parser, args)
-
-
-if __name__ == "__main__":
-    pathoc()
\ No newline at end of file
diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py
index ba831fc6..9ff03eca 100644
--- a/libpathod/pathoc.py
+++ b/libpathod/pathoc.py
@@ -191,4 +191,37 @@ class Pathoc(tcp.TCPClient):
             return True
 
 
-
+def main(args):
+    try:
+        for i in range(args.repeat):
+            p = Pathoc(
+                (args.host, args.port),
+                ssl=args.ssl,
+                sni=args.sni,
+                sslversion=args.sslversion,
+                clientcert=args.clientcert,
+                ciphers=args.ciphers
+            )
+            try:
+                p.connect(args.connect_to)
+            except (tcp.NetLibError, PathocError), v:
+                print >> sys.stderr, str(v)
+                sys.exit(1)
+            if args.timeout:
+                p.settimeout(args.timeout)
+            for spec in args.request:
+                ret = p.print_request(
+                    spec,
+                    showreq=args.showreq,
+                    showresp=args.showresp,
+                    explain=args.explain,
+                    showssl=args.showssl,
+                    hexdump=args.hexdump,
+                    ignorecodes=args.ignorecodes,
+                    ignoretimeout=args.ignoretimeout
+                )
+                sys.stdout.flush()
+                if ret and args.oneshot:
+                    sys.exit(0)
+    except KeyboardInterrupt:
+        pass
diff --git a/libpathod/pathod.py b/libpathod/pathod.py
index ecf1cc7e..25f072fa 100644
--- a/libpathod/pathod.py
+++ b/libpathod/pathod.py
@@ -369,3 +369,92 @@ class Pathod(tcp.TCPServer):
 
     def get_log(self):
         return self.log
+
+
+def main(parser, args):
+    certs = []
+    for i in args.ssl_certs:
+        parts = i.split("=", 1)
+        if len(parts) == 1:
+            parts = ["*", parts[0]]
+        parts[1] = os.path.expanduser(parts[1])
+        if not os.path.exists(parts[1]):
+            parser.error("Certificate file does not exist: %s"%parts[1])
+        certs.append(parts)
+
+    ssloptions = SSLOptions(
+        cn = args.cn,
+        confdir = args.confdir,
+        not_after_connect = args.ssl_not_after_connect,
+        ciphers = args.ciphers,
+        sslversion = utils.SSLVERSIONS[args.sslversion],
+        certs = certs
+    )
+
+    alst = []
+    for i in args.anchors:
+        parts = utils.parse_anchor_spec(i)
+        if not parts:
+            parser.error("Invalid anchor specification: %s"%i)
+        alst.append(parts)
+
+    root = logging.getLogger()
+    if root.handlers:
+        for handler in root.handlers:
+            root.removeHandler(handler)
+
+    log = logging.getLogger('pathod')
+    log.setLevel(logging.DEBUG)
+    fmt = logging.Formatter(
+        '%(asctime)s: %(message)s',
+        datefmt='%d-%m-%y %H:%M:%S',
+    )
+    if args.logfile:
+        fh = logging.handlers.WatchedFileHandler(args.logfile)
+        fh.setFormatter(fmt)
+        log.addHandler(fh)
+    if not args.daemonize:
+        sh = logging.StreamHandler()
+        sh.setFormatter(fmt)
+        log.addHandler(sh)
+
+    sizelimit = None
+    if args.sizelimit:
+        try:
+            sizelimit = utils.parse_size(args.sizelimit)
+        except ValueError, v:
+            parser.error(v)
+
+    try:
+        pd = Pathod(
+            (args.address, args.port),
+            craftanchor = args.craftanchor,
+            ssl = args.ssl,
+            ssloptions = ssloptions,
+            staticdir = args.staticdir,
+            anchors = alst,
+            sizelimit = sizelimit,
+            noweb = args.noweb,
+            nocraft = args.nocraft,
+            noapi = args.noapi,
+            nohang = args.nohang,
+            timeout = args.timeout,
+            logreq = args.logreq,
+            logresp = args.logresp,
+            hexdump = args.hexdump,
+            explain = args.explain,
+        )
+    except PathodError, v:
+        parser.error(str(v))
+    except language.FileAccessDenied, v:
+        parser.error("%s You probably want to a -d argument."%str(v))
+
+    try:
+        print "%s listening on %s:%s"%(
+            version.NAMEVERSION,
+            pd.address.host,
+            pd.address.port
+        )
+        pd.serve_forever()
+    except KeyboardInterrupt:
+        pass
-- 
cgit v1.2.3


From c6c05098360c32dca82621cc4aaf91064f81e3f3 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Fri, 24 Oct 2014 17:19:55 +1300
Subject: Strictly separate argument parsing from main loops

---
 libpathod/cmdline.py | 60 ++++++++++++++++++++++++++--------------------------
 libpathod/pathod.py  | 41 ++++++++++-------------------------
 libpathod/utils.py   | 27 +++++++++++++++++++++++
 3 files changed, 68 insertions(+), 60 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py
index fa2c10cd..6d1573f1 100644
--- a/libpathod/cmdline.py
+++ b/libpathod/cmdline.py
@@ -2,7 +2,7 @@
 import argparse
 import sys
 import os
-from . import pathoc, pathod, version
+from . import pathoc, pathod, version, utils
 from netlib import http_uastrings
 
 
@@ -144,32 +144,6 @@ def go_pathoc():
     pathoc.main(args)
 
 
-def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
-    try:
-        pid = os.fork()
-        if pid > 0:
-            sys.exit(0)
-    except OSError, e:
-        sys.stderr.write("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror))
-        sys.exit(1)
-    os.chdir("/")
-    os.umask(0)
-    os.setsid()
-    try:
-        pid = os.fork()
-        if pid > 0:
-            sys.exit(0)
-    except OSError, e:
-        sys.stderr.write("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror))
-        sys.exit(1)
-    si = open(stdin, 'rb')
-    so = open(stdout, 'a+b')
-    se = open(stderr, 'a+b', 0)
-    os.dup2(si.fileno(), sys.stdin.fileno())
-    os.dup2(so.fileno(), sys.stdout.fileno())
-    os.dup2(se.fileno(), sys.stderr.fileno())
-
-
 def go_pathod():
     parser = argparse.ArgumentParser(
         description='A pathological HTTP/S daemon.'
@@ -308,7 +282,33 @@ def go_pathod():
         help="Log request/response in hexdump format"
     )
     args = parser.parse_args()
-    if args.daemonize:
-        daemonize()
-    pathod.main(parser, args)
+
+    certs = []
+    for i in args.ssl_certs:
+        parts = i.split("=", 1)
+        if len(parts) == 1:
+            parts = ["*", parts[0]]
+        parts[1] = os.path.expanduser(parts[1])
+        if not os.path.exists(parts[1]):
+            parser.error("Certificate file does not exist: %s"%parts[1])
+        certs.append(parts)
+    args.ssl_certs = certs
+
+    alst = []
+    for i in args.anchors:
+        parts = utils.parse_anchor_spec(i)
+        if not parts:
+            parser.error("Invalid anchor specification: %s"%i)
+        alst.append(parts)
+    args.anchors = alst
+
+    sizelimit = None
+    if args.sizelimit:
+        try:
+            sizelimit = utils.parse_size(args.sizelimit)
+        except ValueError, v:
+            parser.error(v)
+    args.sizelimit = sizelimit
+
+    pathod.main(args)
 
diff --git a/libpathod/pathod.py b/libpathod/pathod.py
index 25f072fa..37b07bb6 100644
--- a/libpathod/pathod.py
+++ b/libpathod/pathod.py
@@ -3,6 +3,7 @@ import threading
 import re
 import logging
 import os
+import sys
 from netlib import tcp, http, wsgi, certutils
 import netlib.utils
 
@@ -371,33 +372,16 @@ class Pathod(tcp.TCPServer):
         return self.log
 
 
-def main(parser, args):
-    certs = []
-    for i in args.ssl_certs:
-        parts = i.split("=", 1)
-        if len(parts) == 1:
-            parts = ["*", parts[0]]
-        parts[1] = os.path.expanduser(parts[1])
-        if not os.path.exists(parts[1]):
-            parser.error("Certificate file does not exist: %s"%parts[1])
-        certs.append(parts)
-
+def main(args):
     ssloptions = SSLOptions(
         cn = args.cn,
         confdir = args.confdir,
         not_after_connect = args.ssl_not_after_connect,
         ciphers = args.ciphers,
         sslversion = utils.SSLVERSIONS[args.sslversion],
-        certs = certs
+        certs = args.ssl_certs
     )
 
-    alst = []
-    for i in args.anchors:
-        parts = utils.parse_anchor_spec(i)
-        if not parts:
-            parser.error("Invalid anchor specification: %s"%i)
-        alst.append(parts)
-
     root = logging.getLogger()
     if root.handlers:
         for handler in root.handlers:
@@ -418,13 +402,6 @@ def main(parser, args):
         sh.setFormatter(fmt)
         log.addHandler(sh)
 
-    sizelimit = None
-    if args.sizelimit:
-        try:
-            sizelimit = utils.parse_size(args.sizelimit)
-        except ValueError, v:
-            parser.error(v)
-
     try:
         pd = Pathod(
             (args.address, args.port),
@@ -432,8 +409,8 @@ def main(parser, args):
             ssl = args.ssl,
             ssloptions = ssloptions,
             staticdir = args.staticdir,
-            anchors = alst,
-            sizelimit = sizelimit,
+            anchors = args.anchors,
+            sizelimit = args.sizelimit,
             noweb = args.noweb,
             nocraft = args.nocraft,
             noapi = args.noapi,
@@ -445,9 +422,13 @@ def main(parser, args):
             explain = args.explain,
         )
     except PathodError, v:
-        parser.error(str(v))
+        print >> sys.stderr, "Error: %s"%v
+        sys.exit(1)
     except language.FileAccessDenied, v:
-        parser.error("%s You probably want to a -d argument."%str(v))
+        print >> sys.stderr, "Error: %s"%v
+
+    if args.daemonize:
+        utils.daemonize()
 
     try:
         print "%s listening on %s:%s"%(
diff --git a/libpathod/utils.py b/libpathod/utils.py
index 110a7170..be49daec 100644
--- a/libpathod/utils.py
+++ b/libpathod/utils.py
@@ -110,3 +110,30 @@ class Data:
 
 
 data = Data(__name__)
+
+def daemonize(stdin='/dev/null', stdout='/dev/null', stderr='/dev/null'):
+    try:
+        pid = os.fork()
+        if pid > 0:
+            sys.exit(0)
+    except OSError, e:
+        sys.stderr.write("fork #1 failed: (%d) %s\n" % (e.errno, e.strerror))
+        sys.exit(1)
+    os.chdir("/")
+    os.umask(0)
+    os.setsid()
+    try:
+        pid = os.fork()
+        if pid > 0:
+            sys.exit(0)
+    except OSError, e:
+        sys.stderr.write("fork #2 failed: (%d) %s\n" % (e.errno, e.strerror))
+        sys.exit(1)
+    si = open(stdin, 'rb')
+    so = open(stdout, 'a+b')
+    se = open(stderr, 'a+b', 0)
+    os.dup2(si.fileno(), sys.stdin.fileno())
+    os.dup2(so.fileno(), sys.stdout.fileno())
+    os.dup2(se.fileno(), sys.stderr.fileno())
+
+
-- 
cgit v1.2.3


From 3de68da3adf15f445e40412fdde4b94857640166 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Fri, 24 Oct 2014 17:21:28 +1300
Subject: Add forgotten import. Oops.

---
 libpathod/utils.py | 1 +
 1 file changed, 1 insertion(+)

(limited to 'libpathod')

diff --git a/libpathod/utils.py b/libpathod/utils.py
index be49daec..d4160a23 100644
--- a/libpathod/utils.py
+++ b/libpathod/utils.py
@@ -1,4 +1,5 @@
 import os
+import sys
 from netlib import tcp
 
 SSLVERSIONS = {
-- 
cgit v1.2.3


From 5aadf92767614b7bd8e2ef677085410ac359e5e8 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 25 Oct 2014 08:18:39 +1300
Subject: Nicer way to specify patterns read for file - just use a path

---
 libpathod/app.py                   |  4 ++--
 libpathod/cmdline.py               | 33 +++++++++++++++++++++++++++++----
 libpathod/language.py              |  8 ++------
 libpathod/pathoc.py                |  4 ++--
 libpathod/pathod.py                |  6 +++---
 libpathod/templates/docs_lang.html |  8 --------
 6 files changed, 38 insertions(+), 25 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/app.py b/libpathod/app.py
index fb1d6a2d..1a54f712 100644
--- a/libpathod/app.py
+++ b/libpathod/app.py
@@ -121,9 +121,9 @@ def make_app(noapi):
 
         try:
             if is_request:
-                r = language.parse_request(app.config["pathod"].request_settings, spec)
+                r = language.parse_request(spec)
             else:
-                r = language.parse_response(app.config["pathod"].request_settings, spec)
+                r = language.parse_response(spec)
         except language.ParseException, v:
             args["syntaxerror"] = str(v)
             args["marked"] = v.marked()
diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py
index 6d1573f1..2c6e094e 100644
--- a/libpathod/cmdline.py
+++ b/libpathod/cmdline.py
@@ -1,7 +1,8 @@
 #!/usr/bin/env python
 import argparse
-import sys
 import os
+import os.path
+import sys
 from . import pathoc, pathod, version, utils
 from netlib import http_uastrings
 
@@ -50,9 +51,11 @@ def go_pathoc():
     )
     parser.add_argument(
         'request', type=str, nargs="+",
-        help='Request specification'
+        help="""
+        Request specification, or path to a file containing a request
+        specifcation
+        """
     )
-
     group = parser.add_argument_group(
         'SSL',
     )
@@ -141,6 +144,16 @@ def go_pathoc():
         args.connect_to = parts
     else:
         args.connect_to = None
+
+    reqs = []
+    for r in args.request:
+        if os.path.exists(r):
+            data = open(r).read()
+            reqs.append(data)
+        else:
+            reqs.append(r)
+    args.request = reqs
+
     pathoc.main(args)
 
 
@@ -174,7 +187,10 @@ def go_pathod():
         type=str,
         action="append",
         metavar="ANCHOR",
-        help='Add an anchor. Specified as a string with the form pattern=pagespec'
+        help="""
+        Add an anchor. Specified as a string with the form pattern=pagespec, or
+        pattern=filepath
+        """
     )
     parser.add_argument(
         "-c", dest='craftanchor', default="/p/", type=str,
@@ -310,5 +326,14 @@ def go_pathod():
             parser.error(v)
     args.sizelimit = sizelimit
 
+    anchors = []
+    for patt, spec in anchors:
+        if os.path.exists(spec):
+            data = open(spec).read()
+            anchors.append((patt, data))
+        else:
+            anchors.append((patt, spec))
+    args.anchors = anchors
+
     pathod.main(args)
 
diff --git a/libpathod/language.py b/libpathod/language.py
index 286a1a8e..002c8205 100644
--- a/libpathod/language.py
+++ b/libpathod/language.py
@@ -961,7 +961,7 @@ def read_file(settings, s):
     return file(s, "rb").read()
 
 
-def parse_response(settings, s):
+def parse_response(s):
     """
         May raise ParseException or FileAccessDenied
     """
@@ -969,15 +969,13 @@ def parse_response(settings, s):
         s = s.decode("ascii")
     except UnicodeError:
         raise ParseException("Spec must be valid ASCII.", 0, 0)
-    if s.startswith(FILESTART):
-        s = read_file(settings, s)
     try:
         return Response(Response.expr().parseString(s, parseAll=True))
     except pp.ParseException, v:
         raise ParseException(v.msg, v.line, v.col)
 
 
-def parse_request(settings, s):
+def parse_request(s):
     """
         May raise ParseException or FileAccessDenied
     """
@@ -985,8 +983,6 @@ def parse_request(settings, s):
         s = s.decode("ascii")
     except UnicodeError:
         raise ParseException("Spec must be valid ASCII.", 0, 0)
-    if s.startswith(FILESTART):
-        s = read_file(settings, s)
     try:
         return Request(Request.expr().parseString(s, parseAll=True))
     except pp.ParseException, v:
diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py
index 9ff03eca..e534bba5 100644
--- a/libpathod/pathoc.py
+++ b/libpathod/pathoc.py
@@ -80,7 +80,7 @@ class Pathoc(tcp.TCPClient):
             May raise language.ParseException, netlib.http.HttpError or
             language.FileAccessDenied.
         """
-        r = language.parse_request(self.settings, spec)
+        r = language.parse_request(spec)
         language.serve(r, self.wfile, self.settings, self.address.host)
         self.wfile.flush()
         ret = list(http.read_response(self.rfile, r.method.string(), None))
@@ -115,7 +115,7 @@ class Pathoc(tcp.TCPClient):
             Returns True if we have a non-ignored response.
         """
         try:
-            r = language.parse_request(self.settings, spec)
+            r = language.parse_request(spec)
         except language.ParseException, v:
             print >> fp, "Error parsing request spec: %s"%v.msg
             print >> fp, v.marked()
diff --git a/libpathod/pathod.py b/libpathod/pathod.py
index 37b07bb6..92e5b2db 100644
--- a/libpathod/pathod.py
+++ b/libpathod/pathod.py
@@ -169,7 +169,7 @@ class PathodHandler(tcp.BaseHandler):
         for i in self.server.anchors:
             if i[0].match(path):
                 self.info("crafting anchor: %s" % path)
-                aresp = language.parse_response(self.server.request_settings, i[1])
+                aresp = language.parse_response(i[1])
                 again, retlog["response"] = self.serve_crafted(aresp)
                 return again, retlog
 
@@ -177,7 +177,7 @@ class PathodHandler(tcp.BaseHandler):
             spec = urllib.unquote(path)[len(self.server.craftanchor):]
             self.info("crafting spec: %s" % spec)
             try:
-                crafted = language.parse_response(self.server.request_settings, spec)
+                crafted = language.parse_response(spec)
             except language.ParseException, v:
                 self.info("Parse error: %s" % v.msg)
                 crafted = language.make_error_response(
@@ -299,7 +299,7 @@ class Pathod(tcp.TCPServer):
                 except re.error:
                     raise PathodError("Invalid regex in anchor: %s" % i[0])
                 try:
-                    language.parse_response(self.request_settings, i[1])
+                    language.parse_response(i[1])
                 except language.ParseException, v:
                     raise PathodError("Invalid page spec in anchor: '%s', %s" % (i[1], str(v)))
                 self.anchors.append((arex, i[1]))
diff --git a/libpathod/templates/docs_lang.html b/libpathod/templates/docs_lang.html
index aef12a8d..7cb3fc5f 100644
--- a/libpathod/templates/docs_lang.html
+++ b/libpathod/templates/docs_lang.html
@@ -183,14 +183,6 @@
 </div>
 
 
-<section id="specifying_requests">
-    <div class="page-header">
-       <h1>Executing specs from file</h1>
-    </div>
-
-        <pre class="example">+./path/to/spec</pre>
-
-</section>
 
 <section id="specifying_requests">
     <div class="page-header">
-- 
cgit v1.2.3


From d6ee5327112182202513ff6ce62d95df1567fdb6 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 25 Oct 2014 14:24:05 +1300
Subject: Whitespace and formatting

---
 libpathod/language.py | 137 +++++++++++++++++++++++++++++++++++---------------
 1 file changed, 96 insertions(+), 41 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/language.py b/libpathod/language.py
index 002c8205..2969055b 100644
--- a/libpathod/language.py
+++ b/libpathod/language.py
@@ -1,4 +1,10 @@
-import operator, string, random, mmap, os, time, copy
+import operator
+import string
+import random
+import mmap
+import os
+import time
+import copy
 import abc
 from email.utils import formatdate
 import contrib.pyparsing as pp
@@ -9,7 +15,9 @@ import utils
 BLOCKSIZE = 1024
 TRUNCATE = 1024
 
-class FileAccessDenied(Exception): pass
+
+class FileAccessDenied(Exception):
+    pass
 
 
 class ParseException(Exception):
@@ -20,7 +28,7 @@ class ParseException(Exception):
         self.col = col
 
     def marked(self):
-        return "%s\n%s"%(self.s, " "*(self.col-1) + "^")
+        return "%s\n%s"%(self.s, " "*(self.col - 1) + "^")
 
     def __str__(self):
         return "%s at char %s"%(self.msg, self.col)
@@ -40,7 +48,9 @@ def send_chunk(fp, val, blocksize, start, end):
 def write_values(fp, vals, actions, sofar=0, skip=0, blocksize=BLOCKSIZE):
     """
         vals: A list of values, which may be strings or Value objects.
-        actions: A list of (offset, action, arg) tuples. Action may be "pause" or "disconnect".
+
+        actions: A list of (offset, action, arg) tuples. Action may be "pause"
+        or "disconnect".
 
         Both vals and actions are in reverse order, with the first items last.
 
@@ -53,7 +63,13 @@ def write_values(fp, vals, actions, sofar=0, skip=0, blocksize=BLOCKSIZE):
             offset = 0
             while actions and actions[-1][0] < (sofar + len(v)):
                 a = actions.pop()
-                offset += send_chunk(fp, v, blocksize, offset, a[0]-sofar-offset)
+                offset += send_chunk(
+                    fp,
+                    v,
+                    blocksize,
+                    offset,
+                    a[0]-sofar-offset
+                )
                 if a[1] == "pause":
                     time.sleep(a[2])
                 elif a[1] == "disconnect":
@@ -128,8 +144,18 @@ v_integer = pp.Regex(r"\d+")\
 
 v_literal = pp.MatchFirst(
     [
-        pp.QuotedString("\"", escChar="\\", unquoteResults=True, multiline=True),
-        pp.QuotedString("'", escChar="\\", unquoteResults=True, multiline=True),
+        pp.QuotedString(
+            "\"",
+            escChar="\\",
+            unquoteResults=True,
+            multiline=True
+        ),
+        pp.QuotedString(
+            "'",
+            escChar="\\",
+            unquoteResults=True,
+            multiline=True
+        ),
     ]
 )
 
@@ -202,6 +228,7 @@ class _Token(object):
         A specification token. Tokens are immutable.
     """
     __metaclass__ = abc.ABCMeta
+
     @abc.abstractmethod
     def expr(klass): # pragma: no cover
         """
@@ -278,7 +305,10 @@ class ValueGenerate(_Token):
     def expr(klass):
         e = pp.Literal("@").suppress() + v_integer
 
-        u = reduce(operator.or_, [pp.Literal(i) for i in utils.SIZE_UNITS.keys()])
+        u = reduce(
+            operator.or_,
+            [pp.Literal(i) for i in utils.SIZE_UNITS.keys()]
+        )
         e = e + pp.Optional(u, default=None)
 
         s = pp.Literal(",").suppress()
@@ -318,7 +348,9 @@ class ValueFile(_Token):
         s = os.path.expanduser(self.path)
         s = os.path.normpath(os.path.abspath(os.path.join(sd, s)))
         if not uf and not s.startswith(sd):
-            raise FileAccessDenied("File access outside of configured directory")
+            raise FileAccessDenied(
+                "File access outside of configured directory"
+            )
         if not os.path.isfile(s):
             raise FileAccessDenied("File not readable")
         return FileGenerator(s)
@@ -347,12 +379,12 @@ NakedValue = pp.MatchFirst(
 
 
 Offset = pp.MatchFirst(
-        [
-            v_integer,
-            pp.Literal("r"),
-            pp.Literal("a")
-        ]
-    )
+    [
+        v_integer,
+        pp.Literal("r"),
+        pp.Literal("a")
+    ]
+)
 
 
 class Raw(_Token):
@@ -392,11 +424,11 @@ class _Header(_Component):
 
     def values(self, settings):
         return [
-                self.key.get_generator(settings),
-                ": ",
-                self.value.get_generator(settings),
-                "\r\n",
-            ]
+            self.key.get_generator(settings),
+            ": ",
+            self.value.get_generator(settings),
+            "\r\n",
+        ]
 
 
 class Header(_Header):
@@ -459,7 +491,10 @@ class ShortcutUserAgent(_Header):
     @classmethod
     def expr(klass):
         e = pp.Literal("u").suppress()
-        u = reduce(operator.or_, [pp.Literal(i[1]) for i in http_uastrings.UASTRINGS])
+        u = reduce(
+            operator.or_,
+            [pp.Literal(i[1]) for i in http_uastrings.UASTRINGS]
+        )
         e += u | Value
         return e.setParseAction(lambda x: klass(*x))
 
@@ -470,7 +505,6 @@ class ShortcutUserAgent(_Header):
         return ShortcutUserAgent(self.value.freeze(settings))
 
 
-
 class Body(_Component):
     def __init__(self, value):
         self.value = value
@@ -483,8 +517,8 @@ class Body(_Component):
 
     def values(self, settings):
         return [
-                self.value.get_generator(settings),
-            ]
+            self.value.get_generator(settings),
+        ]
 
     def spec(self):
         return "b%s"%(self.value.spec())
@@ -506,8 +540,8 @@ class Path(_Component):
 
     def values(self, settings):
         return [
-                self.value.get_generator(settings),
-            ]
+            self.value.get_generator(settings),
+        ]
 
     def spec(self):
         return "%s"%(self.value.spec())
@@ -527,6 +561,7 @@ class Method(_Component):
         "trace",
         "connect",
     ]
+
     def __init__(self, value):
         # If it's a string, we were passed one of the methods, so we upper-case
         # it to be canonical. The user can specify a different case by using a
@@ -645,11 +680,11 @@ class PauseAt(_Action):
         e += Offset
         e += pp.Literal(",").suppress()
         e += pp.MatchFirst(
-                    [
-                        v_integer,
-                        pp.Literal("f")
-                    ]
-                )
+            [
+                v_integer,
+                pp.Literal("f")
+            ]
+        )
         return e.setParseAction(lambda x: klass(*x))
 
     def spec(self):
@@ -700,10 +735,10 @@ class InjectAt(_Action):
 
     def intermediate(self, settings):
         return (
-                self.offset,
-                "inject",
-                self.value.get_generator(settings)
-            )
+            self.offset,
+            "inject",
+            self.value.get_generator(settings)
+        )
 
     def freeze(self, settings):
         return InjectAt(self.offset, self.value.freeze(settings))
@@ -712,6 +747,7 @@ class InjectAt(_Action):
 class _Message(object):
     __metaclass__ = abc.ABCMeta
     version = "HTTP/1.1"
+
     def __init__(self, tokens):
         self.tokens = tokens
 
@@ -741,7 +777,8 @@ class _Message(object):
 
     def length(self, settings):
         """
-            Calculate the length of the base message without any applied actions.
+            Calculate the length of the base message without any applied
+            actions.
         """
         return sum(len(x) for x in self.values(settings))
 
@@ -754,7 +791,8 @@ class _Message(object):
 
     def maximum_length(self, settings):
         """
-            Calculate the maximum length of the base message with all applied actions.
+            Calculate the maximum length of the base message with all applied
+            actions.
         """
         l = self.length(settings)
         for i in self.actions:
@@ -786,7 +824,13 @@ class _Message(object):
                     tokens.append(
                         Header(
                             ValueLiteral("Date"),
-                            ValueLiteral(formatdate(timeval=None, localtime=False, usegmt=True))
+                            ValueLiteral(
+                                formatdate(
+                                    timeval=None,
+                                    localtime=False,
+                                    usegmt=True
+                                )
+                            )
                         )
                     )
         intermediate = self.__class__(tokens)
@@ -807,7 +851,8 @@ class _Message(object):
         ret = {}
         for i in self.logattrs:
             v = getattr(self, i)
-            # Careful not to log any VALUE specs without sanitizing them first. We truncate at 1k.
+            # Careful not to log any VALUE specs without sanitizing them first.
+            # We truncate at 1k.
             if hasattr(v, "values"):
                 v = [x[:TRUNCATE] for x in v.values(settings)]
                 v = "".join(v).encode("string_escape")
@@ -838,6 +883,7 @@ class _Message(object):
 
 Sep = pp.Optional(pp.Literal(":")).suppress()
 
+
 class Response(_Message):
     comps = (
         Body,
@@ -851,6 +897,7 @@ class Response(_Message):
         Reason
     )
     logattrs = ["code", "reason", "version", "body"]
+
     @property
     def code(self):
         return self._get_token(Code)
@@ -866,7 +913,14 @@ class Response(_Message):
         if self.reason:
             l.extend(self.reason.values(settings))
         else:
-            l.append(LiteralGenerator(http_status.RESPONSES.get(int(self.code.code), "Unknown code")))
+            l.append(
+                LiteralGenerator(
+                    http_status.RESPONSES.get(
+                        int(self.code.code),
+                        "Unknown code"
+                    )
+                )
+            )
         return l
 
     @classmethod
@@ -897,6 +951,7 @@ class Request(_Message):
         Raw
     )
     logattrs = ["method", "path", "body"]
+
     @property
     def method(self):
         return self._get_token(Method)
@@ -944,7 +999,7 @@ def make_error_response(reason, body=None):
     ]
     return PathodErrorResponse(tokens)
 
-FILESTART = "+"
+
 def read_file(settings, s):
     uf = settings.get("unconstrained_file_access")
     sd = settings.get("staticdir")
-- 
cgit v1.2.3


From 00d0ee5ad56d8243b1e9bfffec9a941e11359d2c Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 25 Oct 2014 15:30:54 +1300
Subject: Parse patterns eagerly on instantiation

---
 libpathod/cmdline.py  | 42 +++++++++++++++++++-------
 libpathod/language.py |  4 +--
 libpathod/pathoc.py   | 81 ++++++++++++++++++++++++++++++++-------------------
 libpathod/pathod.py   | 66 ++++++++++++++++++++++++-----------------
 4 files changed, 123 insertions(+), 70 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py
index 2c6e094e..1a000a93 100644
--- a/libpathod/cmdline.py
+++ b/libpathod/cmdline.py
@@ -3,7 +3,8 @@ import argparse
 import os
 import os.path
 import sys
-from . import pathoc, pathod, version, utils
+import re
+from . import pathoc, pathod, version, utils, language
 from netlib import http_uastrings
 
 
@@ -78,7 +79,10 @@ def go_pathoc():
     group.add_argument(
         "--sslversion", dest="sslversion", type=int, default=4,
         choices=[1, 2, 3, 4],
-        help="Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default to SSLv23."
+        help="""
+            Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default
+            to SSLv23.
+        """
     )
 
     group = parser.add_argument_group(
@@ -149,9 +153,14 @@ def go_pathoc():
     for r in args.request:
         if os.path.exists(r):
             data = open(r).read()
-            reqs.append(data)
-        else:
-            reqs.append(r)
+            r = data
+        try:
+            req = language.parse_request(r)
+        except language.ParseException, v:
+            print >> sys.stderr, "Error parsing request spec: %s"%v.msg
+            print >> sys.stderr, v.marked()
+            sys.exit(1)
+        reqs.append(req)
     args.request = reqs
 
     pathoc.main(args)
@@ -267,7 +276,8 @@ def go_pathod():
     group.add_argument(
         "--sslversion", dest="sslversion", type=int, default=4,
         choices=[1, 2, 3, 4],
-        help="Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default to SSLv23."
+        help=""""Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default
+        to SSLv23."""
     )
 
     group = parser.add_argument_group(
@@ -327,13 +337,23 @@ def go_pathod():
     args.sizelimit = sizelimit
 
     anchors = []
-    for patt, spec in anchors:
+    for patt, spec in args.anchors:
         if os.path.exists(spec):
             data = open(spec).read()
-            anchors.append((patt, data))
-        else:
-            anchors.append((patt, spec))
+            spec = data
+
+        try:
+            req = language.parse_response(spec)
+        except language.ParseException, v:
+            print >> sys.stderr, "Error parsing anchor spec: %s"%v.msg
+            print >> sys.stderr, v.marked()
+            sys.exit(1)
+        try:
+            arex = re.compile(patt)
+        except re.error:
+            print >> sys.stderr, "Invalid regex in anchor: %s" % patt
+            sys.exit(1)
+        anchors.append((arex, req))
     args.anchors = anchors
 
     pathod.main(args)
-
diff --git a/libpathod/language.py b/libpathod/language.py
index 2969055b..b4b59167 100644
--- a/libpathod/language.py
+++ b/libpathod/language.py
@@ -1018,7 +1018,7 @@ def read_file(settings, s):
 
 def parse_response(s):
     """
-        May raise ParseException or FileAccessDenied
+        May raise ParseException
     """
     try:
         s = s.decode("ascii")
@@ -1032,7 +1032,7 @@ def parse_response(s):
 
 def parse_request(s):
     """
-        May raise ParseException or FileAccessDenied
+        May raise ParseException
     """
     try:
         s = s.decode("ascii")
diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py
index e534bba5..ae1e98cf 100644
--- a/libpathod/pathoc.py
+++ b/libpathod/pathoc.py
@@ -2,10 +2,14 @@ import sys
 import os
 from netlib import tcp, http, certutils
 import netlib.utils
-import language, utils
+
+import language
+import utils
 import OpenSSL.crypto
 
-class PathocError(Exception): pass
+
+class PathocError(Exception):
+    pass
 
 
 class SSLInfo:
@@ -24,7 +28,14 @@ class Response:
 
 
 class Pathoc(tcp.TCPClient):
-    def __init__(self, address, ssl=None, sni=None, sslversion=4, clientcert=None, ciphers=None):
+    def __init__(
+            self,
+            address,
+            ssl=None,
+            sni=None,
+            sslversion=4,
+            clientcert=None,
+            ciphers=None):
         tcp.TCPClient.__init__(self, address)
         self.settings = dict(
             staticdir = os.getcwd(),
@@ -37,9 +48,9 @@ class Pathoc(tcp.TCPClient):
 
     def http_connect(self, connect_to):
         self.wfile.write(
-                    'CONNECT %s:%s HTTP/1.1\r\n'%tuple(connect_to) +
-                    '\r\n'
-                    )
+            'CONNECT %s:%s HTTP/1.1\r\n'%tuple(connect_to) +
+            '\r\n'
+        )
         self.wfile.flush()
         l = self.rfile.readline()
         if not l:
@@ -61,17 +72,17 @@ class Pathoc(tcp.TCPClient):
         if self.ssl:
             try:
                 self.convert_to_ssl(
-                        sni=self.sni,
-                        cert=self.clientcert,
-                        method=self.sslversion,
-                        cipher_list = self.ciphers
-                    )
+                    sni=self.sni,
+                    cert=self.clientcert,
+                    method=self.sslversion,
+                    cipher_list = self.ciphers
+                )
             except tcp.NetLibError, v:
                 raise PathocError(str(v))
             self.sslinfo = SSLInfo(
-                        self.connection.get_peer_cert_chain(),
-                        self.get_current_cipher()
-                    )
+                self.connection.get_peer_cert_chain(),
+                self.get_current_cipher()
+            )
 
     def request(self, spec):
         """
@@ -88,7 +99,9 @@ class Pathoc(tcp.TCPClient):
         return Response(*ret)
 
     def _show_summary(self, fp, httpversion, code, msg, headers, content):
-        print >> fp, "<< %s %s: %s bytes"%(code, utils.xrepr(msg), len(content))
+        print >> fp, "<< %s %s: %s bytes"%(
+            code, utils.xrepr(msg), len(content)
+        )
 
     def _show(self, fp, header, data, hexdump):
         if hexdump:
@@ -99,7 +112,18 @@ class Pathoc(tcp.TCPClient):
             print >> fp, "%s (unprintables escaped):"%header
             print >> fp, netlib.utils.cleanBin(data)
 
-    def print_request(self, spec, showreq, showresp, explain, showssl, hexdump, ignorecodes, ignoretimeout, fp=sys.stdout):
+    def print_request(
+        self,
+        r,
+        showreq,
+        showresp,
+        explain,
+        showssl,
+        hexdump,
+        ignorecodes,
+        ignoretimeout,
+        fp=sys.stdout
+    ):
         """
             Performs a series of requests, and prints results to the specified
             file descriptor.
@@ -114,16 +138,6 @@ class Pathoc(tcp.TCPClient):
 
             Returns True if we have a non-ignored response.
         """
-        try:
-            r = language.parse_request(spec)
-        except language.ParseException, v:
-            print >> fp, "Error parsing request spec: %s"%v.msg
-            print >> fp, v.marked()
-            return
-        except language.FileAccessDenied, v:
-            print >> fp, "File access error: %s"%v
-            return
-
         if explain:
             r = r.freeze(self.settings, self.address.host)
 
@@ -133,7 +147,12 @@ class Pathoc(tcp.TCPClient):
         if showresp:
             self.rfile.start_log()
         try:
-            req = language.serve(r, self.wfile, self.settings, self.address.host)
+            req = language.serve(
+                r,
+                self.wfile,
+                self.settings,
+                self.address.host
+            )
             self.wfile.flush()
             resp = http.read_response(self.rfile, r.method.string(), None)
         except http.HttpError, v:
@@ -174,13 +193,15 @@ class Pathoc(tcp.TCPClient):
                         print >> fp, "%s=%s"%cn,
                     print >> fp
                     print >> fp, "\tVersion: %s"%i.get_version()
-                    print >> fp, "\tValidity: %s - %s"%(i.get_notBefore(),i.get_notAfter())
+                    print >> fp, "\tValidity: %s - %s"%(
+                        i.get_notBefore(), i.get_notAfter()
+                    )
                     print >> fp, "\tSerial: %s"%i.get_serial_number()
                     print >> fp, "\tAlgorithm: %s"%i.get_signature_algorithm()
                     pk = i.get_pubkey()
                     types = {
-                            OpenSSL.crypto.TYPE_RSA: "RSA",
-                            OpenSSL.crypto.TYPE_DSA: "DSA"
+                        OpenSSL.crypto.TYPE_RSA: "RSA",
+                        OpenSSL.crypto.TYPE_DSA: "DSA"
                     }
                     t = types.get(pk.type(), "Uknown")
                     print >> fp, "\tPubkey: %s bit %s"%(pk.bits(), t)
diff --git a/libpathod/pathod.py b/libpathod/pathod.py
index 92e5b2db..173773cf 100644
--- a/libpathod/pathod.py
+++ b/libpathod/pathod.py
@@ -1,13 +1,12 @@
 import urllib
 import threading
-import re
 import logging
 import os
 import sys
 from netlib import tcp, http, wsgi, certutils
 import netlib.utils
 
-import version, app, language, utils
+from . import version, app, language, utils
 
 
 DEFAULT_CERT_DOMAIN = "pathod.net"
@@ -71,7 +70,12 @@ class PathodHandler(tcp.BaseHandler):
         if self.server.explain and not isinstance(crafted, language.PathodErrorResponse):
             crafted = crafted.freeze(self.server.request_settings, None)
             self.info(">> Spec: %s" % crafted.spec())
-        response_log = language.serve(crafted, self.wfile, self.server.request_settings, None)
+        response_log = language.serve(
+            crafted,
+            self.wfile,
+            self.server.request_settings,
+            None
+        )
         if response_log["disconnect"]:
             return False, response_log
         return True, response_log
@@ -169,8 +173,7 @@ class PathodHandler(tcp.BaseHandler):
         for i in self.server.anchors:
             if i[0].match(path):
                 self.info("crafting anchor: %s" % path)
-                aresp = language.parse_response(i[1])
-                again, retlog["response"] = self.serve_crafted(aresp)
+                again, retlog["response"] = self.serve_crafted(i[1])
                 return again, retlog
 
         if not self.server.nocraft and path.startswith(self.server.craftanchor):
@@ -189,7 +192,10 @@ class PathodHandler(tcp.BaseHandler):
         elif self.server.noweb:
             crafted = language.make_error_response("Access Denied")
             language.serve(crafted, self.wfile, self.server.request_settings)
-            return False, dict(type="error", msg="Access denied: web interface disabled")
+            return False, dict(
+                type="error",
+                msg="Access denied: web interface disabled"
+            )
         else:
             self.info("app: %s %s" % (method, path))
             req = wsgi.Request("http", method, path, headers, content)
@@ -259,19 +265,34 @@ class Pathod(tcp.TCPServer):
     LOGBUF = 500
 
     def __init__(
-            self, addr, confdir=CONFDIR, ssl=False, ssloptions=None,
-            craftanchor="/p/", staticdir=None, anchors=None,
-            sizelimit=None, noweb=False, nocraft=False, noapi=False,
-            nohang=False, timeout=None, logreq=False, logresp=False,
-            explain=False, hexdump=False
+        self,
+        addr,
+        confdir=CONFDIR,
+        ssl=False,
+        ssloptions=None,
+        craftanchor="/p/",
+        staticdir=None,
+        anchors=(),
+        sizelimit=None,
+        noweb=False,
+        nocraft=False,
+        noapi=False,
+        nohang=False,
+        timeout=None,
+        logreq=False,
+        logresp=False,
+        explain=False,
+        hexdump=False
     ):
         """
             addr: (address, port) tuple. If port is 0, a free port will be
             automatically chosen.
             ssloptions: an SSLOptions object.
-            craftanchor: string specifying the path under which to anchor response generation.
+            craftanchor: string specifying the path under which to anchor
+            response generation.
             staticdir: path to a directory of static resources, or None.
-            anchors: A list of (regex, spec) tuples, or None.
+            anchors: List of (regex object, language.Request object) tuples, or
+            None.
             sizelimit: Limit size of served data.
             nocraft: Disable response crafting.
             noapi: Disable the API.
@@ -283,26 +304,17 @@ class Pathod(tcp.TCPServer):
         self.staticdir = staticdir
         self.craftanchor = craftanchor
         self.sizelimit = sizelimit
-        self.noweb, self.nocraft, self.noapi, self.nohang = noweb, nocraft, noapi, nohang
-        self.timeout, self.logreq, self.logresp, self.hexdump = timeout, logreq, logresp, hexdump
+        self.noweb, self.nocraft = noweb, nocraft
+        self.noapi, self.nohang = noapi, nohang
+        self.timeout, self.logreq = timeout, logreq
+        self.logresp, self.hexdump = logresp, hexdump
         self.explain = explain
 
         self.app = app.make_app(noapi)
         self.app.config["pathod"] = self
         self.log = []
         self.logid = 0
-        self.anchors = []
-        if anchors:
-            for i in anchors:
-                try:
-                    arex = re.compile(i[0])
-                except re.error:
-                    raise PathodError("Invalid regex in anchor: %s" % i[0])
-                try:
-                    language.parse_response(i[1])
-                except language.ParseException, v:
-                    raise PathodError("Invalid page spec in anchor: '%s', %s" % (i[1], str(v)))
-                self.anchors.append((arex, i[1]))
+        self.anchors = anchors
 
     def check_policy(self, req, settings):
         """
-- 
cgit v1.2.3


From d4e6c25d45f680df4f807805ce878b498a5fb0c5 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 25 Oct 2014 15:37:57 +1300
Subject: Update pyparsing

---
 libpathod/contrib/pyparsing.py | 755 ++++++++++++++++++++---------------------
 1 file changed, 377 insertions(+), 378 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/contrib/pyparsing.py b/libpathod/contrib/pyparsing.py
index 9be97dc2..7dfe1043 100644
--- a/libpathod/contrib/pyparsing.py
+++ b/libpathod/contrib/pyparsing.py
@@ -1,6 +1,6 @@
 # module pyparsing.py
 #
-# Copyright (c) 2003-2011  Paul T. McGuire
+# Copyright (c) 2003-2013  Paul T. McGuire
 #
 # Permission is hereby granted, free of charge, to any person obtaining
 # a copy of this software and associated documentation files (the
@@ -21,7 +21,6 @@
 # TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
 # SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
 #
-#from __future__ import generators
 
 __doc__ = \
 """
@@ -40,7 +39,7 @@ Here is a program to parse "Hello, World!" (or any greeting of the form C{"<salu
     greet = Word( alphas ) + "," + Word( alphas ) + "!"
 
     hello = "Hello, World!"
-    print hello, "->", greet.parseString( hello )
+    print (hello, "->", greet.parseString( hello ))
 
 The program outputs the following::
 
@@ -58,8 +57,8 @@ The pyparsing module handles some of the problems that are typically vexing when
  - embedded comments
 """
 
-__version__ = "1.5.6"
-__versionTime__ = "26 June 2011 10:53"
+__version__ = "2.0.2"
+__versionTime__ = "13 April 2014 12:10"
 __author__ = "Paul McGuire <ptmcg@users.sourceforge.net>"
 
 import string
@@ -69,6 +68,8 @@ import sys
 import warnings
 import re
 import sre_constants
+import collections
+import pprint
 #~ sys.stderr.write( "testing pyparsing module, version %s, %s\n" % (__version__,__versionTime__ ) )
 
 __all__ = [
@@ -81,32 +82,29 @@ __all__ = [
 'White', 'Word', 'WordEnd', 'WordStart', 'ZeroOrMore',
 'alphanums', 'alphas', 'alphas8bit', 'anyCloseTag', 'anyOpenTag', 'cStyleComment', 'col',
 'commaSeparatedList', 'commonHTMLEntity', 'countedArray', 'cppStyleComment', 'dblQuotedString',
-'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'getTokensEndLoc', 'hexnums',
+'dblSlashComment', 'delimitedList', 'dictOf', 'downcaseTokens', 'empty', 'hexnums',
 'htmlComment', 'javaStyleComment', 'keepOriginalText', 'line', 'lineEnd', 'lineStart', 'lineno',
 'makeHTMLTags', 'makeXMLTags', 'matchOnlyAtCol', 'matchPreviousExpr', 'matchPreviousLiteral',
 'nestedExpr', 'nullDebugAction', 'nums', 'oneOf', 'opAssoc', 'operatorPrecedence', 'printables',
 'punc8bit', 'pythonStyleComment', 'quotedString', 'removeQuotes', 'replaceHTMLEntity', 
 'replaceWith', 'restOfLine', 'sglQuotedString', 'srange', 'stringEnd',
 'stringStart', 'traceParseAction', 'unicodeString', 'upcaseTokens', 'withAttribute',
-'indentedBlock', 'originalTextFor',
+'indentedBlock', 'originalTextFor', 'ungroup', 'infixNotation','locatedExpr',
 ]
 
-"""
-Detect if we are running version 3.X and make appropriate changes
-Robert A. Clark
-"""
-_PY3K = sys.version_info[0] > 2
-if _PY3K:
+PY_3 = sys.version.startswith('3')
+if PY_3:
     _MAX_INT = sys.maxsize
     basestring = str
     unichr = chr
     _ustr = str
-    alphas = string.ascii_lowercase + string.ascii_uppercase
+
+    # build list of single arg builtins, that can be used as parse actions
+    singleArgBuiltins = [sum, len, sorted, reversed, list, tuple, set, any, all, min, max]
+
 else:
     _MAX_INT = sys.maxint
     range = xrange
-    set = lambda s : dict( [(c,0) for c in s] )
-    alphas = string.lowercase + string.uppercase
 
     def _ustr(obj):
         """Drop-in replacement for str(obj) that tries to be Unicode friendly. It first tries
@@ -134,24 +132,24 @@ else:
             # Replace unprintables with question marks?
             #return unicode(obj).encode(sys.getdefaultencoding(), 'replace')
             # ...
-            
-    alphas = string.lowercase + string.uppercase
-
-# build list of single arg builtins, tolerant of Python version, that can be used as parse actions
-singleArgBuiltins = []
-import __builtin__
-for fname in "sum len enumerate sorted reversed list tuple set any all".split():
-    try:
-        singleArgBuiltins.append(getattr(__builtin__,fname))
-    except AttributeError:
-        continue
 
+    # build list of single arg builtins, tolerant of Python version, that can be used as parse actions
+    singleArgBuiltins = []
+    import __builtin__
+    for fname in "sum len sorted reversed list tuple set any all min max".split():
+        try:
+            singleArgBuiltins.append(getattr(__builtin__,fname))
+        except AttributeError:
+            continue
+            
+_generatorType = type((y for y in range(1)))
+ 
 def _xml_escape(data):
     """Escape &, <, >, ", ', etc. in a string of data."""
 
     # ampersand must be replaced first
     from_symbols = '&><"\''
-    to_symbols = ['&'+s+';' for s in "amp gt lt quot apos".split()]
+    to_symbols = ('&'+s+';' for s in "amp gt lt quot apos".split())
     for from_,to_ in zip(from_symbols, to_symbols):
         data = data.replace(from_, to_)
     return data
@@ -159,11 +157,12 @@ def _xml_escape(data):
 class _Constants(object):
     pass
 
-nums       = string.digits
+alphas = string.ascii_lowercase + string.ascii_uppercase
+nums       = "0123456789"
 hexnums    = nums + "ABCDEFabcdef"
 alphanums  = alphas + nums
 _bslash    = chr(92)
-printables = "".join( [ c for c in string.printable if c not in string.whitespace ] )
+printables = "".join(c for c in string.printable if c not in string.whitespace)
 
 class ParseBaseException(Exception):
     """base exception class for all parsing runtime exceptions"""
@@ -206,12 +205,12 @@ class ParseBaseException(Exception):
         line_str = self.line
         line_column = self.column - 1
         if markerString:
-            line_str = "".join( [line_str[:line_column],
-                                markerString, line_str[line_column:]])
+            line_str = "".join((line_str[:line_column],
+                                markerString, line_str[line_column:]))
         return line_str.strip()
     def __dir__(self):
         return "loc msg pstr parserElement lineno col line " \
-               "markInputLine __str__ __repr__".split()
+               "markInputline __str__ __repr__".split()
 
 class ParseException(ParseBaseException):
     """exception thrown when parse expressions don't match class;
@@ -228,8 +227,8 @@ class ParseFatalException(ParseBaseException):
     pass
 
 class ParseSyntaxException(ParseFatalException):
-    """just like C{ParseFatalException}, but thrown internally when an
-       C{ErrorStop} ('-' operator) indicates that parsing is to stop immediately because
+    """just like C{L{ParseFatalException}}, but thrown internally when an
+       C{L{ErrorStop<And._ErrorStop>}} ('-' operator) indicates that parsing is to stop immediately because
        an unbacktrackable syntax error has been found"""
     def __init__(self, pe):
         super(ParseSyntaxException, self).__init__(
@@ -272,7 +271,6 @@ class ParseResults(object):
        - by list index (C{results[0], results[1]}, etc.)
        - by attribute (C{results.<resultsName>})
        """
-    #~ __slots__ = ( "__toklist", "__tokdict", "__doinit", "__name", "__parent", "__accumNames", "__weakref__" )
     def __new__(cls, toklist, name=None, asList=True, modal=True ):
         if isinstance(toklist, cls):
             return toklist
@@ -290,6 +288,8 @@ class ParseResults(object):
             self.__accumNames = {}
             if isinstance(toklist, list):
                 self.__toklist = toklist[:]
+            elif isinstance(toklist, _generatorType):
+                self.__toklist = list(toklist)
             else:
                 self.__toklist = [toklist]
             self.__tokdict = dict()
@@ -367,16 +367,64 @@ class ParseResults(object):
     __nonzero__ = __bool__
     def __iter__( self ): return iter( self.__toklist )
     def __reversed__( self ): return iter( self.__toklist[::-1] )
-    def keys( self ):
+    def iterkeys( self ):
         """Returns all named result keys."""
-        return self.__tokdict.keys()
+        if hasattr(self.__tokdict, "iterkeys"):
+            return self.__tokdict.iterkeys()
+        else:
+            return iter(self.__tokdict)
 
-    def pop( self, index=-1 ):
+    def itervalues( self ):
+        """Returns all named result values."""
+        return (self[k] for k in self.iterkeys())
+            
+    def iteritems( self ):
+        return ((k, self[k]) for k in self.iterkeys())
+
+    if PY_3:
+        keys = iterkeys
+        values = itervalues
+        items = iteritems
+    else:
+        def keys( self ):
+            """Returns all named result keys."""
+            return list(self.iterkeys())
+
+        def values( self ):
+            """Returns all named result values."""
+            return list(self.itervalues())
+                
+        def items( self ):
+            """Returns all named result keys and values as a list of tuples."""
+            return list(self.iteritems())
+
+    def haskeys( self ):
+        """Since keys() returns an iterator, this method is helpful in bypassing
+           code that looks for the existence of any defined results names."""
+        return bool(self.__tokdict)
+        
+    def pop( self, *args, **kwargs):
         """Removes and returns item at specified index (default=last).
-           Will work with either numeric indices or dict-key indicies."""
-        ret = self[index]
-        del self[index]
-        return ret
+           Supports both list and dict semantics for pop(). If passed no
+           argument or an integer argument, it will use list semantics
+           and pop tokens from the list of parsed tokens. If passed a 
+           non-integer argument (most likely a string), it will use dict
+           semantics and pop the corresponding value from any defined 
+           results names. A second default return value argument is 
+           supported, just as in dict.pop()."""
+        if not args:
+            args = [-1]
+        if 'default' in kwargs:
+            args.append(kwargs['default'])
+        if (isinstance(args[0], int) or 
+                        len(args) == 1 or 
+                        args[0] in self):
+            ret = self[index]
+            del self[index]
+            return ret
+        else:
+            defaultvalue = args[1]
+            return defaultvalue
 
     def get(self, key, defaultValue=None):
         """Returns named result matching the given key, or if there is no
@@ -396,24 +444,35 @@ class ParseResults(object):
             for k, (value, position) in enumerate(occurrences):
                 occurrences[k] = _ParseResultsWithOffset(value, position + (position > index))
 
-    def items( self ):
-        """Returns all named result keys and values as a list of tuples."""
-        return [(k,self[k]) for k in self.__tokdict]
+    def append( self, item ):
+        """Add single element to end of ParseResults list of elements."""
+        self.__toklist.append(item)
 
-    def values( self ):
-        """Returns all named result values."""
-        return [ v[-1][0] for v in self.__tokdict.values() ]
+    def extend( self, itemseq ):
+        """Add sequence of elements to end of ParseResults list of elements."""
+        if isinstance(itemseq, ParseResults):
+            self += itemseq
+        else:
+            self.__toklist.extend(itemseq)
+
+    def clear( self ):
+        """Clear all elements and results names."""
+        del self.__toklist[:]
+        self.__tokdict.clear()
 
     def __getattr__( self, name ):
-        if True: #name not in self.__slots__:
-            if name in self.__tokdict:
-                if name not in self.__accumNames:
-                    return self.__tokdict[name][-1][0]
-                else:
-                    return ParseResults([ v[0] for v in self.__tokdict[name] ])
+        try:
+            return self[name]
+        except KeyError:
+            return ""
+            
+        if name in self.__tokdict:
+            if name not in self.__accumNames:
+                return self.__tokdict[name][-1][0]
             else:
-                return ""
-        return None
+                return ParseResults([ v[0] for v in self.__tokdict[name] ])
+        else:
+            return ""
 
     def __add__( self, other ):
         ret = self.copy()
@@ -444,16 +503,13 @@ class ParseResults(object):
         return "(%s, %s)" % ( repr( self.__toklist ), repr( self.__tokdict ) )
 
     def __str__( self ):
-        out = "["
-        sep = ""
+        out = []
         for i in self.__toklist:
             if isinstance(i, ParseResults):
-                out += sep + _ustr(i)
+                out.append(_ustr(i))
             else:
-                out += sep + repr(i)
-            sep = ", "
-        out += "]"
-        return out
+                out.append(repr(i))
+        return '[' + ', '.join(out) + ']'
 
     def _asStringList( self, sep='' ):
         out = []
@@ -478,7 +534,10 @@ class ParseResults(object):
 
     def asDict( self ):
         """Returns the named parse results as dictionary."""
-        return dict( self.items() )
+        if PY_3:
+            return dict( self.items() )
+        else:
+            return dict( self.iteritems() )
 
     def copy( self ):
         """Returns a new copy of a C{ParseResults} object."""
@@ -493,8 +552,8 @@ class ParseResults(object):
         """Returns the parse results as XML. Tags are created for tokens and lists that have defined results names."""
         nl = "\n"
         out = []
-        namedItems = dict( [ (v[1],k) for (k,vlist) in self.__tokdict.items()
-                                                            for v in vlist ] )
+        namedItems = dict((v[1],k) for (k,vlist) in self.__tokdict.items()
+                                                            for v in vlist)
         nextLevelIndent = indent + "  "
 
         # collapse out indents if formatting is not desired
@@ -579,14 +638,13 @@ class ParseResults(object):
            in a nested display of other data."""
         out = []
         out.append( indent+_ustr(self.asList()) )
-        keys = self.items()
-        keys.sort()
-        for k,v in keys:
+        items = sorted(self.items())
+        for k,v in items:
             if out:
                 out.append('\n')
             out.append( "%s%s- %s: " % (indent,('  '*depth), k) )
             if isinstance(v,ParseResults):
-                if v.keys():
+                if v.haskeys():
                     out.append( v.dump(indent,depth+1) )
                 else:
                     out.append(_ustr(v))
@@ -594,6 +652,12 @@ class ParseResults(object):
                 out.append(_ustr(v))
         return "".join(out)
 
+    def pprint(self, *args, **kwargs):
+        """Pretty-printer for parsed results as a list, using the C{pprint} module.
+           Accepts additional positional or keyword args as defined for the 
+           C{pprint.pprint} method. (U{http://docs.python.org/3/library/pprint.html#pprint.pprint})"""
+        pprint.pprint(self.asList(), *args, **kwargs)
+
     # add support for pickle protocol
     def __getstate__(self):
         return ( self.__toklist,
@@ -616,7 +680,9 @@ class ParseResults(object):
             self.__parent = None
 
     def __dir__(self):
-        return dir(super(ParseResults,self)) + self.keys()
+        return dir(super(ParseResults,self)) + list(self.keys())
+
+collections.MutableMapping.register(ParseResults)
 
 def col (loc,strg):
     """Returns current column within a string, counting newlines as line separators.
@@ -624,7 +690,7 @@ def col (loc,strg):
 
    Note: the default parsing behavior is to expand tabs in the input string
    before starting the parsing process.  See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
-   on parsing strings containing <TAB>s, and suggested methods to maintain a
+   on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
    consistent view of the parsed string, the parse location, and line and column
    positions within the parsed string.
    """
@@ -636,7 +702,7 @@ def lineno(loc,strg):
 
    Note: the default parsing behavior is to expand tabs in the input string
    before starting the parsing process.  See L{I{ParserElement.parseString}<ParserElement.parseString>} for more information
-   on parsing strings containing <TAB>s, and suggested methods to maintain a
+   on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
    consistent view of the parsed string, the parse location, and line and column
    positions within the parsed string.
    """
@@ -653,7 +719,7 @@ def line( loc, strg ):
         return strg[lastCR+1:]
 
 def _defaultStartDebugAction( instring, loc, expr ):
-    print ("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) ))
+    print (("Match " + _ustr(expr) + " at loc " + _ustr(loc) + "(%d,%d)" % ( lineno(loc,instring), col(loc,instring) )))
 
 def _defaultSuccessDebugAction( instring, startloc, endloc, expr, toks ):
     print ("Matched " + _ustr(expr) + " -> " + str(toks.asList()))
@@ -665,35 +731,47 @@ def nullDebugAction(*args):
     """'Do-nothing' debug action, to suppress debugging output during parsing."""
     pass
 
+# Only works on Python 3.x - nonlocal is toxic to Python 2 installs
+#~ 'decorator to trim function calls to match the arity of the target'
+#~ def _trim_arity(func, maxargs=3):
+    #~ if func in singleArgBuiltins:
+        #~ return lambda s,l,t: func(t)
+    #~ limit = 0
+    #~ foundArity = False
+    #~ def wrapper(*args):
+        #~ nonlocal limit,foundArity
+        #~ while 1:
+            #~ try:
+                #~ ret = func(*args[limit:])
+                #~ foundArity = True
+                #~ return ret
+            #~ except TypeError:
+                #~ if limit == maxargs or foundArity:
+                    #~ raise
+                #~ limit += 1
+                #~ continue
+    #~ return wrapper
+
+# this version is Python 2.x-3.x cross-compatible
 'decorator to trim function calls to match the arity of the target'
-if not _PY3K:
-    def _trim_arity(func, maxargs=2):
-        limit = [0]
-        def wrapper(*args):
-            while 1:
-                try:
-                    return func(*args[limit[0]:])
-                except TypeError:
-                    if limit[0] <= maxargs:
-                        limit[0] += 1
-                        continue
-                    raise
-        return wrapper
-else:
-    def _trim_arity(func, maxargs=2):
-        limit = maxargs
-        def wrapper(*args):
-            #~ nonlocal limit
-            while 1:
-                try:
-                    return func(*args[limit:])
-                except TypeError:
-                    if limit:
-                        limit -= 1
-                        continue
-                    raise
-        return wrapper
-    
+def _trim_arity(func, maxargs=2):
+    if func in singleArgBuiltins:
+        return lambda s,l,t: func(t)
+    limit = [0]
+    foundArity = [False]
+    def wrapper(*args):
+        while 1:
+            try:
+                ret = func(*args[limit[0]:])
+                foundArity[0] = True
+                return ret
+            except TypeError:
+                if limit[0] <= maxargs and not foundArity[0]:
+                    limit[0] += 1
+                    continue
+                raise
+    return wrapper
+ 
 class ParserElement(object):
     """Abstract base level parser element class."""
     DEFAULT_WHITE_CHARS = " \n\t\r"
@@ -705,6 +783,13 @@ class ParserElement(object):
         ParserElement.DEFAULT_WHITE_CHARS = chars
     setDefaultWhitespaceChars = staticmethod(setDefaultWhitespaceChars)
 
+    def inlineLiteralsUsing(cls):
+        """
+        Set class to be used for inclusion of string literals into a parser.
+        """
+        ParserElement.literalStringClass = cls
+    inlineLiteralsUsing = staticmethod(inlineLiteralsUsing)
+
     def __init__( self, savelist=False ):
         self.parseAction = list()
         self.failAction = None
@@ -789,14 +874,14 @@ class ParserElement(object):
            C{fn(loc,toks)}, C{fn(toks)}, or just C{fn()}, where:
             - s   = the original string being parsed (see note below)
             - loc = the location of the matching substring
-            - toks = a list of the matched tokens, packaged as a ParseResults object
+            - toks = a list of the matched tokens, packaged as a C{L{ParseResults}} object
            If the functions in fns modify the tokens, they can return them as the return
            value from fn, and the modified list of tokens will replace the original.
            Otherwise, fn does not need to return any value.
 
            Note: the default parsing behavior is to expand tabs in the input string
            before starting the parsing process.  See L{I{parseString}<parseString>} for more information
-           on parsing strings containing <TAB>s, and suggested methods to maintain a
+           on parsing strings containing C{<TAB>}s, and suggested methods to maintain a
            consistent view of the parsed string, the parse location, and line and column
            positions within the parsed string.
            """
@@ -818,7 +903,7 @@ class ParserElement(object):
             - loc = location where expression match was attempted and failed
             - expr = the parse expression that failed
             - err = the exception thrown
-           The function returns no value.  It may throw C{ParseFatalException}
+           The function returns no value.  It may throw C{L{ParseFatalException}}
            if it is desired to stop parsing immediately."""
         self.failAction = fn
         return self
@@ -872,15 +957,11 @@ class ParserElement(object):
                     loc,tokens = self.parseImpl( instring, preloc, doActions )
                 except IndexError:
                     raise ParseException( instring, len(instring), self.errmsg, self )
-            except ParseBaseException:
+            except ParseBaseException as err:
                 #~ print ("Exception raised:", err)
-                err = None
                 if self.debugActions[2]:
-                    err = sys.exc_info()[1]
                     self.debugActions[2]( instring, tokensStart, self, err )
                 if self.failAction:
-                    if err is None:
-                        err = sys.exc_info()[1]
                     self.failAction( instring, tokensStart, self, err )
                 raise
         else:
@@ -910,10 +991,9 @@ class ParserElement(object):
                                                       self.resultsName,
                                                       asList=self.saveAsList and isinstance(tokens,(ParseResults,list)),
                                                       modal=self.modalResults )
-                except ParseBaseException:
+                except ParseBaseException as err:
                     #~ print "Exception raised in user parse action:", err
                     if (self.debugActions[2] ):
-                        err = sys.exc_info()[1]
                         self.debugActions[2]( instring, tokensStart, self, err )
                     raise
             else:
@@ -952,8 +1032,8 @@ class ParserElement(object):
                 value = self._parseNoCache( instring, loc, doActions, callPreParse )
                 ParserElement._exprArgCache[ lookup ] = (value[0],value[1].copy())
                 return value
-            except ParseBaseException:
-                pe = sys.exc_info()[1]
+            except ParseBaseException as pe:
+                pe.__traceback__ = None
                 ParserElement._exprArgCache[ lookup ] = pe
                 raise
 
@@ -994,7 +1074,7 @@ class ParserElement(object):
 
            If you want the grammar to require that the entire input string be
            successfully parsed, then set C{parseAll} to True (equivalent to ending
-           the grammar with C{StringEnd()}).
+           the grammar with C{L{StringEnd()}}).
 
            Note: C{parseString} implicitly calls C{expandtabs()} on the input string,
            in order to report proper column numbers in parse actions.
@@ -1023,12 +1103,11 @@ class ParserElement(object):
                 loc = self.preParse( instring, loc )
                 se = Empty() + StringEnd()
                 se._parse( instring, loc )
-        except ParseBaseException:
+        except ParseBaseException as exc:
             if ParserElement.verbose_stacktrace:
                 raise
             else:
                 # catch and re-raise exception from here, clears out pyparsing internal stack trace
-                exc = sys.exc_info()[1]
                 raise exc
         else:
             return tokens
@@ -1076,16 +1155,15 @@ class ParserElement(object):
                             loc = nextLoc
                     else:
                         loc = preloc+1
-        except ParseBaseException:
+        except ParseBaseException as exc:
             if ParserElement.verbose_stacktrace:
                 raise
             else:
                 # catch and re-raise exception from here, clears out pyparsing internal stack trace
-                exc = sys.exc_info()[1]
                 raise exc
 
     def transformString( self, instring ):
-        """Extension to C{scanString}, to modify matching text with modified tokens that may
+        """Extension to C{L{scanString}}, to modify matching text with modified tokens that may
            be returned from a parse action.  To use C{transformString}, define a grammar and
            attach a parse action to it that modifies the returned token list.
            Invoking C{transformString()} on a target string will then scan for matches,
@@ -1110,33 +1188,31 @@ class ParserElement(object):
             out.append(instring[lastE:])
             out = [o for o in out if o]
             return "".join(map(_ustr,_flatten(out)))
-        except ParseBaseException:
+        except ParseBaseException as exc:
             if ParserElement.verbose_stacktrace:
                 raise
             else:
                 # catch and re-raise exception from here, clears out pyparsing internal stack trace
-                exc = sys.exc_info()[1]
                 raise exc
 
     def searchString( self, instring, maxMatches=_MAX_INT ):
-        """Another extension to C{scanString}, simplifying the access to the tokens found
+        """Another extension to C{L{scanString}}, simplifying the access to the tokens found
            to match the given parse expression.  May be called with optional
            C{maxMatches} argument, to clip searching after 'n' matches are found.
         """
         try:
             return ParseResults([ t for t,s,e in self.scanString( instring, maxMatches ) ])
-        except ParseBaseException:
+        except ParseBaseException as exc:
             if ParserElement.verbose_stacktrace:
                 raise
             else:
                 # catch and re-raise exception from here, clears out pyparsing internal stack trace
-                exc = sys.exc_info()[1]
                 raise exc
 
     def __add__(self, other ):
-        """Implementation of + operator - returns And"""
+        """Implementation of + operator - returns C{L{And}}"""
         if isinstance( other, basestring ):
-            other = Literal( other )
+            other = ParserElement.literalStringClass( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
@@ -1144,9 +1220,9 @@ class ParserElement(object):
         return And( [ self, other ] )
 
     def __radd__(self, other ):
-        """Implementation of + operator when left operand is not a C{ParserElement}"""
+        """Implementation of + operator when left operand is not a C{L{ParserElement}}"""
         if isinstance( other, basestring ):
-            other = Literal( other )
+            other = ParserElement.literalStringClass( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
@@ -1154,9 +1230,9 @@ class ParserElement(object):
         return other + self
 
     def __sub__(self, other):
-        """Implementation of - operator, returns C{And} with error stop"""
+        """Implementation of - operator, returns C{L{And}} with error stop"""
         if isinstance( other, basestring ):
-            other = Literal( other )
+            other = ParserElement.literalStringClass( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
@@ -1164,9 +1240,9 @@ class ParserElement(object):
         return And( [ self, And._ErrorStop(), other ] )
 
     def __rsub__(self, other ):
-        """Implementation of - operator when left operand is not a C{ParserElement}"""
+        """Implementation of - operator when left operand is not a C{L{ParserElement}}"""
         if isinstance( other, basestring ):
-            other = Literal( other )
+            other = ParserElement.literalStringClass( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
@@ -1179,12 +1255,12 @@ class ParserElement(object):
            tuple, similar to C{{min,max}} multipliers in regular expressions.  Tuples
            may also include C{None} as in:
             - C{expr*(n,None)} or C{expr*(n,)} is equivalent
-              to C{expr*n + ZeroOrMore(expr)}
+              to C{expr*n + L{ZeroOrMore}(expr)}
               (read as "at least n instances of C{expr}")
             - C{expr*(None,n)} is equivalent to C{expr*(0,n)}
               (read as "0 to n instances of C{expr}")
-            - C{expr*(None,None)} is equivalent to C{ZeroOrMore(expr)}
-            - C{expr*(1,None)} is equivalent to C{OneOrMore(expr)}
+            - C{expr*(None,None)} is equivalent to C{L{ZeroOrMore}(expr)}
+            - C{expr*(1,None)} is equivalent to C{L{OneOrMore}(expr)}
 
            Note that C{expr*(None,n)} does not raise an exception if
            more than n exprs exist in the input stream; that is,
@@ -1245,9 +1321,9 @@ class ParserElement(object):
         return self.__mul__(other)
 
     def __or__(self, other ):
-        """Implementation of | operator - returns C{MatchFirst}"""
+        """Implementation of | operator - returns C{L{MatchFirst}}"""
         if isinstance( other, basestring ):
-            other = Literal( other )
+            other = ParserElement.literalStringClass( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
@@ -1255,9 +1331,9 @@ class ParserElement(object):
         return MatchFirst( [ self, other ] )
 
     def __ror__(self, other ):
-        """Implementation of | operator when left operand is not a C{ParserElement}"""
+        """Implementation of | operator when left operand is not a C{L{ParserElement}}"""
         if isinstance( other, basestring ):
-            other = Literal( other )
+            other = ParserElement.literalStringClass( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
@@ -1265,9 +1341,9 @@ class ParserElement(object):
         return other | self
 
     def __xor__(self, other ):
-        """Implementation of ^ operator - returns C{Or}"""
+        """Implementation of ^ operator - returns C{L{Or}}"""
         if isinstance( other, basestring ):
-            other = Literal( other )
+            other = ParserElement.literalStringClass( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
@@ -1275,9 +1351,9 @@ class ParserElement(object):
         return Or( [ self, other ] )
 
     def __rxor__(self, other ):
-        """Implementation of ^ operator when left operand is not a C{ParserElement}"""
+        """Implementation of ^ operator when left operand is not a C{L{ParserElement}}"""
         if isinstance( other, basestring ):
-            other = Literal( other )
+            other = ParserElement.literalStringClass( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
@@ -1285,9 +1361,9 @@ class ParserElement(object):
         return other ^ self
 
     def __and__(self, other ):
-        """Implementation of & operator - returns C{Each}"""
+        """Implementation of & operator - returns C{L{Each}}"""
         if isinstance( other, basestring ):
-            other = Literal( other )
+            other = ParserElement.literalStringClass( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
@@ -1295,9 +1371,9 @@ class ParserElement(object):
         return Each( [ self, other ] )
 
     def __rand__(self, other ):
-        """Implementation of & operator when left operand is not a C{ParserElement}"""
+        """Implementation of & operator when left operand is not a C{L{ParserElement}}"""
         if isinstance( other, basestring ):
-            other = Literal( other )
+            other = ParserElement.literalStringClass( other )
         if not isinstance( other, ParserElement ):
             warnings.warn("Cannot combine element of type %s with ParserElement" % type(other),
                     SyntaxWarning, stacklevel=2)
@@ -1305,19 +1381,24 @@ class ParserElement(object):
         return other & self
 
     def __invert__( self ):
-        """Implementation of ~ operator - returns C{NotAny}"""
+        """Implementation of ~ operator - returns C{L{NotAny}}"""
         return NotAny( self )
 
-    def __call__(self, name):
-        """Shortcut for C{setResultsName}, with C{listAllMatches=default}::
+    def __call__(self, name=None):
+        """Shortcut for C{L{setResultsName}}, with C{listAllMatches=default}::
              userdata = Word(alphas).setResultsName("name") + Word(nums+"-").setResultsName("socsecno")
            could be written as::
              userdata = Word(alphas)("name") + Word(nums+"-")("socsecno")
              
            If C{name} is given with a trailing C{'*'} character, then C{listAllMatches} will be
            passed as C{True}.
+           
+           If C{name} is omitted, same as calling C{L{copy}}.
            """
-        return self.setResultsName(name)
+        if name is not None:
+            return self.setResultsName(name)
+        else:
+            return self.copy()
 
     def suppress( self ):
         """Suppresses the output of this C{ParserElement}; useful to keep punctuation from
@@ -1403,25 +1484,17 @@ class ParserElement(object):
         try:
             file_contents = file_or_filename.read()
         except AttributeError:
-            f = open(file_or_filename, "rb")
+            f = open(file_or_filename, "r")
             file_contents = f.read()
             f.close()
         try:
             return self.parseString(file_contents, parseAll)
-        except ParseBaseException:
-            # catch and re-raise exception from here, clears out pyparsing internal stack trace
-            exc = sys.exc_info()[1]
-            raise exc
-
-    def getException(self):
-        return ParseException("",0,self.errmsg,self)
-
-    def __getattr__(self,aname):
-        if aname == "myException":
-            self.myException = ret = self.getException();
-            return ret;
-        else:
-            raise AttributeError("no such attribute " + aname)
+        except ParseBaseException as exc:
+            if ParserElement.verbose_stacktrace:
+                raise
+            else:
+                # catch and re-raise exception from here, clears out pyparsing internal stack trace
+                raise exc
 
     def __eq__(self,other):
         if isinstance(other, ParserElement):
@@ -1478,10 +1551,7 @@ class NoMatch(Token):
         self.errmsg = "Unmatchable token"
 
     def parseImpl( self, instring, loc, doActions=True ):
-        exc = self.myException
-        exc.loc = loc
-        exc.pstr = instring
-        raise exc
+        raise ParseException(instring, loc, self.errmsg, self)
 
 
 class Literal(Token):
@@ -1509,16 +1579,13 @@ class Literal(Token):
         if (instring[loc] == self.firstMatchChar and
             (self.matchLen==1 or instring.startswith(self.match,loc)) ):
             return loc+self.matchLen, self.match
-        #~ raise ParseException( instring, loc, self.errmsg )
-        exc = self.myException
-        exc.loc = loc
-        exc.pstr = instring
-        raise exc
+        raise ParseException(instring, loc, self.errmsg, self)
 _L = Literal
+ParserElement.literalStringClass = Literal
 
 class Keyword(Token):
     """Token to exactly match a specified string as a keyword, that is, it must be
-       immediately followed by a non-keyword character.  Compare with C{Literal}::
+       immediately followed by a non-keyword character.  Compare with C{L{Literal}}::
          Literal("if") will match the leading C{'if'} in C{'ifAndOnlyIf'}.
          Keyword("if") will not; it will only match the leading C{'if'} in C{'if x=1'}, or C{'if(y==2)'}
        Accepts two optional constructor arguments in addition to the keyword string:
@@ -1559,11 +1626,7 @@ class Keyword(Token):
                 (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen] not in self.identChars) and
                 (loc == 0 or instring[loc-1] not in self.identChars) ):
                 return loc+self.matchLen, self.match
-        #~ raise ParseException( instring, loc, self.errmsg )
-        exc = self.myException
-        exc.loc = loc
-        exc.pstr = instring
-        raise exc
+        raise ParseException(instring, loc, self.errmsg, self)
 
     def copy(self):
         c = super(Keyword,self).copy()
@@ -1591,11 +1654,7 @@ class CaselessLiteral(Literal):
     def parseImpl( self, instring, loc, doActions=True ):
         if instring[ loc:loc+self.matchLen ].upper() == self.match:
             return loc+self.matchLen, self.returnString
-        #~ raise ParseException( instring, loc, self.errmsg )
-        exc = self.myException
-        exc.loc = loc
-        exc.pstr = instring
-        raise exc
+        raise ParseException(instring, loc, self.errmsg, self)
 
 class CaselessKeyword(Keyword):
     def __init__( self, matchString, identChars=Keyword.DEFAULT_KEYWORD_CHARS ):
@@ -1605,11 +1664,7 @@ class CaselessKeyword(Keyword):
         if ( (instring[ loc:loc+self.matchLen ].upper() == self.caselessmatch) and
              (loc >= len(instring)-self.matchLen or instring[loc+self.matchLen].upper() not in self.identChars) ):
             return loc+self.matchLen, self.match
-        #~ raise ParseException( instring, loc, self.errmsg )
-        exc = self.myException
-        exc.loc = loc
-        exc.pstr = instring
-        raise exc
+        raise ParseException(instring, loc, self.errmsg, self)
 
 class Word(Token):
     """Token for matching words composed of allowed character sets.
@@ -1626,9 +1681,9 @@ class Word(Token):
     def __init__( self, initChars, bodyChars=None, min=1, max=0, exact=0, asKeyword=False, excludeChars=None ):
         super(Word,self).__init__()
         if excludeChars:
-            initChars = ''.join([c for c in initChars if c not in excludeChars])
+            initChars = ''.join(c for c in initChars if c not in excludeChars)
             if bodyChars:
-                bodyChars = ''.join([c for c in bodyChars if c not in excludeChars])
+                bodyChars = ''.join(c for c in bodyChars if c not in excludeChars)
         self.initCharsOrig = initChars
         self.initChars = set(initChars)
         if bodyChars :
@@ -1681,20 +1736,14 @@ class Word(Token):
         if self.re:
             result = self.re.match(instring,loc)
             if not result:
-                exc = self.myException
-                exc.loc = loc
-                exc.pstr = instring
-                raise exc
+                raise ParseException(instring, loc, self.errmsg, self)
 
             loc = result.end()
             return loc, result.group()
 
         if not(instring[ loc ] in self.initChars):
-            #~ raise ParseException( instring, loc, self.errmsg )
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
+
         start = loc
         loc += 1
         instrlen = len(instring)
@@ -1714,11 +1763,7 @@ class Word(Token):
                 throwException = True
 
         if throwException:
-            #~ raise ParseException( instring, loc, self.errmsg )
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
 
         return loc, instring[start:loc]
 
@@ -1787,10 +1832,7 @@ class Regex(Token):
     def parseImpl( self, instring, loc, doActions=True ):
         result = self.re.match(instring,loc)
         if not result:
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
 
         loc = result.end()
         d = result.groupdict()
@@ -1821,9 +1863,9 @@ class QuotedString(Token):
             - quoteChar - string of one or more characters defining the quote delimiting string
             - escChar - character to escape quotes, typically backslash (default=None)
             - escQuote - special quote sequence to escape an embedded quote string (such as SQL's "" to escape an embedded ") (default=None)
-            - multiline - boolean indicating whether quotes can span multiple lines (default=False)
-            - unquoteResults - boolean indicating whether the matched text should be unquoted (default=True)
-            - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=None => same as quoteChar)
+            - multiline - boolean indicating whether quotes can span multiple lines (default=C{False})
+            - unquoteResults - boolean indicating whether the matched text should be unquoted (default=C{True})
+            - endQuoteChar - string of one or more characters defining the end of the quote delimited string (default=C{None} => same as quoteChar)
         """
         super(QuotedString,self).__init__()
 
@@ -1864,9 +1906,9 @@ class QuotedString(Token):
                   (escChar is not None and _escapeRegexRangeChars(escChar) or '') )
         if len(self.endQuoteChar) > 1:
             self.pattern += (
-                '|(?:' + ')|(?:'.join(["%s[^%s]" % (re.escape(self.endQuoteChar[:i]),
+                '|(?:' + ')|(?:'.join("%s[^%s]" % (re.escape(self.endQuoteChar[:i]),
                                                _escapeRegexRangeChars(self.endQuoteChar[i]))
-                                    for i in range(len(self.endQuoteChar)-1,0,-1)]) + ')'
+                                    for i in range(len(self.endQuoteChar)-1,0,-1)) + ')'
                 )
         if escQuote:
             self.pattern += (r'|(?:%s)' % re.escape(escQuote))
@@ -1892,10 +1934,7 @@ class QuotedString(Token):
     def parseImpl( self, instring, loc, doActions=True ):
         result = instring[loc] == self.firstQuoteChar and self.re.match(instring,loc) or None
         if not result:
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
 
         loc = result.end()
         ret = result.group()
@@ -1961,11 +2000,7 @@ class CharsNotIn(Token):
 
     def parseImpl( self, instring, loc, doActions=True ):
         if instring[loc] in self.notChars:
-            #~ raise ParseException( instring, loc, self.errmsg )
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
 
         start = loc
         loc += 1
@@ -1976,11 +2011,7 @@ class CharsNotIn(Token):
             loc += 1
 
         if loc - start < self.minLen:
-            #~ raise ParseException( instring, loc, self.errmsg )
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
 
         return loc, instring[start:loc]
 
@@ -2003,7 +2034,7 @@ class White(Token):
        by pyparsing grammars.  This class is included when some whitespace structures
        are significant.  Define with a string containing the whitespace characters to be
        matched; default is C{" \\t\\r\\n"}.  Also takes optional C{min}, C{max}, and C{exact} arguments,
-       as defined for the C{Word} class."""
+       as defined for the C{L{Word}} class."""
     whiteStrs = {
         " " : "<SPC>",
         "\t": "<TAB>",
@@ -2014,9 +2045,9 @@ class White(Token):
     def __init__(self, ws=" \t\r\n", min=1, max=0, exact=0):
         super(White,self).__init__()
         self.matchWhite = ws
-        self.setWhitespaceChars( "".join([c for c in self.whiteChars if c not in self.matchWhite]) )
+        self.setWhitespaceChars( "".join(c for c in self.whiteChars if c not in self.matchWhite) )
         #~ self.leaveWhitespace()
-        self.name = ("".join([White.whiteStrs[c] for c in self.matchWhite]))
+        self.name = ("".join(White.whiteStrs[c] for c in self.matchWhite))
         self.mayReturnEmpty = True
         self.errmsg = "Expected " + self.name
 
@@ -2033,11 +2064,7 @@ class White(Token):
 
     def parseImpl( self, instring, loc, doActions=True ):
         if not(instring[ loc ] in self.matchWhite):
-            #~ raise ParseException( instring, loc, self.errmsg )
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
         start = loc
         loc += 1
         maxloc = start + self.maxLen
@@ -2046,11 +2073,7 @@ class White(Token):
             loc += 1
 
         if loc - start < self.minLen:
-            #~ raise ParseException( instring, loc, self.errmsg )
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
 
         return loc, instring[start:loc]
 
@@ -2102,11 +2125,7 @@ class LineStart(_PositionToken):
         if not( loc==0 or
             (loc == self.preParse( instring, 0 )) or
             (instring[loc-1] == "\n") ): #col(loc, instring) != 1:
-            #~ raise ParseException( instring, loc, "Expected start of line" )
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
         return loc, []
 
 class LineEnd(_PositionToken):
@@ -2121,18 +2140,11 @@ class LineEnd(_PositionToken):
             if instring[loc] == "\n":
                 return loc+1, "\n"
             else:
-                #~ raise ParseException( instring, loc, "Expected end of line" )
-                exc = self.myException
-                exc.loc = loc
-                exc.pstr = instring
-                raise exc
+                raise ParseException(instring, loc, self.errmsg, self)
         elif loc == len(instring):
             return loc+1, []
         else:
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
 
 class StringStart(_PositionToken):
     """Matches if current position is at the beginning of the parse string"""
@@ -2144,11 +2156,7 @@ class StringStart(_PositionToken):
         if loc != 0:
             # see if entire string up to here is just whitespace and ignoreables
             if loc != self.preParse( instring, 0 ):
-                #~ raise ParseException( instring, loc, "Expected start of text" )
-                exc = self.myException
-                exc.loc = loc
-                exc.pstr = instring
-                raise exc
+                raise ParseException(instring, loc, self.errmsg, self)
         return loc, []
 
 class StringEnd(_PositionToken):
@@ -2159,20 +2167,13 @@ class StringEnd(_PositionToken):
 
     def parseImpl( self, instring, loc, doActions=True ):
         if loc < len(instring):
-            #~ raise ParseException( instring, loc, "Expected end of text" )
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
         elif loc == len(instring):
             return loc+1, []
         elif loc > len(instring):
             return loc, []
         else:
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
 
 class WordStart(_PositionToken):
     """Matches if the current position is at the beginning of a Word, and
@@ -2190,10 +2191,7 @@ class WordStart(_PositionToken):
         if loc != 0:
             if (instring[loc-1] in self.wordChars or
                 instring[loc] not in self.wordChars):
-                exc = self.myException
-                exc.loc = loc
-                exc.pstr = instring
-                raise exc
+                raise ParseException(instring, loc, self.errmsg, self)
         return loc, []
 
 class WordEnd(_PositionToken):
@@ -2214,11 +2212,7 @@ class WordEnd(_PositionToken):
         if instrlen>0 and loc<instrlen:
             if (instring[loc] in self.wordChars or
                 instring[loc-1] not in self.wordChars):
-                #~ raise ParseException( instring, loc, "Expected end of word" )
-                exc = self.myException
-                exc.loc = loc
-                exc.pstr = instring
-                raise exc
+                raise ParseException(instring, loc, self.errmsg, self)
         return loc, []
 
 
@@ -2226,10 +2220,16 @@ class ParseExpression(ParserElement):
     """Abstract subclass of ParserElement, for combining and post-processing parsed tokens."""
     def __init__( self, exprs, savelist = False ):
         super(ParseExpression,self).__init__(savelist)
-        if isinstance( exprs, list ):
-            self.exprs = exprs
-        elif isinstance( exprs, basestring ):
+        if isinstance( exprs, _generatorType ):
+            exprs = list(exprs)
+
+        if isinstance( exprs, basestring ):
             self.exprs = [ Literal( exprs ) ]
+        elif isinstance( exprs, collections.Sequence ):
+            # if sequence of strings provided, wrap with Literal
+            if all(isinstance(expr, basestring) for expr in exprs):
+                exprs = map(Literal, exprs)
+            self.exprs = list(exprs)
         else:
             try:
                 self.exprs = list( exprs )
@@ -2331,16 +2331,13 @@ class And(ParseExpression):
 
     class _ErrorStop(Empty):
         def __init__(self, *args, **kwargs):
-            super(Empty,self).__init__(*args, **kwargs)
+            super(And._ErrorStop,self).__init__(*args, **kwargs)
+            self.name = '-'
             self.leaveWhitespace()
 
     def __init__( self, exprs, savelist = True ):
         super(And,self).__init__(exprs, savelist)
-        self.mayReturnEmpty = True
-        for e in self.exprs:
-            if not e.mayReturnEmpty:
-                self.mayReturnEmpty = False
-                break
+        self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs)
         self.setWhitespaceChars( exprs[0].whiteChars )
         self.skipWhitespace = exprs[0].skipWhitespace
         self.callPreparse = True
@@ -2359,14 +2356,14 @@ class And(ParseExpression):
                     loc, exprtokens = e._parse( instring, loc, doActions )
                 except ParseSyntaxException:
                     raise
-                except ParseBaseException:
-                    pe = sys.exc_info()[1]
+                except ParseBaseException as pe:
+                    pe.__traceback__ = None
                     raise ParseSyntaxException(pe)
                 except IndexError:
                     raise ParseSyntaxException( ParseException(instring, len(instring), self.errmsg, self) )
             else:
                 loc, exprtokens = e._parse( instring, loc, doActions )
-            if exprtokens or exprtokens.keys():
+            if exprtokens or exprtokens.haskeys():
                 resultlist += exprtokens
         return loc, resultlist
 
@@ -2387,7 +2384,7 @@ class And(ParseExpression):
             return self.name
 
         if self.strRepr is None:
-            self.strRepr = "{" + " ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
+            self.strRepr = "{" + " ".join(_ustr(e) for e in self.exprs) + "}"
 
         return self.strRepr
 
@@ -2399,11 +2396,10 @@ class Or(ParseExpression):
     """
     def __init__( self, exprs, savelist = False ):
         super(Or,self).__init__(exprs, savelist)
-        self.mayReturnEmpty = False
-        for e in self.exprs:
-            if e.mayReturnEmpty:
-                self.mayReturnEmpty = True
-                break
+        if self.exprs:
+            self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs)
+        else:
+            self.mayReturnEmpty = True
 
     def parseImpl( self, instring, loc, doActions=True ):
         maxExcLoc = -1
@@ -2412,8 +2408,8 @@ class Or(ParseExpression):
         for e in self.exprs:
             try:
                 loc2 = e.tryParse( instring, loc )
-            except ParseException:
-                err = sys.exc_info()[1]
+            except ParseException as err:
+                err.__traceback__ = None
                 if err.loc > maxExcLoc:
                     maxException = err
                     maxExcLoc = err.loc
@@ -2436,7 +2432,7 @@ class Or(ParseExpression):
 
     def __ixor__(self, other ):
         if isinstance( other, basestring ):
-            other = Literal( other )
+            other = ParserElement.literalStringClass( other )
         return self.append( other ) #Or( [ self, other ] )
 
     def __str__( self ):
@@ -2444,7 +2440,7 @@ class Or(ParseExpression):
             return self.name
 
         if self.strRepr is None:
-            self.strRepr = "{" + " ^ ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
+            self.strRepr = "{" + " ^ ".join(_ustr(e) for e in self.exprs) + "}"
 
         return self.strRepr
 
@@ -2461,12 +2457,8 @@ class MatchFirst(ParseExpression):
     """
     def __init__( self, exprs, savelist = False ):
         super(MatchFirst,self).__init__(exprs, savelist)
-        if exprs:
-            self.mayReturnEmpty = False
-            for e in self.exprs:
-                if e.mayReturnEmpty:
-                    self.mayReturnEmpty = True
-                    break
+        if self.exprs:
+            self.mayReturnEmpty = any(e.mayReturnEmpty for e in self.exprs)
         else:
             self.mayReturnEmpty = True
 
@@ -2477,7 +2469,7 @@ class MatchFirst(ParseExpression):
             try:
                 ret = e._parse( instring, loc, doActions )
                 return ret
-            except ParseException, err:
+            except ParseException as err:
                 if err.loc > maxExcLoc:
                     maxException = err
                     maxExcLoc = err.loc
@@ -2495,7 +2487,7 @@ class MatchFirst(ParseExpression):
 
     def __ior__(self, other ):
         if isinstance( other, basestring ):
-            other = Literal( other )
+            other = ParserElement.literalStringClass( other )
         return self.append( other ) #MatchFirst( [ self, other ] )
 
     def __str__( self ):
@@ -2503,7 +2495,7 @@ class MatchFirst(ParseExpression):
             return self.name
 
         if self.strRepr is None:
-            self.strRepr = "{" + " | ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
+            self.strRepr = "{" + " | ".join(_ustr(e) for e in self.exprs) + "}"
 
         return self.strRepr
 
@@ -2520,11 +2512,7 @@ class Each(ParseExpression):
     """
     def __init__( self, exprs, savelist = True ):
         super(Each,self).__init__(exprs, savelist)
-        self.mayReturnEmpty = True
-        for e in self.exprs:
-            if not e.mayReturnEmpty:
-                self.mayReturnEmpty = False
-                break
+        self.mayReturnEmpty = all(e.mayReturnEmpty for e in self.exprs)
         self.skipWhitespace = True
         self.initExprGroups = True
 
@@ -2562,7 +2550,7 @@ class Each(ParseExpression):
                 keepMatching = False
 
         if tmpReqd:
-            missing = ", ".join( [ _ustr(e) for e in tmpReqd ] )
+            missing = ", ".join(_ustr(e) for e in tmpReqd)
             raise ParseException(instring,loc,"Missing one or more required elements (%s)" % missing )
 
         # add any unmatched Optionals, in case they have default values defined
@@ -2577,7 +2565,7 @@ class Each(ParseExpression):
         for r in resultlist:
             dups = {}
             for k in r.keys():
-                if k in finalResults.keys():
+                if k in finalResults:
                     tmp = ParseResults(finalResults[k])
                     tmp += ParseResults(r[k])
                     dups[k] = tmp
@@ -2591,7 +2579,7 @@ class Each(ParseExpression):
             return self.name
 
         if self.strRepr is None:
-            self.strRepr = "{" + " & ".join( [ _ustr(e) for e in self.exprs ] ) + "}"
+            self.strRepr = "{" + " & ".join(_ustr(e) for e in self.exprs) + "}"
 
         return self.strRepr
 
@@ -2706,11 +2694,7 @@ class NotAny(ParseElementEnhance):
         except (ParseException,IndexError):
             pass
         else:
-            #~ raise ParseException(instring, loc, self.errmsg )
-            exc = self.myException
-            exc.loc = loc
-            exc.pstr = instring
-            raise exc
+            raise ParseException(instring, loc, self.errmsg, self)
         return loc, []
 
     def __str__( self ):
@@ -2740,7 +2724,7 @@ class ZeroOrMore(ParseElementEnhance):
                 else:
                     preloc = loc
                 loc, tmptokens = self.expr._parse( instring, preloc, doActions )
-                if tmptokens or tmptokens.keys():
+                if tmptokens or tmptokens.haskeys():
                     tokens += tmptokens
         except (ParseException,IndexError):
             pass
@@ -2775,7 +2759,7 @@ class OneOrMore(ParseElementEnhance):
                 else:
                     preloc = loc
                 loc, tmptokens = self.expr._parse( instring, preloc, doActions )
-                if tmptokens or tmptokens.keys():
+                if tmptokens or tmptokens.haskeys():
                     tokens += tmptokens
         except (ParseException,IndexError):
             pass
@@ -2809,8 +2793,8 @@ class Optional(ParseElementEnhance):
        A default return string can also be specified, if the optional expression
        is not found.
     """
-    def __init__( self, exprs, default=_optionalNotMatched ):
-        super(Optional,self).__init__( exprs, savelist=False )
+    def __init__( self, expr, default=_optionalNotMatched ):
+        super(Optional,self).__init__( expr, savelist=False )
         self.defaultValue = default
         self.mayReturnEmpty = True
 
@@ -2878,7 +2862,7 @@ class SkipTo(ParseElementEnhance):
                     while 1:
                         try:
                             loc = self.ignoreExpr.tryParse(instring,loc)
-                            # print "found ignoreExpr, advance to", loc
+                            # print("found ignoreExpr, advance to", loc)
                         except ParseBaseException:
                             break
                 expr._parse( instring, loc, doActions=False, callPreParse=False )
@@ -2898,10 +2882,7 @@ class SkipTo(ParseElementEnhance):
                     raise
                 else:
                     loc += 1
-        exc = self.myException
-        exc.loc = loc
-        exc.pstr = instring
-        raise exc
+        raise ParseException(instring, loc, self.errmsg, self)
 
 class Forward(ParseElementEnhance):
     """Forward declaration of an expression to be defined later -
@@ -2916,13 +2897,14 @@ class Forward(ParseElementEnhance):
        thereby leaving b and c out as parseable alternatives.  It is recommended that you
        explicitly group the values inserted into the C{Forward}::
           fwdExpr << (a | b | c)
+       Converting to use the '<<=' operator instead will avoid this problem.
     """
     def __init__( self, other=None ):
         super(Forward,self).__init__( other, savelist=False )
 
     def __lshift__( self, other ):
         if isinstance( other, basestring ):
-            other = Literal(other)
+            other = ParserElement.literalStringClass(other)
         self.expr = other
         self.mayReturnEmpty = other.mayReturnEmpty
         self.strRepr = None
@@ -2932,8 +2914,11 @@ class Forward(ParseElementEnhance):
         self.skipWhitespace = self.expr.skipWhitespace
         self.saveAsList = self.expr.saveAsList
         self.ignoreExprs.extend(self.expr.ignoreExprs)
-        return None
-
+        return self
+        
+    def __ilshift__(self, other):
+        return self << other
+    
     def leaveWhitespace( self ):
         self.skipWhitespace = False
         return self
@@ -2972,7 +2957,7 @@ class Forward(ParseElementEnhance):
             return super(Forward,self).copy()
         else:
             ret = Forward()
-            ret << self
+            ret <<= self
             return ret
 
 class _ForwardNoRecurse(Forward):
@@ -2993,7 +2978,7 @@ class Upcase(TokenConverter):
                        DeprecationWarning,stacklevel=2)
 
     def postParse( self, instring, loc, tokenlist ):
-        return list(map( string.upper, tokenlist ))
+        return list(map( str.upper, tokenlist ))
 
 
 class Combine(TokenConverter):
@@ -3023,13 +3008,13 @@ class Combine(TokenConverter):
         del retToks[:]
         retToks += ParseResults([ "".join(tokenlist._asStringList(self.joinString)) ], modal=self.modalResults)
 
-        if self.resultsName and len(retToks.keys())>0:
+        if self.resultsName and retToks.haskeys():
             return [ retToks ]
         else:
             return retToks
 
 class Group(TokenConverter):
-    """Converter to return the matched tokens as a list - useful for returning tokens of C{ZeroOrMore} and C{OneOrMore} expressions."""
+    """Converter to return the matched tokens as a list - useful for returning tokens of C{L{ZeroOrMore}} and C{L{OneOrMore}} expressions."""
     def __init__( self, expr ):
         super(Group,self).__init__( expr )
         self.saveAsList = True
@@ -3042,8 +3027,8 @@ class Dict(TokenConverter):
        Each element can also be referenced using the first token in the expression as its key.
        Useful for tabular report scraping when the first column can be used as a item key.
     """
-    def __init__( self, exprs ):
-        super(Dict,self).__init__( exprs )
+    def __init__( self, expr ):
+        super(Dict,self).__init__( expr )
         self.saveAsList = True
 
     def postParse( self, instring, loc, tokenlist ):
@@ -3060,7 +3045,7 @@ class Dict(TokenConverter):
             else:
                 dictvalue = tok.copy() #ParseResults(i)
                 del dictvalue[0]
-                if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.keys()):
+                if len(dictvalue)!= 1 or (isinstance(dictvalue,ParseResults) and dictvalue.haskeys()):
                     tokenlist[ikey] = _ParseResultsWithOffset(dictvalue,i)
                 else:
                     tokenlist[ikey] = _ParseResultsWithOffset(dictvalue[0],i)
@@ -3105,8 +3090,7 @@ def traceParseAction(f):
         sys.stderr.write( ">>entering %s(line: '%s', %d, %s)\n" % (thisFunc,line(l,s),l,t) )
         try:
             ret = f(*paArgs)
-        except Exception:
-            exc = sys.exc_info()[1]
+        except Exception as exc:
             sys.stderr.write( "<<leaving %s (exception: %s)\n" % (thisFunc,exc) )
             raise
         sys.stderr.write( "<<leaving %s (ret: %s)\n" % (thisFunc,ret) )
@@ -3124,7 +3108,7 @@ def delimitedList( expr, delim=",", combine=False ):
     """Helper to define a delimited list of expressions - the delimiter defaults to ','.
        By default, the list elements and delimiters can have intervening whitespace, and
        comments, but this can be overridden by passing C{combine=True} in the constructor.
-       If C{combine} is set to True, the matching tokens are returned as a single token
+       If C{combine} is set to C{True}, the matching tokens are returned as a single token
        string, with the delimiters included; otherwise, the matching tokens are returned
        as a list of tokens, with the delimiters suppressed.
     """
@@ -3204,7 +3188,7 @@ def matchPreviousExpr(expr):
     """
     rep = Forward()
     e2 = expr.copy()
-    rep << e2
+    rep <<= e2
     def copyTokenToRepeater(s,l,t):
         matchTokens = _flatten(t.asList())
         def mustMatchTheseTokens(s,l,t):
@@ -3226,7 +3210,7 @@ def _escapeRegexRangeChars(s):
 def oneOf( strs, caseless=False, useRegex=True ):
     """Helper to quickly define a set of alternative Literals, and makes sure to do
        longest-first testing when there is a conflict, regardless of the input order,
-       but returns a C{MatchFirst} for best performance.
+       but returns a C{L{MatchFirst}} for best performance.
 
        Parameters:
         - strs - a string of space-delimited literals, or a list of string literals
@@ -3244,10 +3228,12 @@ def oneOf( strs, caseless=False, useRegex=True ):
         masks = ( lambda a,b: b.startswith(a) )
         parseElementClass = Literal
 
-    if isinstance(strs,(list,tuple)):
-        symbols = list(strs[:])
-    elif isinstance(strs,basestring):
+    if isinstance(strs,basestring):
         symbols = strs.split()
+    elif isinstance(strs, collections.Sequence):
+        symbols = list(strs[:])
+    elif isinstance(strs, _generatorType):
+        symbols = list(strs)
     else:
         warnings.warn("Invalid argument to oneOf, expected string or list",
                 SyntaxWarning, stacklevel=2)
@@ -3271,9 +3257,9 @@ def oneOf( strs, caseless=False, useRegex=True ):
         #~ print (strs,"->", "|".join( [ _escapeRegexChars(sym) for sym in symbols] ))
         try:
             if len(symbols)==len("".join(symbols)):
-                return Regex( "[%s]" % "".join( [ _escapeRegexRangeChars(sym) for sym in symbols] ) )
+                return Regex( "[%s]" % "".join(_escapeRegexRangeChars(sym) for sym in symbols) )
             else:
-                return Regex( "|".join( [ re.escape(sym) for sym in symbols] ) )
+                return Regex( "|".join(re.escape(sym) for sym in symbols) )
         except:
             warnings.warn("Exception creating Regex for oneOf, building MatchFirst",
                     SyntaxWarning, stacklevel=2)
@@ -3284,7 +3270,7 @@ def oneOf( strs, caseless=False, useRegex=True ):
 
 def dictOf( key, value ):
     """Helper to easily and clearly define a dictionary by specifying the respective patterns
-       for the key and value.  Takes care of defining the C{Dict}, C{ZeroOrMore}, and C{Group} tokens
+       for the key and value.  Takes care of defining the C{L{Dict}}, C{L{ZeroOrMore}}, and C{L{Group}} tokens
        in the proper order.  The key pattern can include delimiting markers or punctuation,
        as long as they are suppressed, thereby leaving the significant key text.  The value
        pattern can include named results, so that the C{Dict} results can include named token
@@ -3301,7 +3287,7 @@ def originalTextFor(expr, asString=True):
        string containing the original parsed text.  
        
        If the optional C{asString} argument is passed as C{False}, then the return value is a 
-       C{ParseResults} containing any results names that were originally matched, and a 
+       C{L{ParseResults}} containing any results names that were originally matched, and a 
        single token containing the original matched text from the input string.  So if 
        the expression passed to C{L{originalTextFor}} contains expressions with defined
        results names, you must set C{asString} to C{False} if you want to preserve those
@@ -3326,6 +3312,20 @@ def ungroup(expr):
        if all but one are non-empty."""
     return TokenConverter(expr).setParseAction(lambda t:t[0])
 
+def locatedExpr(expr):
+    """Helper to decorate a returned token with its starting and ending locations in the input string.
+       This helper adds the following results names:
+        - locn_start = location where matched expression begins
+        - locn_end = location where matched expression ends
+        - value = the actual parsed results
+
+       Be careful if the input text contains C{<TAB>} characters, you may want to call
+       C{L{ParserElement.parseWithTabs}}
+    """
+    locator = Empty().setParseAction(lambda s,l,t: l)
+    return Group(locator("locn_start") + expr("value") + locator.copy().leaveWhitespace()("locn_end"))
+
+
 # convenience constants for positional expressions
 empty       = Empty().setName("empty")
 lineStart   = LineStart().setName("lineStart")
@@ -3334,15 +3334,12 @@ stringStart = StringStart().setName("stringStart")
 stringEnd   = StringEnd().setName("stringEnd")
 
 _escapedPunc = Word( _bslash, r"\[]-*.$+^?()~ ", exact=2 ).setParseAction(lambda s,l,t:t[0][1])
-_printables_less_backslash = "".join([ c for c in printables if c not in  r"\]" ])
-_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],16)))
+_escapedHexChar = Regex(r"\\0?[xX][0-9a-fA-F]+").setParseAction(lambda s,l,t:unichr(int(t[0].lstrip(r'\0x'),16)))
 _escapedOctChar = Regex(r"\\0[0-7]+").setParseAction(lambda s,l,t:unichr(int(t[0][1:],8)))
-_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(_printables_less_backslash,exact=1)
+_singleChar = _escapedPunc | _escapedHexChar | _escapedOctChar | Word(printables, excludeChars=r'\]', exact=1)
 _charRange = Group(_singleChar + Suppress("-") + _singleChar)
 _reBracketExpr = Literal("[") + Optional("^").setResultsName("negate") + Group( OneOrMore( _charRange | _singleChar ) ).setResultsName("body") + "]"
 
-_expanded = lambda p: (isinstance(p,ParseResults) and ''.join([ unichr(c) for c in range(ord(p[0]),ord(p[1])+1) ]) or p)
-
 def srange(s):
     r"""Helper to easily define string ranges for use in Word construction.  Borrows
        syntax from regexp '[]' string range definitions::
@@ -3360,8 +3357,9 @@ def srange(s):
           a range of any of the above, separated by a dash ('a-z', etc.)
           any combination of the above ('aeiouy', 'a-zA-Z0-9_$', etc.)
     """
+    _expanded = lambda p: p if not isinstance(p,ParseResults) else ''.join(unichr(c) for c in range(ord(p[0]),ord(p[1])+1))
     try:
-        return "".join([_expanded(part) for part in _reBracketExpr.parseString(s).body])
+        return "".join(_expanded(part) for part in _reBracketExpr.parseString(s).body)
     except:
         return ""
 
@@ -3376,7 +3374,7 @@ def matchOnlyAtCol(n):
 
 def replaceWith(replStr):
     """Helper method for common parse actions that simply return a literal value.  Especially
-       useful when used with C{transformString()}.
+       useful when used with C{L{transformString<ParserElement.transformString>}()}.
     """
     def _replFunc(*args):
         return [replStr]
@@ -3398,7 +3396,7 @@ def downcaseTokens(s,l,t):
     return [ tt.lower() for tt in map(_ustr,t) ]
 
 def keepOriginalText(s,startLoc,t):
-    """DEPRECATED - use new helper method C{originalTextFor}.
+    """DEPRECATED - use new helper method C{L{originalTextFor}}.
        Helper parse action to preserve original parsed text,
        overriding any nested parse actions."""
     try:
@@ -3440,7 +3438,7 @@ def _makeTags(tagStr, xml):
                 Dict(ZeroOrMore(Group( tagAttrName + Suppress("=") + tagAttrValue ))) + \
                 Optional("/",default=[False]).setResultsName("empty").setParseAction(lambda s,l,t:t[0]=='/') + Suppress(">")
     else:
-        printablesLessRAbrack = "".join( [ c for c in printables if c not in ">" ] )
+        printablesLessRAbrack = "".join(c for c in printables if c not in ">")
         tagAttrValue = quotedString.copy().setParseAction( removeQuotes ) | Word(printablesLessRAbrack)
         openTag = Suppress("<") + tagStr("tag") + \
                 Dict(ZeroOrMore(Group( tagAttrName.setParseAction(downcaseTokens) + \
@@ -3464,7 +3462,7 @@ def makeXMLTags(tagStr):
 
 def withAttribute(*args,**attrDict):
     """Helper to create a validating parse action to be used with start tags created
-       with C{makeXMLTags} or C{makeHTMLTags}. Use C{withAttribute} to qualify a starting tag
+       with C{L{makeXMLTags}} or C{L{makeHTMLTags}}. Use C{withAttribute} to qualify a starting tag
        with a required attribute value, to avoid false matches on common tags such as
        C{<TD>} or C{<DIV>}.
 
@@ -3499,7 +3497,7 @@ opAssoc = _Constants()
 opAssoc.LEFT = object()
 opAssoc.RIGHT = object()
 
-def operatorPrecedence( baseExpr, opList ):
+def infixNotation( baseExpr, opList, lpar=Suppress('('), rpar=Suppress(')') ):
     """Helper method for constructing grammars of expressions made up of
        operators working in a precedence hierarchy.  Operators may be unary or
        binary, left- or right-associative.  Parse actions can also be attached
@@ -3518,13 +3516,15 @@ def operatorPrecedence( baseExpr, opList ):
               be 1, 2, or 3)
            - rightLeftAssoc is the indicator whether the operator is
               right or left associative, using the pyparsing-defined
-              constants opAssoc.RIGHT and opAssoc.LEFT.
+              constants C{opAssoc.RIGHT} and C{opAssoc.LEFT}.
            - parseAction is the parse action to be associated with
               expressions matching this operator expression (the
               parse action tuple member may be omitted)
+        - lpar - expression for matching left-parentheses (default=Suppress('('))
+        - rpar - expression for matching right-parentheses (default=Suppress(')'))
     """
     ret = Forward()
-    lastExpr = baseExpr | ( Suppress('(') + ret + Suppress(')') )
+    lastExpr = baseExpr | ( lpar + ret + rpar )
     for i,operDef in enumerate(opList):
         opExpr,arity,rightLeftAssoc,pa = (operDef + (None,))[:4]
         if arity == 3:
@@ -3565,10 +3565,11 @@ def operatorPrecedence( baseExpr, opList ):
             raise ValueError("operator must indicate right or left associativity")
         if pa:
             matchExpr.setParseAction( pa )
-        thisExpr << ( matchExpr | lastExpr )
+        thisExpr <<= ( matchExpr | lastExpr )
         lastExpr = thisExpr
-    ret << lastExpr
+    ret <<= lastExpr
     return ret
+operatorPrecedence = infixNotation
 
 dblQuotedString = Regex(r'"(?:[^"\n\r\\]|(?:"")|(?:\\x[0-9a-fA-F]+)|(?:\\.))*"').setName("string enclosed in double quotes")
 sglQuotedString = Regex(r"'(?:[^'\n\r\\]|(?:'')|(?:\\x[0-9a-fA-F]+)|(?:\\.))*'").setName("string enclosed in single quotes")
@@ -3622,9 +3623,9 @@ def nestedExpr(opener="(", closer=")", content=None, ignoreExpr=quotedString.cop
             raise ValueError("opening and closing arguments must be strings if no content expression is given")
     ret = Forward()
     if ignoreExpr is not None:
-        ret << Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) )
+        ret <<= Group( Suppress(opener) + ZeroOrMore( ignoreExpr | ret | content ) + Suppress(closer) )
     else:
-        ret << Group( Suppress(opener) + ZeroOrMore( ret | content )  + Suppress(closer) )
+        ret <<= Group( Suppress(opener) + ZeroOrMore( ret | content )  + Suppress(closer) )
     return ret
 
 def indentedBlock(blockStatementExpr, indentStack, indent=True):
@@ -3697,8 +3698,7 @@ cppStyleComment = Regex(r"/(?:\*(?:[^*]*\*+)+?/|/[^\n]*(?:\n[^\n]*)*?(?:(?<!\\)|
 
 javaStyleComment = cppStyleComment
 pythonStyleComment = Regex(r"#.*").setName("Python style comment")
-_noncomma = "".join( [ c for c in printables if c != "," ] )
-_commasepitem = Combine(OneOrMore(Word(_noncomma) +
+_commasepitem = Combine(OneOrMore(Word(printables, excludeChars=',') +
                                   Optional( Word(" \t") +
                                             ~Literal(",") + ~LineEnd() ) ) ).streamline().setName("commaItem")
 commaSeparatedList = delimitedList( Optional( quotedString.copy() | _commasepitem, default="") ).setName("commaSeparatedList")
@@ -3715,8 +3715,7 @@ if __name__ == "__main__":
             print ("tokens.columns = " + str(tokens.columns))
             print ("tokens.tables = "  + str(tokens.tables))
             print (tokens.asXML("SQL",True))
-        except ParseBaseException:
-            err = sys.exc_info()[1]
+        except ParseBaseException as err:
             print (teststring + "->")
             print (err.line)
             print (" "*(err.column-1) + "^")
-- 
cgit v1.2.3


From 6d8431ab3e96568b3579a85e680371fd20c961aa Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 25 Oct 2014 16:20:23 +1300
Subject: Allow specification of multiple patterns from file and on command
 line

---
 libpathod/app.py      | 21 +++------------------
 libpathod/cmdline.py  |  4 +---
 libpathod/language.py |  9 +++++++--
 libpathod/pathoc.py   | 15 ++++++++++++---
 4 files changed, 23 insertions(+), 26 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/app.py b/libpathod/app.py
index 1a54f712..1910e80e 100644
--- a/libpathod/app.py
+++ b/libpathod/app.py
@@ -4,6 +4,8 @@ import version, language, utils
 from netlib import http_uastrings
 
 logging.basicConfig(level="DEBUG")
+
+
 def make_app(noapi):
     app = Flask(__name__)
 
@@ -14,20 +16,17 @@ def make_app(noapi):
                 version = version.IVERSION
             )
 
-
     @app.route('/api/log')
     def api_log():
         return jsonify(
             log = app.config["pathod"].get_log()
         )
 
-
     @app.route('/api/clear_log')
     def api_clear_log():
         app.config["pathod"].clear_log()
         return "OK"
 
-
     def render(s, cacheable, **kwargs):
         kwargs["noapi"] = app.config["pathod"].noapi
         kwargs["nocraft"] = app.config["pathod"].nocraft
@@ -37,30 +36,25 @@ def make_app(noapi):
             resp.headers["Cache-control"] = "public, max-age=4320"
         return resp
 
-
     @app.route('/')
     @app.route('/index.html')
     def index():
         return render("index.html", True, section="main")
 
-
     @app.route('/download')
     @app.route('/download.html')
     def download():
         return render("download.html", True, section="download", version=version.VERSION)
 
-
     @app.route('/about')
     @app.route('/about.html')
     def about():
         return render("about.html", True, section="about")
 
-
     @app.route('/docs/pathod')
     def docs_pathod():
         return render("docs_pathod.html", True, section="docs", subsection="pathod")
 
-
     @app.route('/docs/language')
     def docs_language():
         return render(
@@ -69,29 +63,24 @@ def make_app(noapi):
             subsection="lang"
         )
 
-
     @app.route('/docs/pathoc')
     def docs_pathoc():
         return render("docs_pathoc.html", True, section="docs", subsection="pathoc")
 
-
     @app.route('/docs/libpathod')
     def docs_libpathod():
         return render("docs_libpathod.html", True, section="docs", subsection="libpathod")
 
-
     @app.route('/docs/test')
     def docs_test():
         return render("docs_test.html", True, section="docs", subsection="test")
 
-
     @app.route('/log')
     def log():
         if app.config["pathod"].noapi:
             abort(404)
         return render("log.html", False, section="log", log=app.config["pathod"].get_log())
 
-
     @app.route('/log/<int:lid>')
     def onelog(lid):
         item = app.config["pathod"].log_by_id(int(lid))
@@ -100,7 +89,6 @@ def make_app(noapi):
         l = pprint.pformat(item)
         return render("onelog.html", False, section="log", alog=l, lid=lid)
 
-
     def _preview(is_request):
         if is_request:
             template = "request_preview.html"
@@ -121,7 +109,7 @@ def make_app(noapi):
 
         try:
             if is_request:
-                r = language.parse_request(spec)
+                r = language.parse_requests(spec)[0]
             else:
                 r = language.parse_response(spec)
         except language.ParseException, v:
@@ -144,14 +132,11 @@ def make_app(noapi):
         args["output"] = utils.escape_unprintables(s.getvalue())
         return render(template, False, **args)
 
-
     @app.route('/response_preview')
     def response_preview():
         return _preview(False)
 
-
     @app.route('/request_preview')
     def request_preview():
         return _preview(True)
     return app
-
diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py
index 1a000a93..affc4015 100644
--- a/libpathod/cmdline.py
+++ b/libpathod/cmdline.py
@@ -155,14 +155,12 @@ def go_pathoc():
             data = open(r).read()
             r = data
         try:
-            req = language.parse_request(r)
+            reqs.extend(language.parse_requests(r))
         except language.ParseException, v:
             print >> sys.stderr, "Error parsing request spec: %s"%v.msg
             print >> sys.stderr, v.marked()
             sys.exit(1)
-        reqs.append(req)
     args.request = reqs
-
     pathoc.main(args)
 
 
diff --git a/libpathod/language.py b/libpathod/language.py
index b4b59167..f40499f6 100644
--- a/libpathod/language.py
+++ b/libpathod/language.py
@@ -1030,7 +1030,7 @@ def parse_response(s):
         raise ParseException(v.msg, v.line, v.col)
 
 
-def parse_request(s):
+def parse_requests(s):
     """
         May raise ParseException
     """
@@ -1039,6 +1039,11 @@ def parse_request(s):
     except UnicodeError:
         raise ParseException("Spec must be valid ASCII.", 0, 0)
     try:
-        return Request(Request.expr().parseString(s, parseAll=True))
+        parts = pp.OneOrMore(
+            pp.Group(
+                Request.expr()
+            )
+        ).parseString(s, parseAll=True)
+        return [Request(i) for i in parts]
     except pp.ParseException, v:
         raise ParseException(v.msg, v.line, v.col)
diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py
index ae1e98cf..b9b202eb 100644
--- a/libpathod/pathoc.py
+++ b/libpathod/pathoc.py
@@ -18,8 +18,17 @@ class SSLInfo:
 
 
 class Response:
-    def __init__(self, httpversion, status_code, msg, headers, content, sslinfo):
-        self.httpversion, self.status_code, self.msg = httpversion, status_code, msg
+    def __init__(
+        self,
+        httpversion,
+        status_code,
+        msg,
+        headers,
+        content,
+        sslinfo
+    ):
+        self.httpversion, self.status_code = httpversion, status_code
+        self.msg = msg
         self.headers, self.content = headers, content
         self.sslinfo = sslinfo
 
@@ -91,7 +100,7 @@ class Pathoc(tcp.TCPClient):
             May raise language.ParseException, netlib.http.HttpError or
             language.FileAccessDenied.
         """
-        r = language.parse_request(spec)
+        r = language.parse_requests(spec)[0]
         language.serve(r, self.wfile, self.settings, self.address.host)
         self.wfile.flush()
         ret = list(http.read_response(self.rfile, r.method.string(), None))
-- 
cgit v1.2.3


From 384abbfdd5830467e4876ea164a49433084132ee Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 25 Oct 2014 16:34:19 +1300
Subject: Nicer port specification for pathoc

---
 libpathod/cmdline.py | 23 ++++++++++++++---------
 1 file changed, 14 insertions(+), 9 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py
index affc4015..3b1a86bf 100644
--- a/libpathod/cmdline.py
+++ b/libpathod/cmdline.py
@@ -38,23 +38,20 @@ def go_pathoc():
         "-n", dest='repeat', default=1, type=int, metavar="N",
         help='Repeat requests N times'
     )
-    parser.add_argument(
-        "-p", dest="port", type=int, default=None,
-        help="Port. Defaults to 80, or 443 if SSL is active"
-    )
     parser.add_argument(
         "-t", dest="timeout", type=int, default=None,
         help="Connection timeout"
     )
     parser.add_argument(
         'host', type=str,
-        help='Host to connect to'
+        metavar = "host[:port]",
+        help='Host and port to connect to'
     )
     parser.add_argument(
         'request', type=str, nargs="+",
         help="""
-        Request specification, or path to a file containing a request
-        specifcation
+        Request specification, or path to a file containing request
+        specifcations
         """
     )
     group = parser.add_argument_group(
@@ -127,10 +124,18 @@ def go_pathoc():
 
     args = parser.parse_args()
 
+    args.port = None
+    if ":" in args.host:
+        h, p = args.host.rsplit(":", 1)
+        try:
+            p = int(p)
+        except ValueError:
+            parser.error("Invalid port in host spec: %s" % args.host)
+        args.host = h
+        args.port = p
+
     if args.port is None:
         args.port = 443 if args.ssl else 80
-    else:
-        args.port = args.port
 
     try:
         args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i]
-- 
cgit v1.2.3


From ac4e9f8cb5a9541dcff0a86efdb04357b9bc027f Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 25 Oct 2014 16:43:01 +1300
Subject: Rationalize pathoc arguments, add -r flag to randomly select a
 request from the provided specs

---
 libpathod/cmdline.py | 17 ++++++++++++-----
 libpathod/pathoc.py  |  9 +++++++--
 2 files changed, 19 insertions(+), 7 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py
index 3b1a86bf..a21fa218 100644
--- a/libpathod/cmdline.py
+++ b/libpathod/cmdline.py
@@ -36,7 +36,14 @@ def go_pathoc():
     )
     parser.add_argument(
         "-n", dest='repeat', default=1, type=int, metavar="N",
-        help='Repeat requests N times'
+        help='Repeat N times'
+    )
+    parser.add_argument(
+        "-r", dest="random", action="store_true", default=False,
+        help="""
+        Select a random request from those specified. If this is not specified,
+        requests are all played in sequence.
+        """
     )
     parser.add_argument(
         "-t", dest="timeout", type=int, default=None,
@@ -48,7 +55,7 @@ def go_pathoc():
         help='Host and port to connect to'
     )
     parser.add_argument(
-        'request', type=str, nargs="+",
+        'requests', type=str, nargs="+",
         help="""
         Request specification, or path to a file containing request
         specifcations
@@ -110,7 +117,7 @@ def go_pathoc():
         help="Print full request"
     )
     group.add_argument(
-        "-r", dest="showresp", action="store_true", default=False,
+        "-p", dest="showresp", action="store_true", default=False,
         help="Print full response"
     )
     group.add_argument(
@@ -155,7 +162,7 @@ def go_pathoc():
         args.connect_to = None
 
     reqs = []
-    for r in args.request:
+    for r in args.requests:
         if os.path.exists(r):
             data = open(r).read()
             r = data
@@ -165,7 +172,7 @@ def go_pathoc():
             print >> sys.stderr, "Error parsing request spec: %s"%v.msg
             print >> sys.stderr, v.marked()
             sys.exit(1)
-    args.request = reqs
+    args.requests = reqs
     pathoc.main(args)
 
 
diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py
index b9b202eb..c85c207a 100644
--- a/libpathod/pathoc.py
+++ b/libpathod/pathoc.py
@@ -1,5 +1,6 @@
 import sys
 import os
+import random
 from netlib import tcp, http, certutils
 import netlib.utils
 
@@ -189,7 +190,7 @@ class Pathoc(tcp.TCPClient):
                 if resp:
                     self._show_summary(fp, *resp)
 
-            if self.sslinfo:
+            if showssl and self.sslinfo:
                 print >> fp, "Cipher: %s, %s bit, %s"%self.sslinfo.cipher
                 print >> fp, "SSL certificate chain:\n"
                 for i in self.sslinfo.certchain:
@@ -239,7 +240,11 @@ def main(args):
                 sys.exit(1)
             if args.timeout:
                 p.settimeout(args.timeout)
-            for spec in args.request:
+            if args.random:
+                playlist = [random.choice(args.requests)]
+            else:
+                playlist = args.requests
+            for spec in playlist:
                 ret = p.print_request(
                     spec,
                     showreq=args.showreq,
-- 
cgit v1.2.3


From 609d6eab30aac0601106993eb8c5d9a953ba402b Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 25 Oct 2014 17:27:08 +1300
Subject: Make grammar less ambiguous for multi-pattern files

---
 libpathod/language.py | 4 ++--
 1 file changed, 2 insertions(+), 2 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/language.py b/libpathod/language.py
index f40499f6..d8f201db 100644
--- a/libpathod/language.py
+++ b/libpathod/language.py
@@ -137,7 +137,7 @@ DATATYPES = dict(
 )
 
 
-v_integer = pp.Regex(r"\d+")\
+v_integer = pp.Word(pp.nums)\
     .setName("integer")\
     .setParseAction(lambda toks: int(toks[0]))
 
@@ -308,7 +308,7 @@ class ValueGenerate(_Token):
         u = reduce(
             operator.or_,
             [pp.Literal(i) for i in utils.SIZE_UNITS.keys()]
-        )
+        ).leaveWhitespace()
         e = e + pp.Optional(u, default=None)
 
         s = pp.Literal(",").suppress()
-- 
cgit v1.2.3


From fc4f9a1c7a0734a190b99265ca50d72014173859 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 25 Oct 2014 17:58:59 +1300
Subject: pathoc -n 0 repeats forever

---
 libpathod/cmdline.py | 2 +-
 libpathod/pathoc.py  | 6 +++++-
 2 files changed, 6 insertions(+), 2 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py
index a21fa218..f80b7aae 100644
--- a/libpathod/cmdline.py
+++ b/libpathod/cmdline.py
@@ -36,7 +36,7 @@ def go_pathoc():
     )
     parser.add_argument(
         "-n", dest='repeat', default=1, type=int, metavar="N",
-        help='Repeat N times'
+        help='Repeat N times. If 0 repeat for ever.'
     )
     parser.add_argument(
         "-r", dest="random", action="store_true", default=False,
diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py
index c85c207a..02d0c06d 100644
--- a/libpathod/pathoc.py
+++ b/libpathod/pathoc.py
@@ -224,7 +224,9 @@ class Pathoc(tcp.TCPClient):
 
 def main(args):
     try:
-        for i in range(args.repeat):
+        cnt = 0
+        while 1:
+            cnt += 1
             p = Pathoc(
                 (args.host, args.port),
                 ssl=args.ssl,
@@ -258,5 +260,7 @@ def main(args):
                 sys.stdout.flush()
                 if ret and args.oneshot:
                     sys.exit(0)
+            if cnt == args.repeat:
+                break
     except KeyboardInterrupt:
         pass
-- 
cgit v1.2.3


From c00ae41486de06192865f8539da0f00985a16a90 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sat, 25 Oct 2014 19:50:48 +1300
Subject: Add a memoize argument to prevent playing the same pattern twice

Also remove addition of Date header, which makes this non-deterministic
---
 libpathod/cmdline.py  |  8 ++++++++
 libpathod/language.py | 14 --------------
 libpathod/pathoc.py   | 28 +++++++++++++++++++++-------
 3 files changed, 29 insertions(+), 21 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py
index f80b7aae..fe5eed68 100644
--- a/libpathod/cmdline.py
+++ b/libpathod/cmdline.py
@@ -34,6 +34,14 @@ def go_pathoc():
         metavar = "HOST:PORT",
         help="Issue an HTTP CONNECT to connect to the specified host."
     )
+    parser.add_argument(
+        "-m", dest='memo', action="store_true", default=False,
+        help="""
+            Remember specs, and never play the same one twice. Note that this
+            means requests have to be rendered in memory, which means that large
+            generated data can cause issues.
+        """
+    )
     parser.add_argument(
         "-n", dest='repeat', default=1, type=int, metavar="N",
         help='Repeat N times. If 0 repeat for ever.'
diff --git a/libpathod/language.py b/libpathod/language.py
index d8f201db..56cbc18b 100644
--- a/libpathod/language.py
+++ b/libpathod/language.py
@@ -819,20 +819,6 @@ class _Message(object):
                             ValueLiteral(request_host)
                         )
                     )
-            else:
-                if not utils.get_header("Date", self.headers):
-                    tokens.append(
-                        Header(
-                            ValueLiteral("Date"),
-                            ValueLiteral(
-                                formatdate(
-                                    timeval=None,
-                                    localtime=False,
-                                    usegmt=True
-                                )
-                            )
-                        )
-                    )
         intermediate = self.__class__(tokens)
         return self.__class__([i.resolve(intermediate, settings) for i in tokens])
 
diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py
index 02d0c06d..0ff02b01 100644
--- a/libpathod/pathoc.py
+++ b/libpathod/pathoc.py
@@ -1,5 +1,6 @@
 import sys
 import os
+import hashlib
 import random
 from netlib import tcp, http, certutils
 import netlib.utils
@@ -148,9 +149,6 @@ class Pathoc(tcp.TCPClient):
 
             Returns True if we have a non-ignored response.
         """
-        if explain:
-            r = r.freeze(self.settings, self.address.host)
-
         resp, req = None, None
         if showreq:
             self.wfile.start_log()
@@ -223,10 +221,15 @@ class Pathoc(tcp.TCPClient):
 
 
 def main(args):
+    memo = set([])
     try:
         cnt = 0
         while 1:
             cnt += 1
+            if args.random:
+                playlist = [random.choice(args.requests)]
+            else:
+                playlist = args.requests
             p = Pathoc(
                 (args.host, args.port),
                 ssl=args.ssl,
@@ -235,6 +238,21 @@ def main(args):
                 clientcert=args.clientcert,
                 ciphers=args.ciphers
             )
+            if args.explain or args.memo:
+                playlist = [
+                    i.freeze(p.settings, p.address.host) for i in playlist
+                ]
+            if args.memo:
+                newlist = []
+                for spec in playlist:
+                    h = hashlib.sha256(spec.spec()).digest()
+                    if h not in memo:
+                        memo.add(h)
+                        newlist.append(spec)
+                playlist = newlist
+            if not playlist:
+                continue
+
             try:
                 p.connect(args.connect_to)
             except (tcp.NetLibError, PathocError), v:
@@ -242,10 +260,6 @@ def main(args):
                 sys.exit(1)
             if args.timeout:
                 p.settimeout(args.timeout)
-            if args.random:
-                playlist = [random.choice(args.requests)]
-            else:
-                playlist = args.requests
             for spec in playlist:
                 ret = p.print_request(
                     spec,
-- 
cgit v1.2.3


From fc1fc80469dca11ff0241c4b263e4b39e5506ddd Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sun, 26 Oct 2014 10:50:32 +1300
Subject: Allow nesting of pathod response specs in pathoc specs

This opens the door to really neat, repeatable, client-side driven
fuzzing, especially of proxies.
---
 libpathod/language.py | 48 ++++++++++++++++++++++++++++++++++++++++++++++--
 1 file changed, 46 insertions(+), 2 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/language.py b/libpathod/language.py
index 56cbc18b..d8e87145 100644
--- a/libpathod/language.py
+++ b/libpathod/language.py
@@ -1,3 +1,4 @@
+from __future__ import print_function
 import operator
 import string
 import random
@@ -6,7 +7,6 @@ import os
 import time
 import copy
 import abc
-from email.utils import formatdate
 import contrib.pyparsing as pp
 from netlib import http_status, tcp, http_uastrings
 
@@ -527,6 +527,43 @@ class Body(_Component):
         return Body(self.value.freeze(settings))
 
 
+class PathodSpec(_Token):
+    def __init__(self, value):
+        self.value = value
+        try:
+            self.parsed = Response(
+                Response.expr().parseString(
+                    value.val,
+                    parseAll=True
+                )
+            )
+        except pp.ParseException, v:
+            raise ParseException(v.msg, v.line, v.col)
+
+    @classmethod
+    def expr(klass):
+        e = pp.Literal("s").suppress()
+        e = e + ValueLiteral.expr()
+        return e.setParseAction(lambda x: klass(*x))
+
+    def values(self, settings):
+        return [
+            self.value.get_generator(settings),
+        ]
+
+    def quote(self, s):
+        quotechar = s[0]
+        s = s[1:-1]
+        s = s.replace(quotechar, "\\" + quotechar)
+        return quotechar + s + quotechar
+
+    def spec(self):
+        return "s%s"%(self.quote(self.value.spec()))
+
+    def freeze(self, settings):
+        return PathodSpec(ValueLiteral(self.parsed.freeze(settings).spec()))
+
+
 class Path(_Component):
     def __init__(self, value):
         if isinstance(value, basestring):
@@ -934,7 +971,8 @@ class Request(_Message):
         InjectAt,
         ShortcutContentType,
         ShortcutUserAgent,
-        Raw
+        Raw,
+        PathodSpec,
     )
     logattrs = ["method", "path", "body"]
 
@@ -946,10 +984,16 @@ class Request(_Message):
     def path(self):
         return self._get_token(Path)
 
+    @property
+    def pathodspec(self):
+        return self._get_token(PathodSpec)
+
     def preamble(self, settings):
         v = self.method.values(settings)
         v.append(" ")
         v.extend(self.path.values(settings))
+        if self.pathodspec:
+            v.append(self.pathodspec.parsed.spec())
         v.append(" ")
         v.append(self.version)
         return v
-- 
cgit v1.2.3


From 974bd9d0f9f3d231ff99496c55ae40355398cfc6 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sun, 26 Oct 2014 12:56:28 +1300
Subject: Resolve a quoting ambiguity in nested response specs

---
 libpathod/language.py | 21 +++++++++++----------
 1 file changed, 11 insertions(+), 10 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/language.py b/libpathod/language.py
index d8e87145..42fffcf8 100644
--- a/libpathod/language.py
+++ b/libpathod/language.py
@@ -1,4 +1,3 @@
-from __future__ import print_function
 import operator
 import string
 import random
@@ -15,6 +14,12 @@ import utils
 BLOCKSIZE = 1024
 TRUNCATE = 1024
 
+def quote(s):
+    quotechar = s[0]
+    s = s[1:-1]
+    s = s.replace(quotechar, "\\" + quotechar)
+    return quotechar + s + quotechar
+
 
 class FileAccessDenied(Exception):
     pass
@@ -272,7 +277,7 @@ class ValueLiteral(_ValueLiteral):
         return e.setParseAction(lambda x: klass(*x))
 
     def spec(self):
-        return '"%s"'%self.val.encode("string_escape")
+        return quote("'%s'"%self.val.encode("string_escape"))
 
 
 class ValueNakedLiteral(_ValueLiteral):
@@ -551,17 +556,13 @@ class PathodSpec(_Token):
             self.value.get_generator(settings),
         ]
 
-    def quote(self, s):
-        quotechar = s[0]
-        s = s[1:-1]
-        s = s.replace(quotechar, "\\" + quotechar)
-        return quotechar + s + quotechar
-
     def spec(self):
-        return "s%s"%(self.quote(self.value.spec()))
+        return "s%s"%(self.value.spec())
 
     def freeze(self, settings):
-        return PathodSpec(ValueLiteral(self.parsed.freeze(settings).spec()))
+        f = self.parsed.freeze(settings).spec()
+        print [f]
+        return PathodSpec(ValueLiteral(f))
 
 
 class Path(_Component):
-- 
cgit v1.2.3


From bd1f7ebb5c3cf3dfa613e194f4728bae1420b241 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sun, 26 Oct 2014 16:27:25 +1300
Subject: Improve netability of grammars

---
 libpathod/language.py | 22 ++++++++++++++++------
 1 file changed, 16 insertions(+), 6 deletions(-)

(limited to 'libpathod')

diff --git a/libpathod/language.py b/libpathod/language.py
index 42fffcf8..e1f6820f 100644
--- a/libpathod/language.py
+++ b/libpathod/language.py
@@ -14,6 +14,11 @@ import utils
 BLOCKSIZE = 1024
 TRUNCATE = 1024
 
+
+def escape_backslash(s):
+    return s.replace("\\", "\\\\")
+
+
 def quote(s):
     quotechar = s[0]
     s = s[1:-1]
@@ -186,7 +191,7 @@ class LiteralGenerator:
         return self.s.__getslice__(a, b)
 
     def __repr__(self):
-        return '"%s"'%self.s
+        return "'%s'"%self.s
 
 
 class RandomGenerator:
@@ -274,10 +279,16 @@ class ValueLiteral(_ValueLiteral):
     @classmethod
     def expr(klass):
         e = v_literal.copy()
-        return e.setParseAction(lambda x: klass(*x))
+        return e.setParseAction(klass.parseAction)
+
+    @classmethod
+    def parseAction(klass, x):
+        v = klass(*x)
+        return v
 
     def spec(self):
-        return quote("'%s'"%self.val.encode("string_escape"))
+        ret = "'%s'"%self.val.encode("string_escape")
+        return ret
 
 
 class ValueNakedLiteral(_ValueLiteral):
@@ -361,7 +372,7 @@ class ValueFile(_Token):
         return FileGenerator(s)
 
     def spec(self):
-        return '<"%s"'%self.path.encode("string_escape")
+        return "<'%s'"%self.path.encode("string_escape")
 
 
 Value = pp.MatchFirst(
@@ -561,8 +572,7 @@ class PathodSpec(_Token):
 
     def freeze(self, settings):
         f = self.parsed.freeze(settings).spec()
-        print [f]
-        return PathodSpec(ValueLiteral(f))
+        return PathodSpec(ValueLiteral(f.encode("string_escape")))
 
 
 class Path(_Component):
-- 
cgit v1.2.3


From 956149c126c0500aa67604996ca310824b2f23a2 Mon Sep 17 00:00:00 2001
From: Aldo Cortesi <aldo@nullcube.com>
Date: Sun, 26 Oct 2014 18:16:47 +1300
Subject: --memo-limit - limit failed attempts to find new generated requests
 to N

---
 libpathod/cmdline.py | 4 ++++
 libpathod/pathoc.py  | 7 +++++++
 2 files changed, 11 insertions(+)

(limited to 'libpathod')

diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py
index fe5eed68..4a2390ed 100644
--- a/libpathod/cmdline.py
+++ b/libpathod/cmdline.py
@@ -34,6 +34,10 @@ def go_pathoc():
         metavar = "HOST:PORT",
         help="Issue an HTTP CONNECT to connect to the specified host."
     )
+    parser.add_argument(
+        "--memo-limit", dest='memolimit', default=5000, type=int, metavar="N",
+        help='Stop if we do not find a valid request after N attempts.'
+    )
     parser.add_argument(
         "-m", dest='memo', action="store_true", default=False,
         help="""
diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py
index 0ff02b01..e7aff520 100644
--- a/libpathod/pathoc.py
+++ b/libpathod/pathoc.py
@@ -222,9 +222,14 @@ class Pathoc(tcp.TCPClient):
 
 def main(args):
     memo = set([])
+    trycount = 0
     try:
         cnt = 0
         while 1:
+            if trycount > args.memolimit:
+                print >> sys.stderr, "Memo limit exceeded..."
+                return
+
             cnt += 1
             if args.random:
                 playlist = [random.choice(args.requests)]
@@ -251,8 +256,10 @@ def main(args):
                         newlist.append(spec)
                 playlist = newlist
             if not playlist:
+                trycount += 1
                 continue
 
+            trycount = 0
             try:
                 p.connect(args.connect_to)
             except (tcp.NetLibError, PathocError), v:
-- 
cgit v1.2.3