aboutsummaryrefslogtreecommitdiffstats
diff options
context:
space:
mode:
authorAldo Cortesi <aldo@nullcube.com>2015-06-18 16:16:40 +1200
committerAldo Cortesi <aldo@nullcube.com>2015-06-18 16:16:40 +1200
commit274d0333f8bbd0bf88214747beeead991f36b72a (patch)
tree590d684d8ab9277cab441ffb07a79cc36412e4c7
parent78cb5fe573ffcc06e700bb2193f9aef212be267e (diff)
parent408b4ffef0a784bea7ec08c252e757bca6e28134 (diff)
downloadmitmproxy-274d0333f8bbd0bf88214747beeead991f36b72a.tar.gz
mitmproxy-274d0333f8bbd0bf88214747beeead991f36b72a.tar.bz2
mitmproxy-274d0333f8bbd0bf88214747beeead991f36b72a.zip
Merge pull request #27 from Kriechi/http2-wip
HTTP/2: add initial support
-rw-r--r--libpathod/cmdline.py442
-rw-r--r--libpathod/language/__init__.py14
-rw-r--r--libpathod/language/http.py6
-rw-r--r--libpathod/language/http2.py118
-rw-r--r--libpathod/pathoc.py19
-rw-r--r--libpathod/pathoc_cmdline.py224
-rw-r--r--libpathod/pathod.py242
-rw-r--r--libpathod/pathod_cmdline.py229
-rw-r--r--libpathod/utils.py10
-rwxr-xr-xpathoc3
-rwxr-xr-xpathod3
-rw-r--r--setup.py4
-rw-r--r--test/test_cmdline.py155
-rw-r--r--test/test_language_http2.py177
-rw-r--r--test/test_pathoc.py61
-rw-r--r--test/test_pathoc_cmdline.py64
-rw-r--r--test/test_pathod.py14
-rw-r--r--test/test_pathod_cmdline.py85
-rw-r--r--test/tutils.py8
19 files changed, 1157 insertions, 721 deletions
diff --git a/libpathod/cmdline.py b/libpathod/cmdline.py
deleted file mode 100644
index 06a6c533..00000000
--- a/libpathod/cmdline.py
+++ /dev/null
@@ -1,442 +0,0 @@
-#!/usr/bin/env python
-import sys
-import argparse
-import os
-import os.path
-import re
-from netlib import http_uastrings
-from . import pathoc, pathod, version, utils, language
-
-
-def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
- preparser = argparse.ArgumentParser(add_help=False)
- preparser.add_argument(
- "--show-uas", dest="showua", action="store_true", default=False,
- help="Print user agent shortcuts and exit."
- )
- pa = preparser.parse_known_args(argv)[0]
- if pa.showua:
- print >> stdout, "User agent strings:"
- for i in http_uastrings.UASTRINGS:
- print >> stdout, " ", i[1], i[0]
- sys.exit(0)
-
- parser = argparse.ArgumentParser(
- description='A perverse HTTP client.', parents=[preparser]
- )
- parser.add_argument(
- '--version',
- action='version',
- version="pathoc " + version.VERSION
- )
- parser.add_argument(
- "-c", dest="connect_to", type=str, default=False,
- metavar = "HOST:PORT",
- help="Issue an HTTP CONNECT to connect to the specified host."
- )
- parser.add_argument(
- "--memo-limit", dest='memolimit', default=5000, type=int, metavar="N",
- help='Stop if we do not find a valid request after N attempts.'
- )
- parser.add_argument(
- "-m", dest='memo', action="store_true", default=False,
- help="""
- Remember specs, and never play the same one twice. Note that this
- means requests have to be rendered in memory, which means that
- large generated data can cause issues.
- """
- )
- parser.add_argument(
- "-n", dest='repeat', default=1, type=int, metavar="N",
- help='Repeat N times. If 0 repeat for ever.'
- )
- parser.add_argument(
- "-w", dest='wait', default=0, type=float, metavar="N",
- help='Wait N seconds between each request.'
- )
- parser.add_argument(
- "-r", dest="random", action="store_true", default=False,
- help="""
- Select a random request from those specified. If this is not specified,
- requests are all played in sequence.
- """
- )
- parser.add_argument(
- "-t", dest="timeout", type=int, default=None,
- help="Connection timeout"
- )
- parser.add_argument(
- "--http2", dest="use_http2", action="store_true", default=False,
- help='Perform all requests over a single HTTP/2 connection.'
- )
- parser.add_argument(
- "--http2-skip-connection-preface",
- dest="http2_skip_connection_preface",
- action="store_true",
- default=False,
- help='Skips the HTTP/2 connection preface before sending requests.')
-
- parser.add_argument(
- 'host', type=str,
- metavar = "host[:port]",
- help='Host and port to connect to'
- )
- parser.add_argument(
- 'requests', type=str, nargs="+",
- help="""
- Request specification, or path to a file containing request
- specifcations
- """
- )
-
- group = parser.add_argument_group(
- 'SSL',
- )
- group.add_argument(
- "-s", dest="ssl", action="store_true", default=False,
- help="Connect with SSL"
- )
- group.add_argument(
- "-C", dest="clientcert", type=str, default=False,
- help="Path to a file containing client certificate and private key"
- )
- group.add_argument(
- "-i", dest="sni", type=str, default=False,
- help="SSL Server Name Indication"
- )
- group.add_argument(
- "--ciphers", dest="ciphers", type=str, default=False,
- help="SSL cipher specification"
- )
- group.add_argument(
- "--sslversion", dest="sslversion", type=int, default=4,
- choices=[1, 2, 3, 4],
- help="""
- Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default
- to SSLv23.
- """
- )
-
- group = parser.add_argument_group(
- 'Controlling Output',
- """
- Some of these options expand generated values for logging - if
- you're generating large data, use them with caution.
- """
- )
- group.add_argument(
- "-I", dest="ignorecodes", type=str, default="",
- help="Comma-separated list of response codes to ignore"
- )
- group.add_argument(
- "-S", dest="showssl", action="store_true", default=False,
- help="Show info on SSL connection"
- )
- group.add_argument(
- "-e", dest="explain", action="store_true", default=False,
- help="Explain requests"
- )
- group.add_argument(
- "-o", dest="oneshot", action="store_true", default=False,
- help="Oneshot - exit after first non-ignored response"
- )
- group.add_argument(
- "-q", dest="showreq", action="store_true", default=False,
- help="Print full request"
- )
- group.add_argument(
- "-p", dest="showresp", action="store_true", default=False,
- help="Print full response"
- )
- group.add_argument(
- "-T", dest="ignoretimeout", action="store_true", default=False,
- help="Ignore timeouts"
- )
- group.add_argument(
- "-x", dest="hexdump", action="store_true", default=False,
- help="Output in hexdump format"
- )
-
- args = parser.parse_args(argv[1:])
-
- args.port = None
- if ":" in args.host:
- h, p = args.host.rsplit(":", 1)
- try:
- p = int(p)
- except ValueError:
- return parser.error("Invalid port in host spec: %s" % args.host)
- args.host = h
- args.port = p
-
- if args.port is None:
- args.port = 443 if args.ssl else 80
-
- try:
- args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i]
- except ValueError:
- return parser.error(
- "Invalid return code specification: %s" %
- args.ignorecodes)
-
- if args.connect_to:
- parts = args.connect_to.split(":")
- if len(parts) != 2:
- return parser.error(
- "Invalid CONNECT specification: %s" %
- args.connect_to)
- try:
- parts[1] = int(parts[1])
- except ValueError:
- return parser.error(
- "Invalid CONNECT specification: %s" %
- args.connect_to)
- args.connect_to = parts
- else:
- args.connect_to = None
-
- reqs = []
- for r in args.requests:
- if os.path.isfile(r):
- data = open(r).read()
- r = data
- try:
- reqs.append(language.parse_pathoc(r, args.use_http2))
- except language.ParseException as v:
- print >> stderr, "Error parsing request spec: %s" % v.msg
- print >> stderr, v.marked()
- sys.exit(1)
- args.requests = reqs
- return args
-
-
-def go_pathoc(): # pragma: nocover
- args = args_pathoc(sys.argv)
- pathoc.main(args)
-
-
-def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr):
- parser = argparse.ArgumentParser(
- description='A pathological HTTP/S daemon.'
- )
- parser.add_argument(
- '--version',
- action='version',
- version="pathod " + version.VERSION
- )
- parser.add_argument(
- "-p",
- dest='port',
- default=9999,
- type=int,
- help='Port. Specify 0 to pick an arbitrary empty port. (9999)'
- )
- parser.add_argument(
- "-l",
- dest='address',
- default="127.0.0.1",
- type=str,
- help='Listening address. (127.0.0.1)'
- )
- parser.add_argument(
- "-a",
- dest='anchors',
- default=[],
- type=str,
- action="append",
- metavar="ANCHOR",
- help="""
- Add an anchor. Specified as a string with the form
- pattern=spec or pattern=filepath, where pattern is a regular
- expression.
- """
- )
- parser.add_argument(
- "-c", dest='craftanchor', default=pathod.DEFAULT_ANCHOR, type=str,
- help="""
- Regular expression specifying anchor point for URL crafting
- commands. (%s)
- """%pathod.DEFAULT_ANCHOR
- )
- parser.add_argument(
- "--confdir",
- action="store", type = str, dest="confdir", default='~/.mitmproxy',
- help = "Configuration directory. (~/.mitmproxy)"
- )
- parser.add_argument(
- "-d", dest='staticdir', default=None, type=str,
- help='Directory for static files.'
- )
- parser.add_argument(
- "-D", dest='daemonize', default=False, action="store_true",
- help='Daemonize.'
- )
- parser.add_argument(
- "-t", dest="timeout", type=int, default=None,
- help="Connection timeout"
- )
- parser.add_argument(
- "--limit-size",
- dest='sizelimit',
- default=None,
- type=str,
- help='Size limit of served responses. Understands size suffixes, i.e. 100k.')
- parser.add_argument(
- "--noapi", dest='noapi', default=False, action="store_true",
- help='Disable API.'
- )
- parser.add_argument(
- "--nohang", dest='nohang', default=False, action="store_true",
- help='Disable pauses during crafted response generation.'
- )
- parser.add_argument(
- "--noweb", dest='noweb', default=False, action="store_true",
- help='Disable both web interface and API.'
- )
- parser.add_argument(
- "--nocraft",
- dest='nocraft',
- default=False,
- action="store_true",
- help='Disable response crafting. If anchors are specified, they still work.')
- parser.add_argument(
- "--webdebug", dest='webdebug', default=False, action="store_true",
- help='Debugging mode for the web app (dev only).'
- )
-
- group = parser.add_argument_group(
- 'SSL',
- )
- group.add_argument(
- "-s", dest='ssl', default=False, action="store_true",
- help='Run in HTTPS mode.'
- )
- group.add_argument(
- "--cn",
- dest="cn",
- type=str,
- default=None,
- help="CN for generated SSL certs. Default: %s" %
- pathod.DEFAULT_CERT_DOMAIN)
- group.add_argument(
- "-C", dest='ssl_not_after_connect', default=False, action="store_true",
- help="Don't expect SSL after a CONNECT request."
- )
- group.add_argument(
- "--cert", dest='ssl_certs', default=[], type=str,
- metavar = "SPEC", action="append",
- help = """
- Add an SSL certificate. SPEC is of the form "[domain=]path". The domain
- may include a wildcard, and is equal to "*" if not specified. The file
- at path is a certificate in PEM format. If a private key is included in
- the PEM, it is used, else the default key in the conf dir is used. Can
- be passed multiple times.
- """
- )
- group.add_argument(
- "--ciphers", dest="ciphers", type=str, default=False,
- help="SSL cipher specification"
- )
- group.add_argument(
- "--san", dest="sans", type=str, default=[], action="append",
- metavar="SAN",
- help="""
- Subject Altnernate Name to add to the server certificate.
- May be passed multiple times.
- """
- )
- group.add_argument(
- "--sslversion", dest="sslversion", type=int, default=4,
- choices=[1, 2, 3, 4],
- help=""""Use a specified protocol - TLSv1, SSLv2, SSLv3, SSLv23. Default
- to SSLv23."""
- )
-
- group = parser.add_argument_group(
- 'Controlling Logging',
- """
- Some of these options expand generated values for logging - if
- you're generating large data, use them with caution.
- """
- )
- group.add_argument(
- "-e", dest="explain", action="store_true", default=False,
- help="Explain responses"
- )
- group.add_argument(
- "-f", dest='logfile', default=None, type=str,
- help='Log to file.'
- )
- group.add_argument(
- "-q", dest="logreq", action="store_true", default=False,
- help="Log full request"
- )
- group.add_argument(
- "-r", dest="logresp", action="store_true", default=False,
- help="Log full response"
- )
- group.add_argument(
- "-x", dest="hexdump", action="store_true", default=False,
- help="Log request/response in hexdump format"
- )
- args = parser.parse_args(argv[1:])
-
- certs = []
- for i in args.ssl_certs:
- parts = i.split("=", 1)
- if len(parts) == 1:
- parts = ["*", parts[0]]
- parts[1] = os.path.expanduser(parts[1])
- if not os.path.isfile(parts[1]):
- return parser.error(
- "Certificate file does not exist: %s" %
- parts[1])
- certs.append(parts)
- args.ssl_certs = certs
-
- alst = []
- for i in args.anchors:
- parts = utils.parse_anchor_spec(i)
- if not parts:
- return parser.error("Invalid anchor specification: %s" % i)
- alst.append(parts)
- args.anchors = alst
-
- sizelimit = None
- if args.sizelimit:
- try:
- sizelimit = utils.parse_size(args.sizelimit)
- except ValueError as v:
- return parser.error(v)
- args.sizelimit = sizelimit
-
- try:
- args.craftanchor = re.compile(args.craftanchor)
- except re.error:
- return parser.error(
- "Invalid regex in craft anchor: %s" % args.craftanchor
- )
-
- anchors = []
- for patt, spec in args.anchors:
- if os.path.isfile(spec):
- data = open(spec).read()
- spec = data
- try:
- req = language.parse_pathod(spec)
- except language.ParseException as v:
- print >> stderr, "Error parsing anchor spec: %s" % v.msg
- print >> stderr, v.marked()
- sys.exit(1)
- try:
- arex = re.compile(patt)
- except re.error:
- return parser.error("Invalid regex in anchor: %s" % patt)
- anchors.append((arex, req))
- args.anchors = anchors
- return args
-
-
-def go_pathod(): # pragma: nocover
- args = args_pathod(sys.argv)
- pathod.main(args)
diff --git a/libpathod/language/__init__.py b/libpathod/language/__init__.py
index ae9a8c76..10050bf8 100644
--- a/libpathod/language/__init__.py
+++ b/libpathod/language/__init__.py
@@ -19,7 +19,7 @@ def expand(msg):
yield msg
-def parse_pathod(s):
+def parse_pathod(s, use_http2=False):
"""
May raise ParseException
"""
@@ -28,12 +28,17 @@ def parse_pathod(s):
except UnicodeError:
raise exceptions.ParseException("Spec must be valid ASCII.", 0, 0)
try:
- reqs = pp.Or(
- [
+ if use_http2:
+ expressions = [
+ # http2.Frame.expr(),
+ http2.Response.expr(),
+ ]
+ else:
+ expressions = [
websockets.WebsocketFrame.expr(),
http.Response.expr(),
]
- ).parseString(s, parseAll=True)
+ reqs = pp.Or(expressions).parseString(s, parseAll=True)
except pp.ParseException as v:
raise exceptions.ParseException(v.msg, v.line, v.col)
return itertools.chain(*[expand(i) for i in reqs])
@@ -55,7 +60,6 @@ def parse_pathoc(s, use_http2=False):
websockets.WebsocketClientFrame.expr(),
http.Request.expr(),
]
-
reqs = pp.OneOrMore(pp.Or(expressions)).parseString(s, parseAll=True)
except pp.ParseException as v:
raise exceptions.ParseException(v.msg, v.line, v.col)
diff --git a/libpathod/language/http.py b/libpathod/language/http.py
index 9a8404f0..115f8069 100644
--- a/libpathod/language/http.py
+++ b/libpathod/language/http.py
@@ -367,10 +367,6 @@ class Request(_HTTPMessage):
return ":".join([i.spec() for i in self.tokens])
-class PathodErrorResponse(Response):
- pass
-
-
def make_error_response(reason, body=None):
tokens = [
Code("800"),
@@ -381,4 +377,4 @@ def make_error_response(reason, body=None):
Reason(base.TokValueLiteral(reason)),
Body(base.TokValueLiteral("pathod error: " + (body or reason))),
]
- return PathodErrorResponse(tokens)
+ return Response(tokens)
diff --git a/libpathod/language/http2.py b/libpathod/language/http2.py
index d78fc5c8..dec2d5fe 100644
--- a/libpathod/language/http2.py
+++ b/libpathod/language/http2.py
@@ -35,8 +35,15 @@ class Path(base.Value):
class Header(base.KeyValue):
+ unique_name = None
preamble = "h"
+ def values(self, settings):
+ return (
+ self.key.get_generator(settings),
+ self.value.get_generator(settings),
+ )
+
class Body(base.Value):
preamble = "b"
@@ -46,13 +53,21 @@ class Times(base.Integer):
preamble = "x"
+class Code(base.Integer):
+ pass
+
+
class Request(message.Message):
comps = (
Header,
Body,
-
Times,
)
+ logattrs = ["method", "path"]
+
+ def __init__(self, tokens):
+ super(Request, self).__init__(tokens)
+ self.rendered_values = None
@property
def method(self):
@@ -87,7 +102,6 @@ class Request(message.Message):
Method.expr(),
base.Sep,
Path.expr(),
- base.Sep,
pp.ZeroOrMore(base.Sep + atom)
]
)
@@ -95,25 +109,99 @@ class Request(message.Message):
return resp
def resolve(self, settings, msg=None):
- tokens = self.tokens[:]
- return self.__class__(
- [i.resolve(settings, self) for i in tokens]
+ return self
+
+ def values(self, settings):
+ if self.rendered_values:
+ return self.rendered_values
+ else:
+ headers = [header.values(settings) for header in self.headers]
+
+ body = self.body
+ if body:
+ body = body.string()
+
+ self.rendered_values = settings.protocol.create_request(
+ self.method.string(),
+ self.path.string(),
+ headers, # TODO: parse that into a dict?!
+ body)
+ return self.rendered_values
+
+ def spec(self):
+ return ":".join([i.spec() for i in self.tokens])
+
+
+class Response(message.Message):
+ unique_name = None
+ comps = (
+ Header,
+ Body,
+ )
+
+ def __init__(self, tokens):
+ super(Response, self).__init__(tokens)
+ self.rendered_values = None
+ self.stream_id = 0
+
+ @property
+ def code(self):
+ return self.tok(Code)
+
+ @property
+ def headers(self):
+ return self.toks(Header)
+
+ @property
+ def body(self):
+ return self.tok(Body)
+
+ @property
+ def actions(self):
+ return []
+
+ def resolve(self, settings, msg=None):
+ return self
+
+ @classmethod
+ def expr(klass):
+ parts = [i.expr() for i in klass.comps]
+ atom = pp.MatchFirst(parts)
+ resp = pp.And(
+ [
+ Code.expr(),
+ pp.ZeroOrMore(base.Sep + atom)
+ ]
)
+ resp = resp.setParseAction(klass)
+ return resp
def values(self, settings):
- return settings.protocol.create_request(
- self.method.value.get_generator(settings),
- self.path,
- self.headers,
- self.body)
+ if self.rendered_values:
+ return self.rendered_values
+ else:
+ headers = [header.values(settings) for header in self.headers]
+
+ body = self.body
+ if body:
+ body = body.string()
+
+ self.rendered_values = settings.protocol.create_response(
+ self.code.string(),
+ self.stream_id,
+ headers, # TODO: parse that into a dict?!
+ body)
+ return self.rendered_values
def spec(self):
return ":".join([i.spec() for i in self.tokens])
+def make_error_response(reason, body=None):
+ tokens = [
+ Code("800"),
+ Body(base.TokValueLiteral("pathod error: " + (body or reason))),
+ ]
+ return Response(tokens)
-# class H2F(base.CaselessLiteral):
-# TOK = "h2f"
-#
-#
-# class WebsocketFrame(message.Message):
+# class Frame(message.Message):
# pass
diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py
index ba06b2f1..c42cc82a 100644
--- a/libpathod/pathoc.py
+++ b/libpathod/pathoc.py
@@ -30,13 +30,8 @@ class SSLInfo:
self.certchain, self.cipher, self.alp = certchain, cipher, alp
def __str__(self):
- if self.alp:
- alp = self.alp
- else:
- alp = '<no protocol negotiated>'
-
parts = [
- "Application Layer Protocol: %s" % alp,
+ "Application Layer Protocol: %s" % self.alp,
"Cipher: %s, %s bit, %s" % self.cipher,
"SSL certificate chain:"
]
@@ -155,13 +150,14 @@ class Pathoc(tcp.TCPClient):
# SSL
ssl=None,
sni=None,
- sslversion=4,
+ sslversion='SSLv23',
clientcert=None,
ciphers=None,
# HTTP/2
use_http2=False,
http2_skip_connection_preface=False,
+ http2_framedump = False,
# Websockets
ws_read_limit = None,
@@ -199,6 +195,7 @@ class Pathoc(tcp.TCPClient):
self.use_http2 = use_http2
self.http2_skip_connection_preface = http2_skip_connection_preface
+ self.http2_framedump = http2_framedump
self.ws_read_limit = ws_read_limit
@@ -216,6 +213,9 @@ class Pathoc(tcp.TCPClient):
self.ws_framereader = None
if self.use_http2:
+ if not OpenSSL._util.lib.Cryptography_HAS_ALPN: # pragma: nocover
+ print >> sys.stderr, "HTTP/2 requires ALPN support. Please use OpenSSL >= 1.0.2."
+ print >> sys.stderr, "Pathoc might not be working as expected without ALPN."
self.protocol = http2.HTTP2Protocol(self)
else:
# TODO: create HTTP or Websockets protocol
@@ -259,7 +259,7 @@ class Pathoc(tcp.TCPClient):
an HTTP CONNECT request.
"""
if self.use_http2 and not self.ssl:
- raise ValueError("HTTP2 without SSL is not supported.")
+ raise NotImplementedError("HTTP2 without SSL is not supported.")
tcp.TCPClient.connect(self)
@@ -294,7 +294,7 @@ class Pathoc(tcp.TCPClient):
if self.use_http2:
self.protocol.check_alpn()
if not self.http2_skip_connection_preface:
- self.protocol.perform_connection_preface()
+ self.protocol.perform_client_connection_preface()
if self.timeout:
self.settimeout(self.timeout)
@@ -462,6 +462,7 @@ def main(args): # pragma: nocover
ciphers = args.ciphers,
use_http2 = args.use_http2,
http2_skip_connection_preface = args.http2_skip_connection_preface,
+ http2_framedump = args.http2_framedump,
showreq = args.showreq,
showresp = args.showresp,
explain = args.explain,
diff --git a/libpathod/pathoc_cmdline.py b/libpathod/pathoc_cmdline.py
new file mode 100644
index 00000000..1d0df3b5
--- /dev/null
+++ b/libpathod/pathoc_cmdline.py
@@ -0,0 +1,224 @@
+#!/usr/bin/env python
+import sys
+import argparse
+import os
+import os.path
+import re
+from netlib import http_uastrings
+from . import pathoc, pathod, version, utils, language
+
+
+def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr):
+ preparser = argparse.ArgumentParser(add_help=False)
+ preparser.add_argument(
+ "--show-uas", dest="showua", action="store_true", default=False,
+ help="Print user agent shortcuts and exit."
+ )
+ pa = preparser.parse_known_args(argv)[0]
+ if pa.showua:
+ print >> stdout, "User agent strings:"
+ for i in http_uastrings.UASTRINGS:
+ print >> stdout, " ", i[1], i[0]
+ sys.exit(0)
+
+ parser = argparse.ArgumentParser(
+ description='A perverse HTTP client.', parents=[preparser]
+ )
+ parser.add_argument(
+ '--version',
+ action='version',
+ version="pathoc " + version.VERSION
+ )
+ parser.add_argument(
+ "-c", dest="connect_to", type=str, default=False,
+ metavar = "HOST:PORT",
+ help="Issue an HTTP CONNECT to connect to the specified host."
+ )
+ parser.add_argument(
+ "--memo-limit", dest='memolimit', default=5000, type=int, metavar="N",
+ help='Stop if we do not find a valid request after N attempts.'
+ )
+ parser.add_argument(
+ "-m", dest='memo', action="store_true", default=False,
+ help="""
+ Remember specs, and never play the same one twice. Note that this
+ means requests have to be rendered in memory, which means that
+ large generated data can cause issues.
+ """
+ )
+ parser.add_argument(
+ "-n", dest='repeat', default=1, type=int, metavar="N",
+ help='Repeat N times. If 0 repeat for ever.'
+ )
+ parser.add_argument(
+ "-w", dest='wait', default=0, type=float, metavar="N",
+ help='Wait N seconds between each request.'
+ )
+ parser.add_argument(
+ "-r", dest="random", action="store_true", default=False,
+ help="""
+ Select a random request from those specified. If this is not specified,
+ requests are all played in sequence.
+ """
+ )
+ parser.add_argument(
+ "-t", dest="timeout", type=int, default=None,
+ help="Connection timeout"
+ )
+ parser.add_argument(
+ "--http2", dest="use_http2", action="store_true", default=False,
+ help='Perform all requests over a single HTTP/2 connection.'
+ )
+ parser.add_argument(
+ "--http2-skip-connection-preface",
+ dest="http2_skip_connection_preface",
+ action="store_true",
+ default=False,
+ help='Skips the HTTP/2 connection preface before sending requests.')
+
+ parser.add_argument(
+ 'host', type=str,
+ metavar = "host[:port]",
+ help='Host and port to connect to'
+ )
+ parser.add_argument(
+ 'requests', type=str, nargs="+",
+ help="""
+ Request specification, or path to a file containing request
+ specifcations
+ """
+ )
+
+ group = parser.add_argument_group(
+ 'SSL',
+ )
+ group.add_argument(
+ "-s", dest="ssl", action="store_true", default=False,
+ help="Connect with SSL"
+ )
+ group.add_argument(
+ "-C", dest="clientcert", type=str, default=False,
+ help="Path to a file containing client certificate and private key"
+ )
+ group.add_argument(
+ "-i", dest="sni", type=str, default=False,
+ help="SSL Server Name Indication"
+ )
+ group.add_argument(
+ "--ciphers", dest="ciphers", type=str, default=False,
+ help="SSL cipher specification"
+ )
+ group.add_argument(
+ "--sslversion", dest="sslversion", type=str, default='SSLv23',
+ choices=utils.SSLVERSIONS.keys(),
+ help=""""
+ Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23.
+ Default to SSLv23."""
+ )
+
+ group = parser.add_argument_group(
+ 'Controlling Output',
+ """
+ Some of these options expand generated values for logging - if
+ you're generating large data, use them with caution.
+ """
+ )
+ group.add_argument(
+ "-I", dest="ignorecodes", type=str, default="",
+ help="Comma-separated list of response codes to ignore"
+ )
+ group.add_argument(
+ "-S", dest="showssl", action="store_true", default=False,
+ help="Show info on SSL connection"
+ )
+ group.add_argument(
+ "-e", dest="explain", action="store_true", default=False,
+ help="Explain requests"
+ )
+ group.add_argument(
+ "-o", dest="oneshot", action="store_true", default=False,
+ help="Oneshot - exit after first non-ignored response"
+ )
+ group.add_argument(
+ "-q", dest="showreq", action="store_true", default=False,
+ help="Print full request"
+ )
+ group.add_argument(
+ "-p", dest="showresp", action="store_true", default=False,
+ help="Print full response"
+ )
+ group.add_argument(
+ "-T", dest="ignoretimeout", action="store_true", default=False,
+ help="Ignore timeouts"
+ )
+ group.add_argument(
+ "-x", dest="hexdump", action="store_true", default=False,
+ help="Output in hexdump format"
+ )
+ group.add_argument(
+ "--http2-framedump", dest="http2_framedump", action="store_true", default=False,
+ help="Output all received & sent HTTP/2 frames"
+ )
+
+ args = parser.parse_args(argv[1:])
+
+ args.port = None
+ if ":" in args.host:
+ h, p = args.host.rsplit(":", 1)
+ try:
+ p = int(p)
+ except ValueError:
+ return parser.error("Invalid port in host spec: %s" % args.host)
+ args.host = h
+ args.port = p
+
+ if args.port is None:
+ args.port = 443 if args.ssl else 80
+
+ try:
+ args.ignorecodes = [int(i) for i in args.ignorecodes.split(",") if i]
+ except ValueError:
+ return parser.error(
+ "Invalid return code specification: %s" %
+ args.ignorecodes)
+
+ if args.connect_to:
+ parts = args.connect_to.split(":")
+ if len(parts) != 2:
+ return parser.error(
+ "Invalid CONNECT specification: %s" %
+ args.connect_to)
+ try:
+ parts[1] = int(parts[1])
+ except ValueError:
+ return parser.error(
+ "Invalid CONNECT specification: %s" %
+ args.connect_to)
+ args.connect_to = parts
+ else:
+ args.connect_to = None
+
+ if args.http2_skip_connection_preface:
+ args.use_http2 = True
+
+ if args.use_http2:
+ args.ssl = True
+
+ reqs = []
+ for r in args.requests:
+ if os.path.isfile(r):
+ data = open(r).read()
+ r = data
+ try:
+ reqs.append(language.parse_pathoc(r, args.use_http2))
+ except language.ParseException as v:
+ print >> stderr, "Error parsing request spec: %s" % v.msg
+ print >> stderr, v.marked()
+ sys.exit(1)
+ args.requests = reqs
+ return args
+
+
+def go_pathoc(): # pragma: nocover
+ args = args_pathoc(sys.argv)
+ pathoc.main(args)
diff --git a/libpathod/pathod.py b/libpathod/pathod.py
index 13f602b4..212abbdc 100644
--- a/libpathod/pathod.py
+++ b/libpathod/pathod.py
@@ -7,7 +7,7 @@ import urllib
import re
import time
-from netlib import tcp, http, wsgi, certutils, websockets
+from netlib import tcp, http, http2, wsgi, certutils, websockets, odict
from . import version, app, language, utils, log
import language.http
@@ -20,7 +20,7 @@ DEFAULT_CERT_DOMAIN = "pathod.net"
CONFDIR = "~/.mitmproxy"
CERTSTORE_BASENAME = "mitmproxy"
CA_CERT_NAME = "mitmproxy-ca.pem"
-DEFAULT_ANCHOR = r"/p/?"
+DEFAULT_CRAFT_ANCHOR = "/p/"
logger = logging.getLogger('pathod')
@@ -39,21 +39,23 @@ class SSLOptions:
request_client_cert=False,
sslversion=tcp.SSLv23_METHOD,
ciphers=None,
- certs=None
+ certs=None,
+ alpn_select=http2.HTTP2Protocol.ALPN_PROTO_H2,
):
self.confdir = confdir
self.cn = cn
+ self.sans = sans
+ self.not_after_connect = not_after_connect
+ self.request_client_cert = request_client_cert
+ self.sslversion = sslversion
+ self.ciphers = ciphers
+ self.alpn_select = alpn_select
self.certstore = certutils.CertStore.from_store(
os.path.expanduser(confdir),
CERTSTORE_BASENAME
)
for i in certs or []:
self.certstore.add_cert_file(*i)
- self.not_after_connect = not_after_connect
- self.request_client_cert = request_client_cert
- self.ciphers = ciphers
- self.sslversion = sslversion
- self.sans = sans
def get_cert(self, name):
if self.cn:
@@ -67,32 +69,37 @@ class PathodHandler(tcp.BaseHandler):
wbufsize = 0
sni = None
- def __init__(self, connection, address, server, logfp, settings):
- self.logfp = logfp
+ def __init__(self, connection, address, server, logfp, settings, http2_framedump=False):
tcp.BaseHandler.__init__(self, connection, address, server)
+ self.logfp = logfp
self.settings = copy.copy(settings)
+ self.protocol = None
+ self.use_http2 = False
+ self.http2_framedump = http2_framedump
- def handle_sni(self, connection):
+ def _handle_sni(self, connection):
self.sni = connection.get_servername()
def http_serve_crafted(self, crafted):
+ """
+ This method is HTTP/1 and HTTP/2 capable.
+ """
+
error, crafted = self.server.check_policy(
crafted, self.settings
)
if error:
- err = language.http.make_error_response(error)
+ err = self.make_http_error_response(error)
language.serve(err, self.wfile, self.settings)
return None, dict(
type="error",
msg = error
)
- if self.server.explain and not isinstance(
- crafted,
- language.http.PathodErrorResponse
- ):
+ if self.server.explain and not hasattr(crafted, 'is_error_response'):
crafted = crafted.freeze(self.settings)
log.write(self.logfp, ">> Spec: %s" % crafted.spec())
+
response_log = language.serve(
crafted,
self.wfile,
@@ -152,6 +159,8 @@ class PathodHandler(tcp.BaseHandler):
def handle_http_connect(self, connect, lg):
"""
+ This method is HTTP/1 only.
+
Handle a CONNECT request.
"""
http.read_headers(self.rfile)
@@ -169,10 +178,11 @@ class PathodHandler(tcp.BaseHandler):
self.convert_to_ssl(
cert,
key,
- handle_sni=self.handle_sni,
+ handle_sni=self._handle_sni,
request_client_cert=self.server.ssloptions.request_client_cert,
cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.sslversion,
+ alpn_select=self.server.ssloptions.alpn_select,
)
except tcp.NetLibError as v:
s = str(v)
@@ -182,10 +192,12 @@ class PathodHandler(tcp.BaseHandler):
def handle_http_app(self, method, path, headers, content, lg):
"""
+ This method is HTTP/1 only.
+
Handle a request to the built-in app.
"""
if self.server.noweb:
- crafted = language.http.make_error_response("Access Denied")
+ crafted = self.make_http_error_response("Access Denied")
language.serve(crafted, self.wfile, self.settings)
return None, dict(
type="error",
@@ -206,6 +218,8 @@ class PathodHandler(tcp.BaseHandler):
def handle_http_request(self):
"""
+ This method is HTTP/1 and HTTP/2 capable.
+
Returns a (handler, log) tuple.
handler: Handler for the next request, or None to disconnect
@@ -214,28 +228,26 @@ class PathodHandler(tcp.BaseHandler):
lr = self.rfile if self.server.logreq else None
lw = self.wfile if self.server.logresp else None
with log.Log(self.logfp, self.server.hexdump, lr, lw) as lg:
- line = http.get_request_line(self.rfile)
- if not line:
- # Normal termination
- return None, None
-
- m = utils.MemBool()
- if m(http.parse_init_connect(line)):
- return self.handle_http_connect(m.v, lg)
- elif m(http.parse_init_proxy(line)):
- method, _, _, _, path, httpversion = m.v
- elif m(http.parse_init_http(line)):
- method, path, httpversion = m.v
+ if self.use_http2:
+ self.protocol.perform_server_connection_preface()
+ stream_id, headers, body = self.protocol.read_request()
+ method = headers[':method']
+ path = headers[':path']
+ headers = odict.ODict(headers)
+ httpversion = ""
else:
- s = "Invalid first line: %s" % repr(line)
- lg(s)
- return None, dict(type="error", msg=s)
-
- headers = http.read_headers(self.rfile)
- if headers is None:
- s = "Invalid headers"
- lg(s)
- return None, dict(type="error", msg=s)
+ req = self.read_http_request(lg)
+ if 'next_handle' in req:
+ return req['next_handle']
+ if 'errors' in req:
+ return None, req['errors']
+ if not 'method' in req or not 'path' in req:
+ return None, None
+ method = req['method']
+ path = req['path']
+ headers = req['headers']
+ body = req['body']
+ httpversion = req['httpversion']
clientcert = None
if self.clientcert:
@@ -265,16 +277,6 @@ class PathodHandler(tcp.BaseHandler):
if self.ssl_established:
retlog["cipher"] = self.get_current_cipher()
- try:
- content = http.read_http_body(
- self.rfile, headers, None,
- method, None, True
- )
- except http.HttpError as s:
- s = str(s)
- lg(s)
- return None, dict(type="error", msg=s)
-
m = utils.MemBool()
websocket_key = websockets.check_client_handshake(headers)
self.settings.websocket_key = websocket_key
@@ -285,27 +287,40 @@ class PathodHandler(tcp.BaseHandler):
anchor_gen = language.parse_pathod("ws")
else:
anchor_gen = None
- for i in self.server.anchors:
- if i[0].match(path):
- anchor_gen = i[1]
+
+ for regex, spec in self.server.anchors:
+ if regex.match(path):
+ anchor_gen = language.parse_pathod(spec, self.use_http2)
break
else:
- if m(self.server.craftanchor.match(path)):
- spec = urllib.unquote(path)[len(m.v.group()):]
+ if m(path.startswith(self.server.craftanchor)):
+ spec = urllib.unquote(path)[len(self.server.craftanchor):]
if spec:
try:
- anchor_gen = language.parse_pathod(spec)
+ anchor_gen = language.parse_pathod(spec, self.use_http2)
except language.ParseException as v:
lg("Parse error: %s" % v.msg)
- anchor_gen = iter([language.http.make_error_response(
+ anchor_gen = iter([self.make_http_error_response(
"Parse Error",
"Error parsing response spec: %s\n" % (
v.msg + v.marked()
)
)])
+ else:
+ if self.use_http2:
+ anchor_gen = iter([self.make_http_error_response(
+ "Spec Error",
+ "HTTP/2 only supports request/response with the craft anchor point: %s" %
+ self.server.craftanchor
+ )])
+
if anchor_gen:
spec = anchor_gen.next()
+
+ if self.use_http2 and isinstance(spec, language.http2.Response):
+ spec.stream_id = stream_id
+
lg("crafting spec: %s" % spec)
nexthandler, retlog["response"] = self.http_serve_crafted(
spec
@@ -315,31 +330,82 @@ class PathodHandler(tcp.BaseHandler):
else:
return nexthandler, retlog
else:
- return self.handle_http_app(method, path, headers, content, lg)
+ return self.handle_http_app(method, path, headers, body, lg)
- def addlog(self, log):
- # FIXME: The bytes in the log should not be escaped. We do this at the
- # moment because JSON encoding can't handle binary data, and I don't
- # want to base64 everything.
- if self.server.logreq:
- bytes = self.rfile.get_log().encode("string_escape")
- log["request_bytes"] = bytes
- if self.server.logresp:
- bytes = self.wfile.get_log().encode("string_escape")
- log["response_bytes"] = bytes
- self.server.add_log(log)
+ def read_http_request(self, lg):
+ """
+ This method is HTTP/1 only.
+ """
+ line = http.get_request_line(self.rfile)
+ if not line:
+ # Normal termination
+ return dict()
+
+ m = utils.MemBool()
+ if m(http.parse_init_connect(line)):
+ return dict(next_handle=self.handle_http_connect(m.v, lg))
+ elif m(http.parse_init_proxy(line)):
+ method, _, _, _, path, httpversion = m.v
+ elif m(http.parse_init_http(line)):
+ method, path, httpversion = m.v
+ else:
+ s = "Invalid first line: %s" % repr(line)
+ lg(s)
+ return dict(errors=dict(type="error", msg=s))
+
+ headers = http.read_headers(self.rfile)
+ if headers is None:
+ s = "Invalid headers"
+ lg(s)
+ return dict(errors=dict(type="error", msg=s))
+
+ try:
+ body = http.read_http_body(
+ self.rfile,
+ headers,
+ None,
+ method,
+ None,
+ True,
+ )
+ except http.HttpError as s:
+ s = str(s)
+ lg(s)
+ return dict(errors=dict(type="error", msg=s))
+
+ return dict(
+ method=method,
+ path=path,
+ headers=headers,
+ body=body,
+ httpversion=httpversion)
+
+ def make_http_error_response(self, reason, body=None):
+ """
+ This method is HTTP/1 and HTTP/2 capable.
+ """
+ if self.use_http2:
+ resp = language.http2.make_error_response(reason, body)
+ else:
+ resp = language.http.make_error_response(reason, body)
+ resp.is_error_response = True
+ return resp
def handle(self):
+ self.settimeout(self.server.timeout)
+
if self.server.ssl:
try:
cert, key, _ = self.server.ssloptions.get_cert(None)
self.convert_to_ssl(
cert,
key,
- handle_sni=self.handle_sni,
+ dhparams=self.server.ssloptions.certstore.dhparams,
+ handle_sni=self._handle_sni,
request_client_cert=self.server.ssloptions.request_client_cert,
cipher_list=self.server.ssloptions.ciphers,
method=self.server.ssloptions.sslversion,
+ alpn_select=self.server.ssloptions.alpn_select,
)
except tcp.NetLibError as v:
s = str(v)
@@ -351,8 +417,20 @@ class PathodHandler(tcp.BaseHandler):
)
log.write(self.logfp, s)
return
- self.settimeout(self.server.timeout)
+
+ alp = self.get_alpn_proto_negotiated()
+ if alp == http2.HTTP2Protocol.ALPN_PROTO_H2:
+ self.protocol = http2.HTTP2Protocol(self, is_server=True, dump_frames=self.http2_framedump)
+ self.use_http2 = True
+
+ # if not self.protocol:
+ # # TODO: create HTTP or Websockets protocol
+ # self.protocol = None
+
+ self.settings.protocol = self.protocol
+
handler = self.handle_http_request
+
while not self.finished:
handler, l = handler()
if l:
@@ -360,6 +438,18 @@ class PathodHandler(tcp.BaseHandler):
if not handler:
return
+ def addlog(self, log):
+ # FIXME: The bytes in the log should not be escaped. We do this at the
+ # moment because JSON encoding can't handle binary data, and I don't
+ # want to base64 everything.
+ if self.server.logreq:
+ bytes = self.rfile.get_log().encode("string_escape")
+ log["request_bytes"] = bytes
+ if self.server.logresp:
+ bytes = self.wfile.get_log().encode("string_escape")
+ log["response_bytes"] = bytes
+ self.server.add_log(log)
+
class Pathod(tcp.TCPServer):
LOGBUF = 500
@@ -369,7 +459,7 @@ class Pathod(tcp.TCPServer):
addr,
ssl=False,
ssloptions=None,
- craftanchor=re.compile(DEFAULT_ANCHOR),
+ craftanchor=DEFAULT_CRAFT_ANCHOR,
staticdir=None,
anchors=(),
sizelimit=None,
@@ -382,6 +472,7 @@ class Pathod(tcp.TCPServer):
logresp=False,
explain=False,
hexdump=False,
+ http2_framedump=False,
webdebug=False,
logfp=sys.stdout,
):
@@ -389,7 +480,7 @@ class Pathod(tcp.TCPServer):
addr: (address, port) tuple. If port is 0, a free port will be
automatically chosen.
ssloptions: an SSLOptions object.
- craftanchor: string specifying the path under which to anchor
+ craftanchor: URL prefix specifying the path under which to anchor
response generation.
staticdir: path to a directory of static resources, or None.
anchors: List of (regex object, language.Request object) tuples, or
@@ -409,6 +500,7 @@ class Pathod(tcp.TCPServer):
self.noapi, self.nohang = noapi, nohang
self.timeout, self.logreq = timeout, logreq
self.logresp, self.hexdump = logresp, hexdump
+ self.http2_framedump = http2_framedump
self.explain = explain
self.logfp = logfp
@@ -446,7 +538,8 @@ class Pathod(tcp.TCPServer):
client_address,
self,
self.logfp,
- self.settings
+ self.settings,
+ self.http2_framedump,
)
try:
h.handle()
@@ -502,7 +595,7 @@ def main(args): # pragma: nocover
ciphers = args.ciphers,
sslversion = utils.SSLVERSIONS[args.sslversion],
certs = args.ssl_certs,
- sans = args.sans
+ sans = args.sans,
)
root = logging.getLogger()
@@ -542,6 +635,7 @@ def main(args): # pragma: nocover
logreq = args.logreq,
logresp = args.logresp,
hexdump = args.hexdump,
+ http2_framedump = args.http2_framedump,
explain = args.explain,
webdebug = args.webdebug
)
diff --git a/libpathod/pathod_cmdline.py b/libpathod/pathod_cmdline.py
new file mode 100644
index 00000000..4343401f
--- /dev/null
+++ b/libpathod/pathod_cmdline.py
@@ -0,0 +1,229 @@
+#!/usr/bin/env python
+import sys
+import argparse
+import os
+import os.path
+import re
+from netlib import http_uastrings
+from . import pathoc, pathod, version, utils, language
+
+
+def args_pathod(argv, stdout=sys.stdout, stderr=sys.stderr):
+ parser = argparse.ArgumentParser(
+ description='A pathological HTTP/S daemon.'
+ )
+ parser.add_argument(
+ '--version',
+ action='version',
+ version="pathod " + version.VERSION
+ )
+ parser.add_argument(
+ "-p",
+ dest='port',
+ default=9999,
+ type=int,
+ help='Port. Specify 0 to pick an arbitrary empty port. (9999)'
+ )
+ parser.add_argument(
+ "-l",
+ dest='address',
+ default="127.0.0.1",
+ type=str,
+ help='Listening address. (127.0.0.1)'
+ )
+ parser.add_argument(
+ "-a",
+ dest='anchors',
+ default=[],
+ type=str,
+ action="append",
+ metavar="ANCHOR",
+ help="""
+ Add an anchor. Specified as a string with the form
+ pattern=spec or pattern=filepath, where pattern is a regular
+ expression.
+ """
+ )
+ parser.add_argument(
+ "-c", dest='craftanchor', default=pathod.DEFAULT_CRAFT_ANCHOR, type=str,
+ help="""
+ URL path specifying prefix for URL crafting
+ commands. (%s)
+ """%pathod.DEFAULT_CRAFT_ANCHOR
+ )
+ parser.add_argument(
+ "--confdir",
+ action="store", type = str, dest="confdir", default='~/.mitmproxy',
+ help = "Configuration directory. (~/.mitmproxy)"
+ )
+ parser.add_argument(
+ "-d", dest='staticdir', default=None, type=str,
+ help='Directory for static files.'
+ )
+ parser.add_argument(
+ "-D", dest='daemonize', default=False, action="store_true",
+ help='Daemonize.'
+ )
+ parser.add_argument(
+ "-t", dest="timeout", type=int, default=None,
+ help="Connection timeout"
+ )
+ parser.add_argument(
+ "--limit-size",
+ dest='sizelimit',
+ default=None,
+ type=str,
+ help='Size limit of served responses. Understands size suffixes, i.e. 100k.')
+ parser.add_argument(
+ "--noapi", dest='noapi', default=False, action="store_true",
+ help='Disable API.'
+ )
+ parser.add_argument(
+ "--nohang", dest='nohang', default=False, action="store_true",
+ help='Disable pauses during crafted response generation.'
+ )
+ parser.add_argument(
+ "--noweb", dest='noweb', default=False, action="store_true",
+ help='Disable both web interface and API.'
+ )
+ parser.add_argument(
+ "--nocraft",
+ dest='nocraft',
+ default=False,
+ action="store_true",
+ help='Disable response crafting. If anchors are specified, they still work.')
+ parser.add_argument(
+ "--webdebug", dest='webdebug', default=False, action="store_true",
+ help='Debugging mode for the web app (dev only).'
+ )
+
+ group = parser.add_argument_group(
+ 'SSL',
+ )
+ group.add_argument(
+ "-s", dest='ssl', default=False, action="store_true",
+ help='Run in HTTPS mode.'
+ )
+ group.add_argument(
+ "--cn",
+ dest="cn",
+ type=str,
+ default=None,
+ help="CN for generated SSL certs. Default: %s" %
+ pathod.DEFAULT_CERT_DOMAIN)
+ group.add_argument(
+ "-C", dest='ssl_not_after_connect', default=False, action="store_true",
+ help="Don't expect SSL after a CONNECT request."
+ )
+ group.add_argument(
+ "--cert", dest='ssl_certs', default=[], type=str,
+ metavar = "SPEC", action="append",
+ help = """
+ Add an SSL certificate. SPEC is of the form "[domain=]path". The domain
+ may include a wildcard, and is equal to "*" if not specified. The file
+ at path is a certificate in PEM format. If a private key is included in
+ the PEM, it is used, else the default key in the conf dir is used. Can
+ be passed multiple times.
+ """
+ )
+ group.add_argument(
+ "--ciphers", dest="ciphers", type=str, default=False,
+ help="SSL cipher specification"
+ )
+ group.add_argument(
+ "--san", dest="sans", type=str, default=[], action="append",
+ metavar="SAN",
+ help="""
+ Subject Altnernate Name to add to the server certificate.
+ May be passed multiple times.
+ """
+ )
+ group.add_argument(
+ "--sslversion", dest="sslversion", type=str, default='SSLv23',
+ choices=utils.SSLVERSIONS.keys(),
+ help=""""
+ Use a specified protocol - TLSv1.2, TLSv1.1, TLSv1, SSLv3, SSLv2, SSLv23.
+ Default to SSLv23."""
+ )
+
+ group = parser.add_argument_group(
+ 'Controlling Logging',
+ """
+ Some of these options expand generated values for logging - if
+ you're generating large data, use them with caution.
+ """
+ )
+ group.add_argument(
+ "-e", dest="explain", action="store_true", default=False,
+ help="Explain responses"
+ )
+ group.add_argument(
+ "-f", dest='logfile', default=None, type=str,
+ help='Log to file.'
+ )
+ group.add_argument(
+ "-q", dest="logreq", action="store_true", default=False,
+ help="Log full request"
+ )
+ group.add_argument(
+ "-r", dest="logresp", action="store_true", default=False,
+ help="Log full response"
+ )
+ group.add_argument(
+ "-x", dest="hexdump", action="store_true", default=False,
+ help="Log request/response in hexdump format"
+ )
+ group.add_argument(
+ "--http2-framedump", dest="http2_framedump", action="store_true", default=False,
+ help="Output all received & sent HTTP/2 frames"
+ )
+
+
+ args = parser.parse_args(argv[1:])
+
+ certs = []
+ for i in args.ssl_certs:
+ parts = i.split("=", 1)
+ if len(parts) == 1:
+ parts = ["*", parts[0]]
+ parts[1] = os.path.expanduser(parts[1])
+ if not os.path.isfile(parts[1]):
+ return parser.error(
+ "Certificate file does not exist: %s" %
+ parts[1])
+ certs.append(parts)
+ args.ssl_certs = certs
+
+ alst = []
+ for i in args.anchors:
+ parts = utils.parse_anchor_spec(i)
+ if not parts:
+ return parser.error("Invalid anchor specification: %s" % i)
+ alst.append(parts)
+ args.anchors = alst
+
+ sizelimit = None
+ if args.sizelimit:
+ try:
+ sizelimit = utils.parse_size(args.sizelimit)
+ except ValueError as v:
+ return parser.error(v)
+ args.sizelimit = sizelimit
+
+ anchors = []
+ for patt, spec in args.anchors:
+ if os.path.isfile(spec):
+ data = open(spec).read()
+ spec = data
+ try:
+ arex = re.compile(patt)
+ except re.error:
+ return parser.error("Invalid regex in anchor: %s" % patt)
+ anchors.append((arex, spec))
+ args.anchors = anchors
+ return args
+
+
+def go_pathod(): # pragma: nocover
+ args = args_pathod(sys.argv)
+ pathod.main(args)
diff --git a/libpathod/utils.py b/libpathod/utils.py
index 9bd2812e..481c5137 100644
--- a/libpathod/utils.py
+++ b/libpathod/utils.py
@@ -3,10 +3,12 @@ import sys
from netlib import tcp
SSLVERSIONS = {
- 1: tcp.TLSv1_METHOD,
- 2: tcp.SSLv2_METHOD,
- 3: tcp.SSLv3_METHOD,
- 4: tcp.SSLv23_METHOD,
+ 'TLSv1.2': tcp.TLSv1_2_METHOD,
+ 'TLSv1.1': tcp.TLSv1_1_METHOD,
+ 'TLSv1': tcp.TLSv1_METHOD,
+ 'SSLv3': tcp.SSLv3_METHOD,
+ 'SSLv2': tcp.SSLv2_METHOD,
+ 'SSLv23': tcp.SSLv23_METHOD,
}
SIZE_UNITS = dict(
diff --git a/pathoc b/pathoc
index cbf8f773..b3121611 100755
--- a/pathoc
+++ b/pathoc
@@ -1,5 +1,6 @@
#!/usr/bin/env python
-from libpathod import cmdline
+
+from libpathod import pathoc_cmdline as cmdline
if __name__ == "__main__":
cmdline.go_pathoc()
diff --git a/pathod b/pathod
index ca0baa57..a79becf1 100755
--- a/pathod
+++ b/pathod
@@ -1,5 +1,6 @@
#!/usr/bin/env python
-from libpathod import cmdline
+
+from libpathod import pathod_cmdline as cmdline
if __name__ == "__main__":
cmdline.go_pathod()
diff --git a/setup.py b/setup.py
index 344712eb..041b164d 100644
--- a/setup.py
+++ b/setup.py
@@ -40,8 +40,8 @@ setup(
include_package_data=True,
entry_points={
'console_scripts': [
- "pathod = libpathod.cmdline:go_pathod",
- "pathoc = libpathod.cmdline:go_pathoc"
+ "pathod = libpathod.pathod_cmdline:go_pathod",
+ "pathoc = libpathod.pathoc_cmdline:go_pathoc"
]
},
install_requires=[
diff --git a/test/test_cmdline.py b/test/test_cmdline.py
deleted file mode 100644
index c1b55608..00000000
--- a/test/test_cmdline.py
+++ /dev/null
@@ -1,155 +0,0 @@
-from libpathod import cmdline
-import tutils
-import cStringIO
-import mock
-
-
-@mock.patch("argparse.ArgumentParser.error")
-def test_pathod(perror):
- assert cmdline.args_pathod(["pathod"])
-
- a = cmdline.args_pathod(
- [
- "pathod",
- "--cert",
- tutils.test_data.path("data/testkey.pem")
- ]
- )
- assert a.ssl_certs
-
- a = cmdline.args_pathod(
- [
- "pathod",
- "--cert",
- "nonexistent"
- ]
- )
- assert perror.called
- perror.reset_mock()
-
- a = cmdline.args_pathod(
- [
- "pathod",
- "-a",
- "foo=200"
- ]
- )
- assert a.anchors
-
- a = cmdline.args_pathod(
- [
- "pathod",
- "-a",
- "foo=" + tutils.test_data.path("data/response")
- ]
- )
- assert a.anchors
-
- a = cmdline.args_pathod(
- [
- "pathod",
- "-a",
- "?=200"
- ]
- )
- assert perror.called
- perror.reset_mock()
-
- a = cmdline.args_pathod(
- [
- "pathod",
- "-a",
- "foo"
- ]
- )
- assert perror.called
- perror.reset_mock()
-
- s = cStringIO.StringIO()
- tutils.raises(
- SystemExit,
- cmdline.args_pathod,
- ["pathod", "-a", "foo=."],
- s,
- s
- )
-
- a = cmdline.args_pathod(
- [
- "pathod",
- "--limit-size",
- "200k"
- ]
- )
- assert a.sizelimit
-
- a = cmdline.args_pathod(
- [
- "pathod",
- "--limit-size",
- "q"
- ]
- )
- assert perror.called
- perror.reset_mock()
-
-
-@mock.patch("argparse.ArgumentParser.error")
-def test_pathoc(perror):
- assert cmdline.args_pathoc(["pathoc", "foo.com", "get:/"])
- s = cStringIO.StringIO()
- tutils.raises(
- SystemExit, cmdline.args_pathoc, [
- "pathoc", "--show-uas"], s, s)
-
- a = cmdline.args_pathoc(["pathoc", "foo.com:8888", "get:/"])
- assert a.port == 8888
-
- a = cmdline.args_pathoc(["pathoc", "foo.com:xxx", "get:/"])
- assert perror.called
- perror.reset_mock()
-
- a = cmdline.args_pathoc(["pathoc", "-I", "10, 20", "foo.com:8888", "get:/"])
- assert a.ignorecodes == [10, 20]
-
- a = cmdline.args_pathoc(["pathoc", "-I", "xx, 20", "foo.com:8888", "get:/"])
- assert perror.called
- perror.reset_mock()
-
- a = cmdline.args_pathoc(["pathoc", "-c", "foo:10", "foo.com:8888", "get:/"])
- assert a.connect_to == ["foo", 10]
-
- a = cmdline.args_pathoc(["pathoc", "-c", "foo", "foo.com:8888", "get:/"])
- assert perror.called
- perror.reset_mock()
-
- a = cmdline.args_pathoc(
- ["pathoc", "-c", "foo:bar", "foo.com:8888", "get:/"])
- assert perror.called
- perror.reset_mock()
-
- a = cmdline.args_pathoc(
- [
- "pathoc",
- "foo.com:8888",
- tutils.test_data.path("data/request")
- ]
- )
- assert len(list(a.requests)) == 1
-
- a = cmdline.args_pathod(
- [
- "pathod",
- "-c",
- "?"
- ]
- )
- assert perror.called
- perror.reset_mock()
-
- tutils.raises(
- SystemExit,
- cmdline.args_pathoc,
- ["pathoc", "foo.com", "invalid"],
- s, s
- )
diff --git a/test/test_language_http2.py b/test/test_language_http2.py
new file mode 100644
index 00000000..0be42253
--- /dev/null
+++ b/test/test_language_http2.py
@@ -0,0 +1,177 @@
+import cStringIO
+
+from netlib import tcp
+from libpathod import language
+from libpathod.language import http2, base
+import netlib
+import tutils
+
+
+def parse_request(s):
+ return language.parse_pathoc(s, True).next()
+
+def parse_response(s):
+ return language.parse_pathod(s, True).next()
+
+def default_settings():
+ return language.Settings(
+ request_host = "foo.com",
+ protocol = netlib.http2.HTTP2Protocol(tcp.TCPClient(('localhost', 1234)))
+ )
+
+
+def test_make_error_response():
+ d = cStringIO.StringIO()
+ s = http2.make_error_response("foo", "bar")
+ language.serve(s, d, default_settings())
+
+
+class TestRequest:
+ def test_cached_values(self):
+ req = parse_request("get:/")
+ req_id = id(req)
+ assert req_id == id(req.resolve(default_settings()))
+ assert req.values(default_settings()) == req.values(default_settings())
+
+ def test_nonascii(self):
+ tutils.raises("ascii", parse_request, "get:\xf0")
+
+ def test_err(self):
+ tutils.raises(language.ParseException, parse_request, 'GET')
+
+ def test_simple(self):
+ r = parse_request('GET:"/foo"')
+ assert r.method.string() == "GET"
+ assert r.path.string() == "/foo"
+ r = parse_request('GET:/foo')
+ assert r.path.string() == "/foo"
+
+ def test_multiple(self):
+ r = list(language.parse_pathoc("GET:/ PUT:/"))
+ assert r[0].method.string() == "GET"
+ assert r[1].method.string() == "PUT"
+ assert len(r) == 2
+
+ l = """
+ GET
+ "/foo"
+
+ PUT
+
+ "/foo
+
+
+
+ bar"
+ """
+ r = list(language.parse_pathoc(l, True))
+ assert len(r) == 2
+ assert r[0].method.string() == "GET"
+ assert r[1].method.string() == "PUT"
+
+ l = """
+ get:"http://localhost:9999/p/200"
+ get:"http://localhost:9999/p/200"
+ """
+ r = list(language.parse_pathoc(l, True))
+ assert len(r) == 2
+ assert r[0].method.string() == "GET"
+ assert r[1].method.string() == "GET"
+
+ def test_render_simple(self):
+ s = cStringIO.StringIO()
+ r = parse_request("GET:'/foo'")
+ assert language.serve(
+ r,
+ s,
+ default_settings(),
+ )
+
+ def test_render_with_headers(self):
+ s = cStringIO.StringIO()
+ r = parse_request('GET:/foo:h"foo"="bar"')
+ assert language.serve(
+ r,
+ s,
+ default_settings(),
+ )
+
+ def test_render_with_body(self):
+ s = cStringIO.StringIO()
+ r = parse_request("GET:'/foo':bfoobar")
+ assert language.serve(
+ r,
+ s,
+ default_settings(),
+ )
+
+ def test_spec(self):
+ def rt(s):
+ s = parse_request(s).spec()
+ assert parse_request(s).spec() == s
+ rt("get:/foo")
+
+
+class TestResponse:
+ def test_cached_values(self):
+ res = parse_response("200")
+ res_id = id(res)
+ assert res_id == id(res.resolve(default_settings()))
+ assert res.values(default_settings()) == res.values(default_settings())
+
+ def test_nonascii(self):
+ tutils.raises("ascii", parse_response, "200:\xf0")
+
+ def test_err(self):
+ tutils.raises(language.ParseException, parse_response, 'GET:/')
+
+ def test_simple(self):
+ r = parse_response('200')
+ assert r.code.string() == "200"
+ assert len(r.headers) == 0
+
+ r = parse_response('200:h"foo"="bar"')
+ assert r.code.string() == "200"
+ assert len(r.headers) == 1
+ assert r.headers[0].values(default_settings()) == ("foo", "bar")
+ assert r.body == None
+
+ r = parse_response('200:h"foo"="bar":bfoobar:h"bla"="fasel"')
+ assert r.code.string() == "200"
+ assert len(r.headers) == 2
+ assert r.headers[0].values(default_settings()) == ("foo", "bar")
+ assert r.headers[1].values(default_settings()) == ("bla", "fasel")
+ assert r.body.string() == "foobar"
+
+ def test_render_simple(self):
+ s = cStringIO.StringIO()
+ r = parse_response('200')
+ assert language.serve(
+ r,
+ s,
+ default_settings(),
+ )
+
+ def test_render_with_headers(self):
+ s = cStringIO.StringIO()
+ r = parse_response('200:h"foo"="bar"')
+ assert language.serve(
+ r,
+ s,
+ default_settings(),
+ )
+
+ def test_render_with_body(self):
+ s = cStringIO.StringIO()
+ r = parse_response('200:bfoobar')
+ assert language.serve(
+ r,
+ s,
+ default_settings(),
+ )
+
+ def test_spec(self):
+ def rt(s):
+ s = parse_response(s).spec()
+ assert parse_response(s).spec() == s
+ rt("200:bfoobar")
diff --git a/test/test_pathoc.py b/test/test_pathoc.py
index c00c7d53..d39f9275 100644
--- a/test/test_pathoc.py
+++ b/test/test_pathoc.py
@@ -1,8 +1,9 @@
import json
import cStringIO
import re
+from mock import Mock
-from netlib import tcp, http
+from netlib import tcp, http, http2
from libpathod import pathoc, test, version, pathod, language
import tutils
@@ -22,7 +23,7 @@ class _TestDaemon:
ssloptions = self.ssloptions,
staticdir = tutils.test_data.path("data"),
anchors = [
- (re.compile("/anchor/.*"), language.parse_pathod("202"))
+ (re.compile("/anchor/.*"), "202")
]
)
@@ -86,8 +87,9 @@ class _TestDaemon:
class TestDaemonSSL(_TestDaemon):
ssl = True
ssloptions = pathod.SSLOptions(
- request_client_cert=True,
- sans = ["test1.com", "test2.com"]
+ request_client_cert = True,
+ sans = ["test1.com", "test2.com"],
+ alpn_select = http2.HTTP2Protocol.ALPN_PROTO_H2,
)
def test_sni(self):
@@ -119,6 +121,14 @@ class TestDaemonSSL(_TestDaemon):
d = json.loads(r.content)
assert d["log"][0]["request"]["clientcert"]["keyinfo"]
+ def test_http2_without_ssl(self):
+ c = pathoc.Pathoc(
+ ("127.0.0.1", self.d.port),
+ use_http2 = True,
+ ssl = False,
+ )
+ tutils.raises(NotImplementedError, c.connect)
+
class TestDaemon(_TestDaemon):
ssl = False
@@ -216,3 +226,46 @@ class TestDaemon(_TestDaemon):
"HTTP/1.1 200 OK\r\n"
)
c.http_connect(to)
+
+
+class TestDaemonHTTP2(_TestDaemon):
+ ssl = True
+
+ def test_http2(self):
+ c = pathoc.Pathoc(
+ ("127.0.0.1", self.d.port),
+ use_http2 = True,
+ ssl = True,
+ )
+ assert isinstance(c.protocol, http2.HTTP2Protocol)
+
+ c = pathoc.Pathoc(
+ ("127.0.0.1", self.d.port),
+ )
+ assert c.protocol == None # TODO: change if other protocols get implemented
+
+ def test_http2_alpn(self):
+ c = pathoc.Pathoc(
+ ("127.0.0.1", self.d.port),
+ ssl = True,
+ use_http2 = True,
+ http2_skip_connection_preface = True,
+ )
+
+ tmp_convert_to_ssl = c.convert_to_ssl
+ c.convert_to_ssl = Mock()
+ c.convert_to_ssl.side_effect = tmp_convert_to_ssl
+ c.connect()
+
+ _, kwargs = c.convert_to_ssl.call_args
+ assert set(kwargs['alpn_protos']) == set([b'http1.1', b'h2'])
+
+ def test_request(self):
+ c = pathoc.Pathoc(
+ ("127.0.0.1", self.d.port),
+ ssl = True,
+ use_http2 = True,
+ )
+ c.connect()
+ resp = c.request("get:/p/200")
+ assert resp.status_code == "200"
diff --git a/test/test_pathoc_cmdline.py b/test/test_pathoc_cmdline.py
new file mode 100644
index 00000000..6c070aed
--- /dev/null
+++ b/test/test_pathoc_cmdline.py
@@ -0,0 +1,64 @@
+from libpathod import pathoc_cmdline as cmdline
+import tutils
+import cStringIO
+import mock
+
+
+@mock.patch("argparse.ArgumentParser.error")
+def test_pathoc(perror):
+ assert cmdline.args_pathoc(["pathoc", "foo.com", "get:/"])
+ s = cStringIO.StringIO()
+ tutils.raises(
+ SystemExit, cmdline.args_pathoc, [
+ "pathoc", "--show-uas"], s, s)
+
+ a = cmdline.args_pathoc(["pathoc", "foo.com:8888", "get:/"])
+ assert a.port == 8888
+
+ a = cmdline.args_pathoc(["pathoc", "foo.com:xxx", "get:/"])
+ assert perror.called
+ perror.reset_mock()
+
+ a = cmdline.args_pathoc(["pathoc", "-I", "10, 20", "foo.com:8888", "get:/"])
+ assert a.ignorecodes == [10, 20]
+
+ a = cmdline.args_pathoc(["pathoc", "-I", "xx, 20", "foo.com:8888", "get:/"])
+ assert perror.called
+ perror.reset_mock()
+
+ a = cmdline.args_pathoc(["pathoc", "-c", "foo:10", "foo.com:8888", "get:/"])
+ assert a.connect_to == ["foo", 10]
+
+ a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2"])
+ assert a.use_http2 == True
+ assert a.ssl == True
+
+ a = cmdline.args_pathoc(["pathoc", "foo.com", "get:/", "--http2-skip-connection-preface"])
+ assert a.use_http2 == True
+ assert a.ssl == True
+ assert a.http2_skip_connection_preface == True
+
+ a = cmdline.args_pathoc(["pathoc", "-c", "foo", "foo.com:8888", "get:/"])
+ assert perror.called
+ perror.reset_mock()
+
+ a = cmdline.args_pathoc(
+ ["pathoc", "-c", "foo:bar", "foo.com:8888", "get:/"])
+ assert perror.called
+ perror.reset_mock()
+
+ a = cmdline.args_pathoc(
+ [
+ "pathoc",
+ "foo.com:8888",
+ tutils.test_data.path("data/request")
+ ]
+ )
+ assert len(list(a.requests)) == 1
+
+ tutils.raises(
+ SystemExit,
+ cmdline.args_pathoc,
+ ["pathoc", "foo.com", "invalid"],
+ s, s
+ )
diff --git a/test/test_pathod.py b/test/test_pathod.py
index f85ef38d..1a3a5004 100644
--- a/test/test_pathod.py
+++ b/test/test_pathod.py
@@ -1,7 +1,7 @@
import sys
import cStringIO
from libpathod import pathod, version
-from netlib import tcp, http
+from netlib import tcp, http, http2
import tutils
@@ -269,3 +269,15 @@ class TestDaemonSSL(CommonTests):
r, _ = self.pathoc([r"get:/p/202"])
assert r[0].status_code == 202
assert self.d.last_log()["cipher"][1] > 0
+
+class TestHTTP2(tutils.DaemonTests):
+ force_http2 = True
+ ssl = True
+ noweb = True
+ noapi = True
+ nohang = True
+
+ def test_http2(self):
+ r, _ = self.pathoc(["GET:/"], ssl=True, use_http2=True)
+ print(r)
+ assert r[0].status_code == "800"
diff --git a/test/test_pathod_cmdline.py b/test/test_pathod_cmdline.py
new file mode 100644
index 00000000..829c4b32
--- /dev/null
+++ b/test/test_pathod_cmdline.py
@@ -0,0 +1,85 @@
+from libpathod import pathod_cmdline as cmdline
+import tutils
+import cStringIO
+import mock
+
+
+@mock.patch("argparse.ArgumentParser.error")
+def test_pathod(perror):
+ assert cmdline.args_pathod(["pathod"])
+
+ a = cmdline.args_pathod(
+ [
+ "pathod",
+ "--cert",
+ tutils.test_data.path("data/testkey.pem")
+ ]
+ )
+ assert a.ssl_certs
+
+ a = cmdline.args_pathod(
+ [
+ "pathod",
+ "--cert",
+ "nonexistent"
+ ]
+ )
+ assert perror.called
+ perror.reset_mock()
+
+ a = cmdline.args_pathod(
+ [
+ "pathod",
+ "-a",
+ "foo=200"
+ ]
+ )
+ assert a.anchors
+
+ a = cmdline.args_pathod(
+ [
+ "pathod",
+ "-a",
+ "foo=" + tutils.test_data.path("data/response")
+ ]
+ )
+ assert a.anchors
+
+ a = cmdline.args_pathod(
+ [
+ "pathod",
+ "-a",
+ "?=200"
+ ]
+ )
+ assert perror.called
+ perror.reset_mock()
+
+ a = cmdline.args_pathod(
+ [
+ "pathod",
+ "-a",
+ "foo"
+ ]
+ )
+ assert perror.called
+ perror.reset_mock()
+
+ a = cmdline.args_pathod(
+ [
+ "pathod",
+ "--limit-size",
+ "200k"
+ ]
+ )
+ assert a.sizelimit
+
+ a = cmdline.args_pathod(
+ [
+ "pathod",
+ "--limit-size",
+ "q"
+ ]
+ )
+ assert perror.called
+ perror.reset_mock()
diff --git a/test/tutils.py b/test/tutils.py
index c56c60d4..2184ade5 100644
--- a/test/tutils.py
+++ b/test/tutils.py
@@ -27,7 +27,7 @@ class DaemonTests(object):
klass.d = test.Daemon(
staticdir=test_data.path("data"),
anchors=[
- (re.compile("/anchor/.*"), language.parse_pathod("202:da"))
+ (re.compile("/anchor/.*"), "202:da")
],
ssl = klass.ssl,
ssloptions = so,
@@ -73,7 +73,8 @@ class DaemonTests(object):
timeout=None,
connect_to=None,
ssl=None,
- ws_read_limit=None
+ ws_read_limit=None,
+ use_http2=False,
):
"""
Returns a (messages, text log) tuple.
@@ -86,7 +87,8 @@ class DaemonTests(object):
ssl=ssl,
ws_read_limit=ws_read_limit,
timeout = timeout,
- fp = logfp
+ fp = logfp,
+ use_http2 = use_http2,
)
c.connect(connect_to)
ret = []