diff options
| -rw-r--r-- | README.rst | 38 | ||||
| -rw-r--r-- | pathod/pathod.py | 18 | ||||
| -rw-r--r-- | pathod/test.py | 36 | ||||
| -rw-r--r-- | setup.cfg | 3 | ||||
| -rw-r--r-- | test/pathod/test_app.py | 6 | ||||
| -rw-r--r-- | test/pathod/test_pathod.py | 22 | ||||
| -rw-r--r-- | test/pathod/test_test.py | 4 | ||||
| -rw-r--r-- | test/pathod/tutils.py | 25 |
8 files changed, 87 insertions, 65 deletions
@@ -3,19 +3,24 @@ mitmproxy |travis| |coveralls| |latest_release| |python_versions| -This repository contains the **mitmproxy** and **pathod** projects, as well as their shared networking library, **netlib**. +This repository contains the **mitmproxy** and **pathod** projects, as well as +their shared networking library, **netlib**. -``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console interface. +``mitmproxy`` is an interactive, SSL-capable intercepting proxy with a console +interface. ``mitmdump`` is the command-line version of mitmproxy. Think tcpdump for HTTP. -``pathoc`` and ``pathod`` are perverse HTTP client and server applications designed to let you craft almost any conceivable HTTP request, including ones that creatively violate the standards. +``pathoc`` and ``pathod`` are perverse HTTP client and server applications +designed to let you craft almost any conceivable HTTP request, including ones +that creatively violate the standards. Documentation & Help -------------------- -Documentation, tutorials and precompiled binaries can be found on the mitmproxy and pathod websites. +Documentation, tutorials and precompiled binaries can be found on the mitmproxy +and pathod websites. |mitmproxy_site| |pathod_site| @@ -39,8 +44,8 @@ Hacking ------- To get started hacking on mitmproxy, make sure you have Python_ 2.7.x. with -virtualenv_ installed (you can find installation instructions for virtualenv here_). -Then do the following: +virtualenv_ installed (you can find installation instructions for virtualenv +here_). Then do the following: .. code-block:: text @@ -49,10 +54,11 @@ Then do the following: ./dev.sh -The *dev* script will create a virtualenv environment in a directory called "venv", -and install all mandatory and optional dependencies into it. -The primary mitmproxy components - mitmproxy, netlib and pathod - are installed as "editable", -so any changes to the source in the repository will be reflected live in the virtualenv. +The *dev* script will create a virtualenv environment in a directory called +"venv", and install all mandatory and optional dependencies into it. The +primary mitmproxy components - mitmproxy, netlib and pathod - are installed as +"editable", so any changes to the source in the repository will be reflected +live in the virtualenv. To confirm that you're up and running, activate the virtualenv, and run the mitmproxy test suite: @@ -63,9 +69,9 @@ mitmproxy test suite: py.test Note that the main executables for the project - ``mitmdump``, ``mitmproxy``, -``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the virtualenv. After activating the -virtualenv, they will be on your $PATH, and you can run them like any other -command: +``mitmweb``, ``pathod``, and ``pathoc`` - are all created within the +virtualenv. After activating the virtualenv, they will be on your $PATH, and +you can run them like any other command: .. code-block:: text @@ -92,9 +98,9 @@ suite. The project tries to maintain 100% test coverage. Documentation ------------- -The mitmproxy documentation is build using Sphinx_, which is installed automatically if you set up a development -environment as described above. -After installation, you can render the documentation like this: +The mitmproxy documentation is build using Sphinx_, which is installed +automatically if you set up a development environment as described above. After +installation, you can render the documentation like this: .. code-block:: text diff --git a/pathod/pathod.py b/pathod/pathod.py index 7795df0e..0449c0c1 100644 --- a/pathod/pathod.py +++ b/pathod/pathod.py @@ -353,6 +353,8 @@ class Pathod(tcp.TCPServer): staticdir=self.staticdir ) + self.loglock = threading.Lock() + def check_policy(self, req, settings): """ A policy check that verifies the request size is within limits. @@ -403,8 +405,7 @@ class Pathod(tcp.TCPServer): def add_log(self, d): if not self.noapi: - lock = threading.Lock() - with lock: + with self.loglock: d["id"] = self.logid self.log.insert(0, d) if len(self.log) > self.LOGBUF: @@ -413,17 +414,18 @@ class Pathod(tcp.TCPServer): return d["id"] def clear_log(self): - lock = threading.Lock() - with lock: + with self.loglock: self.log = [] def log_by_id(self, identifier): - for i in self.log: - if i["id"] == identifier: - return i + with self.loglock: + for i in self.log: + if i["id"] == identifier: + return i def get_log(self): - return self.log + with self.loglock: + return self.log def main(args): # pragma: no cover diff --git a/pathod/test.py b/pathod/test.py index 23b7a5b6..32b37731 100644 --- a/pathod/test.py +++ b/pathod/test.py @@ -1,13 +1,11 @@ from six.moves import cStringIO as StringIO import threading +import time + from six.moves import queue -import requests -import requests.packages.urllib3 from . import pathod -requests.packages.urllib3.disable_warnings() - class Daemon: IFACE = "127.0.0.1" @@ -39,39 +37,40 @@ class Daemon: """ return "%s/p/%s" % (self.urlbase, spec) - def info(self): - """ - Return some basic info about the remote daemon. - """ - resp = requests.get("%s/api/info" % self.urlbase, verify=False) - return resp.json() - def text_log(self): return self.logfp.getvalue() + def expect_log(self, n, timeout=5): + l = [] + start = time.time() + while True: + l = self.log() + if time.time() - start >= timeout: + return None + if len(l) >= n: + break + return l + def last_log(self): """ Returns the last logged request, or None. """ - l = self.log() + l = self.expect_log(1) if not l: return None - return l[0] + return l[-1] def log(self): """ Return the log buffer as a list of dictionaries. """ - resp = requests.get("%s/api/log" % self.urlbase, verify=False) - return resp.json()["log"] + return self.thread.server.get_log() def clear_log(self): """ Clear the log. """ - self.logfp.truncate(0) - resp = requests.get("%s/api/clear_log" % self.urlbase, verify=False) - return resp.ok + return self.thread.server.clear_log() def shutdown(self): """ @@ -88,6 +87,7 @@ class _PaThread(threading.Thread): self.name = "PathodThread" self.iface, self.q, self.ssl = iface, q, ssl self.daemonargs = daemonargs + self.server = None def run(self): self.server = pathod.Pathod( @@ -11,8 +11,7 @@ addopts = --capture=no [coverage:run] branch = True -include = *mitmproxy*, *netlib*, *pathod* -omit = *contrib*, *tnetstring*, *platform*, *console*, *main.py +omit = *contrib*, *tnetstring*, *platform*, *main.py [coverage:report] show_missing = True diff --git a/test/pathod/test_app.py b/test/pathod/test_app.py index ac89c44c..fbaa773c 100644 --- a/test/pathod/test_app.py +++ b/test/pathod/test_app.py @@ -11,11 +11,11 @@ class TestApp(tutils.DaemonTests): def test_about(self): r = self.getpath("/about") - assert r.ok + assert r.status_code == 200 def test_download(self): r = self.getpath("/download") - assert r.ok + assert r.status_code == 200 def test_docs(self): assert self.getpath("/docs/pathod").status_code == 200 @@ -27,7 +27,7 @@ class TestApp(tutils.DaemonTests): def test_log(self): assert self.getpath("/log").status_code == 200 assert self.get("200:da").status_code == 200 - id = self.d.log()[0]["id"] + id = self.d.expect_log(1)[0]["id"] assert self.getpath("/log").status_code == 200 assert self.getpath("/log/%s" % id).status_code == 200 assert self.getpath("/log/9999999").status_code == 404 diff --git a/test/pathod/test_pathod.py b/test/pathod/test_pathod.py index 4d969158..ec9c169f 100644 --- a/test/pathod/test_pathod.py +++ b/test/pathod/test_pathod.py @@ -1,7 +1,6 @@ from six.moves import cStringIO as StringIO -import pytest -from pathod import pathod, version +from pathod import pathod from netlib import tcp from netlib.exceptions import HttpException, TlsException import tutils @@ -129,7 +128,6 @@ class CommonTests(tutils.DaemonTests): assert self.d.last_log() # FIXME: Other binary data elements - @pytest.mark.skip(reason="race condition") def test_sizelimit(self): r = self.get("200:b@1g") assert r.status_code == 800 @@ -140,21 +138,15 @@ class CommonTests(tutils.DaemonTests): r, _ = self.pathoc([r"get:'/p/200':i0,'\r\n'"]) assert r[0].status_code == 200 - def test_info(self): - assert tuple(self.d.info()["version"]) == version.IVERSION - - @pytest.mark.skip(reason="race condition") def test_logs(self): - assert self.d.clear_log() - assert not self.d.last_log() + self.d.clear_log() assert self.get("202:da") - assert len(self.d.log()) == 1 - assert self.d.clear_log() + assert self.d.expect_log(1) + self.d.clear_log() assert len(self.d.log()) == 0 def test_disconnect(self): - rsp = self.get("202:b@100k:d200") - assert len(rsp.content) < 200 + tutils.raises("unexpected eof", self.get, "202:b@100k:d200") def test_parserr(self): rsp = self.get("400:msg,b:") @@ -166,7 +158,7 @@ class CommonTests(tutils.DaemonTests): assert rsp.content.strip() == "testfile" def test_anchor(self): - rsp = self.getpath("anchor/foo") + rsp = self.getpath("/anchor/foo") assert rsp.status_code == 202 def test_invalid_first_line(self): @@ -223,7 +215,6 @@ class CommonTests(tutils.DaemonTests): ) assert r[1].payload == "test" - @pytest.mark.skip(reason="race condition") def test_websocket_frame_reflect_error(self): r, _ = self.pathoc( ["ws:/p/", "wf:-mask:knone:f'wf:b@10':i13,'a'"], @@ -233,7 +224,6 @@ class CommonTests(tutils.DaemonTests): # FIXME: Race Condition? assert "Parse error" in self.d.text_log() - @pytest.mark.skip(reason="race condition") def test_websocket_frame_disconnect_error(self): self.pathoc(["ws:/p/", "wf:b@10:d3"], ws_read_limit=0) assert self.d.last_log() diff --git a/test/pathod/test_test.py b/test/pathod/test_test.py index cee286a4..6399894e 100644 --- a/test/pathod/test_test.py +++ b/test/pathod/test_test.py @@ -2,6 +2,10 @@ import logging import requests from pathod import test import tutils + +import requests.packages.urllib3 + +requests.packages.urllib3.disable_warnings() logging.disable(logging.CRITICAL) diff --git a/test/pathod/tutils.py b/test/pathod/tutils.py index f7bb22e5..a99a2fd3 100644 --- a/test/pathod/tutils.py +++ b/test/pathod/tutils.py @@ -3,6 +3,7 @@ import re import shutil import requests from six.moves import cStringIO as StringIO +import urllib from netlib import tcp from netlib import utils @@ -66,7 +67,7 @@ class DaemonTests(object): if not (self.noweb or self.noapi): self.d.clear_log() - def getpath(self, path, params=None): + def _getpath(self, path, params=None): scheme = "https" if self.ssl else "http" resp = requests.get( "%s://localhost:%s/%s" % ( @@ -79,8 +80,28 @@ class DaemonTests(object): ) return resp + def getpath(self, path, params=None): + logfp = StringIO() + c = pathoc.Pathoc( + ("localhost", self.d.port), + ssl=self.ssl, + fp=logfp, + ) + c.connect() + if params: + path = path + "?" + urllib.urlencode(params) + resp = c.request("get:%s" % path) + return resp + def get(self, spec): - resp = requests.get(self.d.p(spec), verify=False) + logfp = StringIO() + c = pathoc.Pathoc( + ("localhost", self.d.port), + ssl=self.ssl, + fp=logfp, + ) + c.connect() + resp = c.request("get:/p/%s" % urllib.quote(spec).encode("string_escape")) return resp def pathoc( |