First-order removal of pathod API and app

6 files changed, 62 insertions, 398 deletions

diff --git a/pathod/app.py b/pathod/app.py
deleted file mode 100644
index e3216c58..00000000
--- a/pathod/app.py
+++ /dev/null
@@ -1,180 +0,0 @@
-import logging
-import pprint
-import io
-import copy
-from flask import Flask, jsonify, render_template, request, abort, make_response
-from . import version, language
-from netlib.http import user_agents
-from netlib import strutils
-
-logging.basicConfig(level="DEBUG")
-EXAMPLE_HOST = "example.com"
-EXAMPLE_WEBSOCKET_KEY = "examplekey"
-
-# pylint: disable=unused-variable
-
-
-def make_app(noapi, debug):
-    app = Flask(__name__)
-    app.debug = debug
-
-    if not noapi:
-        @app.route('/api/info')
-        def api_info():
-            return jsonify(
-                version=version.IVERSION
-            )
-
-    @app.route('/api/log')
-    def api_log():
-        return jsonify(
-            log=app.config["pathod"].get_log()
-        )
-
-    @app.route('/api/clear_log')
-    def api_clear_log():
-        app.config["pathod"].clear_log()
-        return "OK"
-
-    def render(s, cacheable, **kwargs):
-        kwargs["noapi"] = app.config["pathod"].noapi
-        kwargs["nocraft"] = app.config["pathod"].nocraft
-        kwargs["craftanchor"] = app.config["pathod"].craftanchor
-        resp = make_response(render_template(s, **kwargs), 200)
-        if cacheable:
-            resp.headers["Cache-control"] = "public, max-age=4320"
-        return resp
-
-    @app.route('/')
-    @app.route('/index.html')
-    def index():
-        return render(
-            "index.html",
-            True,
-            section="main",
-            version=version.VERSION
-        )
-
-    @app.route('/download')
-    @app.route('/download.html')
-    def download():
-        return render(
-            "download.html", True, section="download", version=version.VERSION
-        )
-
-    @app.route('/about')
-    @app.route('/about.html')
-    def about():
-        return render("about.html", True, section="about")
-
-    @app.route('/docs/pathod')
-    def docs_pathod():
-        return render(
-            "docs_pathod.html", True, section="docs", subsection="pathod"
-        )
-
-    @app.route('/docs/language')
-    def docs_language():
-        return render(
-            "docs_lang.html", True,
-            section="docs", uastrings=user_agents.UASTRINGS,
-            subsection="lang"
-        )
-
-    @app.route('/docs/pathoc')
-    def docs_pathoc():
-        return render(
-            "docs_pathoc.html", True, section="docs", subsection="pathoc"
-        )
-
-    @app.route('/docs/lib_pathod')
-    def docs_lib_pathod():
-        return render(
-            "docs_lib_pathod.html", True, section="docs", subsection="pathod"
-        )
-
-    @app.route('/docs/test')
-    def docs_test():
-        return render(
-            "docs_test.html", True, section="docs", subsection="test"
-        )
-
-    @app.route('/log')
-    def log():
-        if app.config["pathod"].noapi:
-            abort(404)
-        return render(
-            "log.html",
-            False,
-            section="log",
-            log=app.config["pathod"].get_log()
-        )
-
-    @app.route('/log/<int:lid>')
-    def onelog(lid):
-        item = app.config["pathod"].log_by_id(int(lid))
-        if not item:
-            abort(404)
-        l = pprint.pformat(item)
-        return render("onelog.html", False, section="log", alog=l, lid=lid)
-
-    def _preview(is_request):
-        if is_request:
-            template = "request_preview.html"
-        else:
-            template = "response_preview.html"
-
-        spec = request.args["spec"]
-
-        args = dict(
-            spec=spec,
-            section="main",
-            syntaxerror=None,
-            error=None,
-        )
-        if not spec.strip():
-            args["error"] = "Can't parse an empty spec."
-            return render(template, False, **args)
-
-        try:
-            if is_request:
-                r = language.parse_pathoc(spec).next()
-            else:
-                r = language.parse_pathod(spec).next()
-        except language.ParseException as v:
-            args["syntaxerror"] = str(v)
-            args["marked"] = v.marked()
-            return render(template, False, **args)
-
-        s = io.BytesIO()
-
-        settings = copy.copy(app.config["pathod"].settings)
-        settings.request_host = EXAMPLE_HOST
-        settings.websocket_key = EXAMPLE_WEBSOCKET_KEY
-
-        safe = r.preview_safe()
-        err, safe = app.config["pathod"].check_policy(
-            safe,
-            settings
-        )
-        if err:
-            args["error"] = err
-            return render(template, False, **args)
-        if is_request:
-            settings.request_host = EXAMPLE_HOST
-            language.serve(safe, s, settings)
-        else:
-            settings.websocket_key = EXAMPLE_WEBSOCKET_KEY
-            language.serve(safe, s, settings)
-
-        args["output"] = strutils.bytes_to_escaped_str(s.getvalue())
-        return render(template, False, **args)
-
-    @app.route('/response_preview')
-    def response_preview():
-        return _preview(False)
-
-    @app.route('/request_preview')
-    def request_preview():
-        return _preview(True)
-    return app
diff --git a/pathod/pathod.py b/pathod/pathod.py
index 0449c0c1..add33944 100644
--- a/pathod/pathod.py
+++ b/pathod/pathod.py
@@ -10,7 +10,7 @@ from netlib import tcp, certutils, websockets
 from netlib.exceptions import HttpException, HttpReadDisconnect, TcpTimeout, TcpDisconnect, \
     TlsException
 
-from . import version, app, language, utils, log, protocols
+from . import version, language, utils, log, protocols
 
 
 DEFAULT_CERT_DOMAIN = "pathod.net"
@@ -136,7 +136,6 @@ class PathodHandler(tcp.BaseHandler):
             path = req.path
             http_version = req.http_version
             headers = req.headers
-            body = req.content
 
             clientcert = None
             if self.clientcert:
@@ -203,24 +202,27 @@ class PathodHandler(tcp.BaseHandler):
                             self.server.craftanchor
                         )])
 
-            if anchor_gen:
-                spec = anchor_gen.next()
+            if not anchor_gen:
+                anchor_gen = iter([self.make_http_error_response(
+                    "Not found",
+                    "No valid craft request found"
+                )])
 
-                if self.use_http2 and isinstance(spec, language.http2.Response):
-                    spec.stream_id = req.stream_id
+            spec = anchor_gen.next()
 
-                lg("crafting spec: %s" % spec)
-                nexthandler, retlog["response"] = self.http_serve_crafted(
-                    spec,
-                    lg
-                )
-                if nexthandler and websocket_key:
-                    self.protocol = protocols.websockets.WebsocketsProtocol(self)
-                    return self.protocol.handle_websocket, retlog
-                else:
-                    return nexthandler, retlog
+            if self.use_http2 and isinstance(spec, language.http2.Response):
+                spec.stream_id = req.stream_id
+
+            lg("crafting spec: %s" % spec)
+            nexthandler, retlog["response"] = self.http_serve_crafted(
+                spec,
+                lg
+            )
+            if nexthandler and websocket_key:
+                self.protocol = protocols.websockets.WebsocketsProtocol(self)
+                return self.protocol.handle_websocket, retlog
             else:
-                return self.protocol.handle_http_app(method, path, headers, body, lg)
+                return nexthandler, retlog
 
     def make_http_error_response(self, reason, body=None):
         resp = self.protocol.make_error_response(reason, body)
@@ -343,8 +345,6 @@ class Pathod(tcp.TCPServer):
         self.explain = explain
         self.logfp = logfp
 
-        self.app = app.make_app(noapi, webdebug)
-        self.app.config["pathod"] = self
         self.log = []
         self.logid = 0
         self.anchors = anchors
diff --git a/pathod/protocols/http.py b/pathod/protocols/http.py
index d09b5bf2..6eefb34f 100644
--- a/pathod/protocols/http.py
+++ b/pathod/protocols/http.py
@@ -1,4 +1,3 @@
-from netlib import wsgi
 from netlib.exceptions import TlsException
 from netlib.http import http1
 from .. import version, language
@@ -11,30 +10,6 @@ class HTTPProtocol(object):
     def make_error_response(self, reason, body):
         return language.http.make_error_response(reason, body)
 
-    def handle_http_app(self, method, path, headers, body, lg):
-        """
-            Handle a request to the built-in app.
-        """
-        if self.pathod_handler.server.noweb:
-            crafted = self.pathod_handler.make_http_error_response("Access Denied")
-            language.serve(crafted, self.pathod_handler.wfile, self.pathod_handler.settings)
-            return None, dict(
-                type="error",
-                msg="Access denied: web interface disabled"
-            )
-        lg("app: %s %s" % (method, path))
-        req = wsgi.Request("http", method, path, b"HTTP/1.1", headers, body)
-        flow = wsgi.Flow(self.pathod_handler.address, req)
-        sn = self.pathod_handler.connection.getsockname()
-        a = wsgi.WSGIAdaptor(
-            self.pathod_handler.server.app,
-            sn[0],
-            self.pathod_handler.server.address.port,
-            version.NAMEVERSION
-        )
-        a.serve(flow, self.pathod_handler.wfile)
-        return self.pathod_handler.handle_http_request, None
-
     def handle_http_connect(self, connect, lg):
         """
             Handle a CONNECT request.
diff --git a/test/pathod/test_app.py b/test/pathod/test_app.py
deleted file mode 100644
index 19888c75..00000000
--- a/test/pathod/test_app.py
+++ /dev/null
@@ -1,82 +0,0 @@
-import tutils
-
-
-class TestApp(tutils.DaemonTests):
-    SSL = False
-
-    def test_index(self):
-        r = self.getpath("/")
-        assert r.status_code == 200
-        assert r.content
-
-    def test_about(self):
-        r = self.getpath("/about")
-        assert r.status_code == 200
-
-    def test_download(self):
-        r = self.getpath("/download")
-        assert r.status_code == 200
-
-    def test_docs(self):
-        assert self.getpath("/docs/pathod").status_code == 200
-        assert self.getpath("/docs/pathoc").status_code == 200
-        assert self.getpath("/docs/language").status_code == 200
-        assert self.getpath("/docs/pathod").status_code == 200
-        assert self.getpath("/docs/test").status_code == 200
-
-    def test_log(self):
-        assert self.getpath("/log").status_code == 200
-        assert self.get("200:da").status_code == 200
-        id = self.d.expect_log(1)[0]["id"]
-        assert self.getpath("/log").status_code == 200
-        assert self.getpath("/log/%s" % id).status_code == 200
-        assert self.getpath("/log/9999999").status_code == 404
-
-    def test_response_preview(self):
-        r = self.getpath("/response_preview", params=dict(spec="200"))
-        assert r.status_code == 200
-        assert 'Response' in r.content
-
-        r = self.getpath("/response_preview", params=dict(spec="foo"))
-        assert r.status_code == 200
-        assert 'Error' in r.content
-
-        r = self.getpath("/response_preview", params=dict(spec="200:b@100m"))
-        assert r.status_code == 200
-        assert "too large" in r.content
-
-        r = self.getpath("/response_preview", params=dict(spec="200:b@5k"))
-        assert r.status_code == 200
-        assert 'Response' in r.content
-
-        r = self.getpath(
-            "/response_preview",
-            params=dict(
-                spec="200:b<nonexistent"))
-        assert r.status_code == 200
-        assert 'File access denied' in r.content
-
-        r = self.getpath("/response_preview", params=dict(spec="200:b<file"))
-        assert r.status_code == 200
-        assert 'testfile' in r.content
-
-    def test_request_preview(self):
-        r = self.getpath("/request_preview", params=dict(spec="get:/"))
-        assert r.status_code == 200
-        assert 'Request' in r.content
-
-        r = self.getpath("/request_preview", params=dict(spec="foo"))
-        assert r.status_code == 200
-        assert 'Error' in r.content
-
-        r = self.getpath("/request_preview", params=dict(spec="get:/:b@100m"))
-        assert r.status_code == 200
-        assert "too large" in r.content
-
-        r = self.getpath("/request_preview", params=dict(spec="get:/:b@5k"))
-        assert r.status_code == 200
-        assert 'Request' in r.content
-
-        r = self.getpath("/request_preview", params=dict(spec=""))
-        assert r.status_code == 200
-        assert 'empty spec' in r.content
diff --git a/test/pathod/test_pathoc.py b/test/pathod/test_pathoc.py
index 18d7f672..a88cfa91 100644
--- a/test/pathod/test_pathoc.py
+++ b/test/pathod/test_pathoc.py
@@ -1,7 +1,4 @@
-import json
 from six.moves import cStringIO as StringIO
-import re
-import pytest
 from mock import Mock
 
 from netlib import http
@@ -9,10 +6,9 @@ from netlib import tcp
 from netlib.exceptions import NetlibException
 from netlib.http import http1, http2
 
-from pathod import pathoc, test, version, pathod, language
+from pathod import pathoc, language
 from netlib.tutils import raises
 import tutils
-from test.mitmproxy.tutils import skip_windows
 
 
 def test_response():
@@ -20,37 +16,7 @@ def test_response():
     assert repr(r)
 
 
-class _TestDaemon:
-    ssloptions = pathod.SSLOptions()
-
-    @classmethod
-    def setup_class(cls):
-        cls.d = test.Daemon(
-            ssl=cls.ssl,
-            ssloptions=cls.ssloptions,
-            staticdir=tutils.test_data.path("data"),
-            anchors=[
-                (re.compile("/anchor/.*"), "202")
-            ]
-        )
-
-    @classmethod
-    def teardown_class(cls):
-        cls.d.shutdown()
-
-    def setUp(self):
-        self.d.clear_log()
-
-    def test_info(self):
-        c = pathoc.Pathoc(
-            ("127.0.0.1", self.d.port),
-            ssl=self.ssl,
-            fp=None
-        )
-        c.connect()
-        resp = c.request("get:/api/info")
-        assert tuple(json.loads(resp.content)["version"]) == version.IVERSION
-
+class PathocTestDaemon(tutils.DaemonTests):
     def tval(
         self,
         requests,
@@ -77,23 +43,23 @@ class _TestDaemon:
             showsummary=showsummary,
             fp=s
         )
-        c.connect(showssl=showssl, fp=s)
-        if timeout:
-            c.settimeout(timeout)
-        for i in requests:
-            r = language.parse_pathoc(i).next()
-            if explain:
-                r = r.freeze(language.Settings())
-            try:
-                c.request(r)
-            except NetlibException:
-                pass
+        with c.connect(showssl=showssl, fp=s):
+            if timeout:
+                c.settimeout(timeout)
+            for i in requests:
+                r = language.parse_pathoc(i).next()
+                if explain:
+                    r = r.freeze(language.Settings())
+                try:
+                    c.request(r)
+                except NetlibException:
+                    pass
         return s.getvalue()
 
 
-class TestDaemonSSL(_TestDaemon):
+class TestDaemonSSL(PathocTestDaemon):
     ssl = True
-    ssloptions = pathod.SSLOptions(
+    ssloptions = dict(
         request_client_cert=True,
         sans=["test1.com", "test2.com"],
         alpn_select=b'h2',
@@ -106,11 +72,10 @@ class TestDaemonSSL(_TestDaemon):
             sni="foobar.com",
             fp=None
         )
-        c.connect()
-        c.request("get:/p/200")
-        r = c.request("get:/api/log")
-        d = json.loads(r.content)
-        assert d["log"][0]["request"]["sni"] == "foobar.com"
+        with c.connect():
+            c.request("get:/p/200")
+        log = self.d.log()
+        assert log[0]["request"]["sni"] == "foobar.com"
 
     def test_showssl(self):
         assert "certificate chain" in self.tval(["get:/p/200"], showssl=True)
@@ -122,11 +87,11 @@ class TestDaemonSSL(_TestDaemon):
             clientcert=tutils.test_data.path("data/clientcert/client.pem"),
             fp=None
         )
-        c.connect()
-        c.request("get:/p/200")
-        r = c.request("get:/api/log")
-        d = json.loads(r.content)
-        assert d["log"][0]["request"]["clientcert"]["keyinfo"]
+        with c.connect():
+            c.request("get:/p/200")
+
+        log = self.d.log()
+        assert log[0]["request"]["clientcert"]["keyinfo"]
 
     def test_http2_without_ssl(self):
         fp = StringIO()
@@ -139,7 +104,7 @@ class TestDaemonSSL(_TestDaemon):
         tutils.raises(NotImplementedError, c.connect)
 
 
-class TestDaemon(_TestDaemon):
+class TestDaemon(PathocTestDaemon):
     ssl = False
 
     def test_ssl_error(self):
@@ -163,7 +128,7 @@ class TestDaemon(_TestDaemon):
                 201])
         assert "202" in self.tval(["get:'/p/202:b@1'"], ignorecodes=[200, 201])
 
-    def test_timeout(self):
+    def _test_timeout(self):
         assert "Timeout" in self.tval(["get:'/p/200:p0,100'"], timeout=0.01)
         assert "HTTP" in self.tval(
             ["get:'/p/200:p5,100'"],
@@ -178,8 +143,8 @@ class TestDaemon(_TestDaemon):
         )
 
     def test_showresp(self):
-        reqs = ["get:/api/info:p0,0", "get:/api/info:p0,0"]
-        assert self.tval(reqs).count("200") == 2
+        reqs = ["get:/p/200:da", "get:/p/200:da"]
+        assert self.tval(reqs).count("200 OK") == 2
         assert self.tval(reqs, showresp=True).count("HTTP/1.1 200 OK") == 2
         assert self.tval(
             reqs, showresp=True, hexdump=True
@@ -195,8 +160,8 @@ class TestDaemon(_TestDaemon):
         assert "b@100" not in self.tval(reqs, explain=True)
 
     def test_showreq(self):
-        reqs = ["get:/api/info:p0,0", "get:/api/info:p0,0"]
-        assert self.tval(reqs, showreq=True).count("GET /api") == 2
+        reqs = ["get:/p/200:da", "get:/p/200:da"]
+        assert self.tval(reqs, showreq=True).count("GET /p/200") == 2
         assert self.tval(
             reqs, showreq=True, hexdump=True
         ).count("0000000000") == 2
@@ -206,23 +171,20 @@ class TestDaemon(_TestDaemon):
 
     def test_websocket_shutdown(self):
         c = pathoc.Pathoc(("127.0.0.1", self.d.port), fp=None)
-        c.connect()
-        c.request("ws:/")
-        c.stop()
+        with c.connect():
+            c.request("ws:/")
 
-    @skip_windows
-    @pytest.mark.skip(reason="race condition")
     def test_wait_finish(self):
         c = pathoc.Pathoc(
             ("127.0.0.1", self.d.port),
             fp=None,
             ws_read_limit=1
         )
-        c.connect()
-        c.request("ws:/")
-        c.request("wf:f'wf:x100'")
-        [i for i in c.wait(timeout=0, finish=False)]
-        [i for i in c.wait(timeout=0)]
+        with c.connect():
+            c.request("ws:/")
+            c.request("wf:f'wf:x100'")
+            [i for i in c.wait(timeout=0, finish=False)]
+            [i for i in c.wait(timeout=0)]
 
     def test_connect_fail(self):
         to = ("foobar", 80)
@@ -264,7 +226,7 @@ class TestDaemon(_TestDaemon):
         c.socks_connect(("example.com", 0xDEAD))
 
 
-class TestDaemonHTTP2(_TestDaemon):
+class TestDaemonHTTP2(PathocTestDaemon):
     ssl = True
 
     if tcp.HAS_ALPN:
@@ -295,10 +257,9 @@ class TestDaemonHTTP2(_TestDaemon):
             tmp_convert_to_ssl = c.convert_to_ssl
             c.convert_to_ssl = Mock()
             c.convert_to_ssl.side_effect = tmp_convert_to_ssl
-            c.connect()
-
-            _, kwargs = c.convert_to_ssl.call_args
-            assert set(kwargs['alpn_protos']) == set([b'http/1.1', b'h2'])
+            with c.connect():
+                _, kwargs = c.convert_to_ssl.call_args
+                assert set(kwargs['alpn_protos']) == set([b'http/1.1', b'h2'])
 
         def test_request(self):
             c = pathoc.Pathoc(
@@ -307,6 +268,6 @@ class TestDaemonHTTP2(_TestDaemon):
                 ssl=True,
                 use_http2=True,
             )
-            c.connect()
-            resp = c.request("get:/p/200")
+            with c.connect():
+                resp = c.request("get:/p/200")
             assert resp.status_code == 200
diff --git a/test/pathod/test_pathod.py b/test/pathod/test_pathod.py
index ec9c169f..f64b0f61 100644
--- a/test/pathod/test_pathod.py
+++ b/test/pathod/test_pathod.py
@@ -43,16 +43,6 @@ class TestTimeout(tutils.DaemonTests):
         assert self.d.last_log()["type"] == "timeout"
 
 
-class TestNoApi(tutils.DaemonTests):
-    noapi = True
-
-    def test_noapi(self):
-        assert self.getpath("/log").status_code == 404
-        r = self.getpath("/")
-        assert r.status_code == 200
-        assert "Log" not in r.content
-
-
 class TestNotAfterConnect(tutils.DaemonTests):
     ssl = False
     ssloptions = dict(