From 78ae3e4a0009d7bb2f2f7f53df2d622fc12d05dc Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 8 Jul 2015 09:34:52 +0200 Subject: use netlib.http_semantics for generic data --- libpathod/pathoc.py | 30 ++++-------------------------- 1 file changed, 4 insertions(+), 26 deletions(-) (limited to 'libpathod') diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index 39d3c3d6..7f7cbb87 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -11,7 +11,7 @@ import threading import OpenSSL.crypto -from netlib import tcp, http, http2, certutils, websockets, socks +from netlib import tcp, http, http2, http_semantics, certutils, websockets, socks import language.http import language.websockets @@ -66,25 +66,6 @@ class SSLInfo(object): return "\n".join(parts) -class Response(object): - - def __init__( - self, - httpversion, - status_code, - msg, - headers, - content, - sslinfo - ): - self.httpversion, self.status_code = httpversion, status_code - self.msg = msg - self.headers, self.content = headers, content - self.sslinfo = sslinfo - - def __repr__(self): - return "Response(%s - %s)" % (self.status_code, self.msg) - class WebsocketFrameReader(threading.Thread): @@ -429,17 +410,14 @@ class Pathoc(tcp.TCPClient): if self.use_http2: status_code, headers, body = self.protocol.read_response() - resp = Response("HTTP/2", status_code, "", headers, body, self.sslinfo) + resp = http_semantics.Response("HTTP/2", status_code, "", headers, body, self.sslinfo) else: - resp = list( - http.read_response( + resp = http.read_response( self.rfile, req["method"], None ) - ) - resp.append(self.sslinfo) - resp = Response(*resp) + resp.sslinfo = self.sslinfo except http.HttpError as v: lg("Invalid server response: %s" % v) raise -- cgit v1.2.3 From c1e84cb32296d9e448df1e9cabcc5312a85fb5fe Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 8 Jul 2015 09:35:20 +0200 Subject: use websockets protocol --- libpathod/language/http.py | 4 ++-- libpathod/pathod.py | 2 +- 2 files changed, 3 insertions(+), 3 deletions(-) (limited to 'libpathod') diff --git a/libpathod/language/http.py b/libpathod/language/http.py index 3c9df484..0c4d9419 100644 --- a/libpathod/language/http.py +++ b/libpathod/language/http.py @@ -194,7 +194,7 @@ class Response(_HTTPMessage): 1, Code(101) ) - hdrs = netlib.websockets.server_handshake_headers( + hdrs = netlib.websockets.WebsocketsProtocol.server_handshake_headers( settings.websocket_key ) for i in hdrs.lst: @@ -306,7 +306,7 @@ class Request(_HTTPMessage): 1, Method("get") ) - for i in netlib.websockets.client_handshake_headers().lst: + for i in netlib.websockets.WebsocketsProtocol.client_handshake_headers().lst: if not get_header(i[0], self.headers): tokens.append( Header( diff --git a/libpathod/pathod.py b/libpathod/pathod.py index 148fa7b2..daad64be 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -280,7 +280,7 @@ class PathodHandler(tcp.BaseHandler): retlog["cipher"] = self.get_current_cipher() m = utils.MemBool() - websocket_key = websockets.check_client_handshake(headers) + websocket_key = websockets.WebsocketsProtocol.check_client_handshake(headers) self.settings.websocket_key = websocket_key # If this is a websocket initiation, we respond with a proper -- cgit v1.2.3 From a2bf19125a92a4fa372708cf2cda7887ab62ac76 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 11 Jul 2015 23:47:43 +0200 Subject: refactor protocol-related pathod methods --- libpathod/pathod.py | 185 +++----------------------------------- libpathod/protocols/__init__.py | 1 + libpathod/protocols/http.py | 110 +++++++++++++++++++++++ libpathod/protocols/http2.py | 20 +++++ libpathod/protocols/websockets.py | 54 +++++++++++ 5 files changed, 195 insertions(+), 175 deletions(-) create mode 100644 libpathod/protocols/__init__.py create mode 100644 libpathod/protocols/http.py create mode 100644 libpathod/protocols/http2.py create mode 100644 libpathod/protocols/websockets.py (limited to 'libpathod') diff --git a/libpathod/pathod.py b/libpathod/pathod.py index daad64be..31d821c2 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -8,7 +8,7 @@ import time from netlib import tcp, http, http2, wsgi, certutils, websockets, odict -from . import version, app, language, utils, log +from . import version, app, language, utils, log, protocols import language.http import language.actions import language.exceptions @@ -88,10 +88,6 @@ class PathodHandler(tcp.BaseHandler): self.sni = connection.get_servername() def http_serve_crafted(self, crafted, logctx): - """ - This method is HTTP/1 and HTTP/2 capable. - """ - error, crafted = self.server.check_policy( crafted, self.settings ) @@ -116,114 +112,9 @@ class PathodHandler(tcp.BaseHandler): return None, response_log return self.handle_http_request, response_log - def handle_websocket(self, logger): - while True: - with logger.ctx() as lg: - started = time.time() - try: - frm = websockets.Frame.from_file(self.rfile) - except tcp.NetLibIncomplete as e: - lg("Error reading websocket frame: %s" % e) - break - ended = time.time() - lg(frm.human_readable()) - retlog = dict( - type="inbound", - protocol="websockets", - started=started, - duration=ended - started, - frame=dict( - ), - cipher=None, - ) - if self.ssl_established: - retlog["cipher"] = self.get_current_cipher() - self.addlog(retlog) - ld = language.websockets.NESTED_LEADER - if frm.payload.startswith(ld): - nest = frm.payload[len(ld):] - try: - wf_gen = language.parse_websocket_frame(nest) - except language.exceptions.ParseException as v: - logger.write( - "Parse error in reflected frame specifcation:" - " %s" % v.msg - ) - return None, None - for frm in wf_gen: - with logger.ctx() as lg: - frame_log = language.serve( - frm, - self.wfile, - self.settings - ) - lg("crafting websocket spec: %s" % frame_log["spec"]) - self.addlog(frame_log) - return self.handle_websocket, None - - def handle_http_connect(self, connect, lg): - """ - This method is HTTP/1 only. - - Handle a CONNECT request. - """ - http.read_headers(self.rfile) - self.wfile.write( - 'HTTP/1.1 200 Connection established\r\n' + - ('Proxy-agent: %s\r\n' % version.NAMEVERSION) + - '\r\n' - ) - self.wfile.flush() - if not self.server.ssloptions.not_after_connect: - try: - cert, key, chain_file_ = self.server.ssloptions.get_cert( - connect[0] - ) - self.convert_to_ssl( - cert, - key, - handle_sni=self._handle_sni, - request_client_cert=self.server.ssloptions.request_client_cert, - cipher_list=self.server.ssloptions.ciphers, - method=self.server.ssloptions.ssl_version, - alpn_select=self.server.ssloptions.alpn_select, - ) - except tcp.NetLibError as v: - s = str(v) - lg(s) - return None, dict(type="error", msg=s) - return self.handle_http_request, None - - def handle_http_app(self, method, path, headers, content, lg): - """ - This method is HTTP/1 only. - - Handle a request to the built-in app. - """ - if self.server.noweb: - crafted = self.make_http_error_response("Access Denied") - language.serve(crafted, self.wfile, self.settings) - return None, dict( - type="error", - msg="Access denied: web interface disabled" - ) - lg("app: %s %s" % (method, path)) - req = wsgi.Request("http", method, path, headers, content) - flow = wsgi.Flow(self.address, req) - sn = self.connection.getsockname() - a = wsgi.WSGIAdaptor( - self.server.app, - sn[0], - self.server.address.port, - version.NAMEVERSION - ) - a.serve(flow, self.wfile) - return self.handle_http_request, None def handle_http_request(self, logger): """ - This method is HTTP/1 and HTTP/2 capable. - Returns a (handler, log) tuple. handler: Handler for the next request, or None to disconnect @@ -231,14 +122,13 @@ class PathodHandler(tcp.BaseHandler): """ with logger.ctx() as lg: if self.use_http2: - self.protocol.perform_server_connection_preface() stream_id, headers, body = self.protocol.read_request() method = headers[':method'] path = headers[':path'] headers = odict.ODict(headers) httpversion = "" else: - req = self.read_http_request(lg) + req = self.protocol.read_request(lg) if 'next_handle' in req: return req['next_handle'] if 'errors' in req: @@ -328,68 +218,15 @@ class PathodHandler(tcp.BaseHandler): lg ) if nexthandler and websocket_key: - return self.handle_websocket, retlog + self.protocol = protocols.websockets.WebsocketsProtocol(self) + return self.protocol.handle_websocket, retlog else: return nexthandler, retlog else: - return self.handle_http_app(method, path, headers, body, lg) - - def read_http_request(self, lg): - """ - This method is HTTP/1 only. - """ - line = http.get_request_line(self.rfile) - if not line: - # Normal termination - return dict() - - m = utils.MemBool() - if m(http.parse_init_connect(line)): - return dict(next_handle=self.handle_http_connect(m.v, lg)) - elif m(http.parse_init_proxy(line)): - method, _, _, _, path, httpversion = m.v - elif m(http.parse_init_http(line)): - method, path, httpversion = m.v - else: - s = "Invalid first line: %s" % repr(line) - lg(s) - return dict(errors=dict(type="error", msg=s)) - - headers = http.read_headers(self.rfile) - if headers is None: - s = "Invalid headers" - lg(s) - return dict(errors=dict(type="error", msg=s)) - - try: - body = http.read_http_body( - self.rfile, - headers, - None, - method, - None, - True, - ) - except http.HttpError as s: - s = str(s) - lg(s) - return dict(errors=dict(type="error", msg=s)) - - return dict( - method=method, - path=path, - headers=headers, - body=body, - httpversion=httpversion) + return self.protocol.handle_http_app(method, path, headers, body, lg) def make_http_error_response(self, reason, body=None): - """ - This method is HTTP/1 and HTTP/2 capable. - """ - if self.use_http2: - resp = language.http2.make_error_response(reason, body) - else: - resp = language.http.make_error_response(reason, body) + resp = self.protocol.make_error_response(reason, body) resp.is_error_response = True return resp @@ -421,14 +258,12 @@ class PathodHandler(tcp.BaseHandler): alp = self.get_alpn_proto_negotiated() if alp == http2.HTTP2Protocol.ALPN_PROTO_H2: - self.protocol = http2.HTTP2Protocol( - self, is_server=True, dump_frames=self.http2_framedump - ) + self.protocol = protocols.http2.HTTP2Protocol(self) self.use_http2 = True - # if not self.protocol: - # # TODO: create HTTP or Websockets protocol - # self.protocol = None + if not self.protocol: + self.protocol = protocols.http.HTTPProtocol(self) + lr = self.rfile if self.server.logreq else None lw = self.wfile if self.server.logresp else None logger = log.ConnectionLogger(self.logfp, self.server.hexdump, lr, lw) diff --git a/libpathod/protocols/__init__.py b/libpathod/protocols/__init__.py new file mode 100644 index 00000000..1a8c7dab --- /dev/null +++ b/libpathod/protocols/__init__.py @@ -0,0 +1 @@ +from . import http, http2, websockets diff --git a/libpathod/protocols/http.py b/libpathod/protocols/http.py new file mode 100644 index 00000000..bccdc786 --- /dev/null +++ b/libpathod/protocols/http.py @@ -0,0 +1,110 @@ +from netlib import tcp, http, http2, wsgi, certutils, websockets, odict +from .. import version, app, language, utils, log + +class HTTPProtocol: + + def __init__(self, pathod_handler): + self.pathod_handler = pathod_handler + + def make_error_response(self, reason, body): + return language.http.make_error_response(reason, body) + + def handle_http_app(self, method, path, headers, content, lg): + """ + Handle a request to the built-in app. + """ + if self.pathod_handler.server.noweb: + crafted = self.pathod_handler.make_http_error_response("Access Denied") + language.serve(crafted, self.pathod_handler.wfile, self.pathod_handler.settings) + return None, dict( + type="error", + msg="Access denied: web interface disabled" + ) + lg("app: %s %s" % (method, path)) + req = wsgi.Request("http", method, path, headers, content) + flow = wsgi.Flow(self.pathod_handler.address, req) + sn = self.pathod_handler.connection.getsockname() + a = wsgi.WSGIAdaptor( + self.pathod_handler.server.app, + sn[0], + self.pathod_handler.server.address.port, + version.NAMEVERSION + ) + a.serve(flow, self.pathod_handler.wfile) + return self.pathod_handler.handle_http_request, None + + def handle_http_connect(self, connect, lg): + """ + Handle a CONNECT request. + """ + http.read_headers(self.pathod_handler.rfile) + self.pathod_handler.wfile.write( + 'HTTP/1.1 200 Connection established\r\n' + + ('Proxy-agent: %s\r\n' % version.NAMEVERSION) + + '\r\n' + ) + self.pathod_handler.wfile.flush() + if not self.pathod_handler.server.ssloptions.not_after_connect: + try: + cert, key, chain_file_ = self.pathod_handler.server.ssloptions.get_cert( + connect[0] + ) + self.pathod_handler.convert_to_ssl( + cert, + key, + handle_sni=self.pathod_handler._handle_sni, + request_client_cert=self.pathod_handler.server.ssloptions.request_client_cert, + cipher_list=self.pathod_handler.server.ssloptions.ciphers, + method=self.pathod_handler.server.ssloptions.ssl_version, + alpn_select=self.pathod_handler.server.ssloptions.alpn_select, + ) + except tcp.NetLibError as v: + s = str(v) + lg(s) + return None, dict(type="error", msg=s) + return self.pathod_handler.handle_http_request, None + + def read_request(self, lg): + line = http.get_request_line(self.pathod_handler.rfile) + if not line: + # Normal termination + return dict() + + m = utils.MemBool() + if m(http.parse_init_connect(line)): + return dict(next_handle=self.handle_http_connect(m.v, lg)) + elif m(http.parse_init_proxy(line)): + method, _, _, _, path, httpversion = m.v + elif m(http.parse_init_http(line)): + method, path, httpversion = m.v + else: + s = "Invalid first line: %s" % repr(line) + lg(s) + return dict(errors=dict(type="error", msg=s)) + + headers = http.read_headers(self.pathod_handler.rfile) + if headers is None: + s = "Invalid headers" + lg(s) + return dict(errors=dict(type="error", msg=s)) + + try: + body = http.read_http_body( + self.pathod_handler.rfile, + headers, + None, + method, + None, + True, + ) + except http.HttpError as s: + s = str(s) + lg(s) + return dict(errors=dict(type="error", msg=s)) + + return dict( + method=method, + path=path, + headers=headers, + body=body, + httpversion=httpversion) diff --git a/libpathod/protocols/http2.py b/libpathod/protocols/http2.py new file mode 100644 index 00000000..29c4e556 --- /dev/null +++ b/libpathod/protocols/http2.py @@ -0,0 +1,20 @@ +from netlib import tcp, http, http2, wsgi, certutils, websockets, odict +from .. import version, app, language, utils, log + +class HTTP2Protocol: + + def __init__(self, pathod_handler): + self.pathod_handler = pathod_handler + self.wire_protocol = http2.HTTP2Protocol( + self.pathod_handler, is_server=True, dump_frames=self.pathod_handler.http2_framedump + ) + + def make_error_response(self, reason, body): + return language.http2.make_error_response(reason, body) + + def read_request(self): + self.wire_protocol.perform_server_connection_preface() + return self.wire_protocol.read_request() + + def create_response(self, code, stream_id, headers, body): + return self.wire_protocol.create_response(code, stream_id, headers, body) diff --git a/libpathod/protocols/websockets.py b/libpathod/protocols/websockets.py new file mode 100644 index 00000000..334f9e9c --- /dev/null +++ b/libpathod/protocols/websockets.py @@ -0,0 +1,54 @@ +import time + +from netlib import tcp, http, http2, wsgi, certutils, websockets, odict +from .. import version, app, language, utils, log + +class WebsocketsProtocol: + + def __init__(self, pathod_handler): + self.pathod_handler = pathod_handler + + def handle_websocket(self, logger): + while True: + with logger.ctx() as lg: + started = time.time() + try: + frm = websockets.Frame.from_file(self.pathod_handler.rfile) + except tcp.NetLibIncomplete as e: + lg("Error reading websocket frame: %s" % e) + break + ended = time.time() + lg(frm.human_readable()) + retlog = dict( + type="inbound", + protocol="websockets", + started=started, + duration=ended - started, + frame=dict( + ), + cipher=None, + ) + if self.pathod_handler.ssl_established: + retlog["cipher"] = self.pathod_handler.get_current_cipher() + self.pathod_handler.addlog(retlog) + ld = language.websockets.NESTED_LEADER + if frm.payload.startswith(ld): + nest = frm.payload[len(ld):] + try: + wf_gen = language.parse_websocket_frame(nest) + except language.exceptions.ParseException as v: + logger.write( + "Parse error in reflected frame specifcation:" + " %s" % v.msg + ) + return None, None + for frm in wf_gen: + with logger.ctx() as lg: + frame_log = language.serve( + frm, + self.pathod_handler.wfile, + self.pathod_handler.settings + ) + lg("crafting websocket spec: %s" % frame_log["spec"]) + self.pathod_handler.addlog(frame_log) + return self.handle_websocket, None -- cgit v1.2.3 From 2b0465dbb93dd4bf0d6db366db597c05e92121d8 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 15 Jul 2015 22:04:25 +0200 Subject: use new netlib module names --- libpathod/app.py | 4 ++-- libpathod/language/http.py | 8 ++++---- libpathod/language/http2.py | 6 +++--- libpathod/pathoc.py | 11 ++++++----- libpathod/pathoc_cmdline.py | 5 +++-- libpathod/pathod.py | 3 ++- libpathod/protocols/http.py | 17 +++++++++-------- libpathod/protocols/http2.py | 2 +- libpathod/protocols/websockets.py | 2 +- 9 files changed, 31 insertions(+), 27 deletions(-) (limited to 'libpathod') diff --git a/libpathod/app.py b/libpathod/app.py index 4a8d2b63..debebaf2 100644 --- a/libpathod/app.py +++ b/libpathod/app.py @@ -4,7 +4,7 @@ import cStringIO import copy from flask import Flask, jsonify, render_template, request, abort, make_response from . import version, language, utils -from netlib import http_uastrings +from netlib.http import user_agents logging.basicConfig(level="DEBUG") EXAMPLE_HOST = "example.com" @@ -76,7 +76,7 @@ def make_app(noapi, debug): def docs_language(): return render( "docs_lang.html", True, - section="docs", uastrings=http_uastrings.UASTRINGS, + section="docs", uastrings=user_agents.UASTRINGS, subsection="lang" ) diff --git a/libpathod/language/http.py b/libpathod/language/http.py index 0c4d9419..380a5d64 100644 --- a/libpathod/language/http.py +++ b/libpathod/language/http.py @@ -4,7 +4,7 @@ import abc import pyparsing as pp import netlib.websockets -from netlib import http_status, http_uastrings +from netlib.http import status_codes, user_agents from . import base, exceptions, actions, message @@ -78,13 +78,13 @@ class ShortcutLocation(_HeaderMixin, base.Value): class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue): preamble = "u" - options = [i[1] for i in http_uastrings.UASTRINGS] + options = [i[1] for i in user_agents.UASTRINGS] key = base.TokValueLiteral("User-Agent") def values(self, settings): value = self.value.val if self.option_used: - value = http_uastrings.get_by_shortcut(value.lower())[2] + value = user_agents.get_by_shortcut(value.lower())[2] return self.format_header( self.key.get_generator(settings), @@ -175,7 +175,7 @@ class Response(_HTTPMessage): l.extend(self.reason.values(settings)) else: l.append( - http_status.RESPONSES.get( + status_codes.RESPONSES.get( code, "Unknown code" ) diff --git a/libpathod/language/http2.py b/libpathod/language/http2.py index 6dd93d5f..8aee9931 100644 --- a/libpathod/language/http2.py +++ b/libpathod/language/http2.py @@ -1,6 +1,6 @@ import pyparsing as pp -from netlib import http_status, http_uastrings +from netlib.http import user_agents from . import base, message """ @@ -116,13 +116,13 @@ class ShortcutLocation(_HeaderMixin, base.Value): class ShortcutUserAgent(_HeaderMixin, base.OptionsOrValue): preamble = "u" - options = [i[1] for i in http_uastrings.UASTRINGS] + options = [i[1] for i in user_agents.UASTRINGS] key = base.TokValueLiteral("user-agent") def values(self, settings): value = self.value.val if self.option_used: - value = http_uastrings.get_by_shortcut(value.lower())[2] + value = user_agents.get_by_shortcut(value.lower())[2] return ( self.key.get_generator(settings), diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index 7f7cbb87..89b0e0c9 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -11,7 +11,8 @@ import threading import OpenSSL.crypto -from netlib import tcp, http, http2, http_semantics, certutils, websockets, socks +from netlib import tcp, http, certutils, websockets, socks +from netlib.http import http1, http2 import language.http import language.websockets @@ -228,12 +229,12 @@ class Pathoc(tcp.TCPClient): l = self.rfile.readline() if not l: raise PathocError("Proxy CONNECT failed") - parsed = http.parse_response_line(l) + parsed = http.http1.parse_response_line(l) if not parsed[1] == 200: raise PathocError( "Proxy CONNECT failed: %s - %s" % (parsed[1], parsed[2]) ) - http.read_headers(self.rfile) + http.http1.read_headers(self.rfile) def socks_connect(self, connect_to): try: @@ -410,9 +411,9 @@ class Pathoc(tcp.TCPClient): if self.use_http2: status_code, headers, body = self.protocol.read_response() - resp = http_semantics.Response("HTTP/2", status_code, "", headers, body, self.sslinfo) + resp = http.Response("HTTP/2", status_code, "", headers, body, self.sslinfo) else: - resp = http.read_response( + resp = http.http1.read_response( self.rfile, req["method"], None diff --git a/libpathod/pathoc_cmdline.py b/libpathod/pathoc_cmdline.py index 8b25b4a1..58963265 100644 --- a/libpathod/pathoc_cmdline.py +++ b/libpathod/pathoc_cmdline.py @@ -3,7 +3,8 @@ import argparse import os import os.path -from netlib import http_uastrings, tcp +from netlib import tcp +from netlib.http import user_agents from . import pathoc, version, language @@ -16,7 +17,7 @@ def args_pathoc(argv, stdout=sys.stdout, stderr=sys.stderr): pa = preparser.parse_known_args(argv)[0] if pa.showua: print >> stdout, "User agent strings:" - for i in http_uastrings.UASTRINGS: + for i in user_agents.UASTRINGS: print >> stdout, " ", i[1], i[0] sys.exit(0) diff --git a/libpathod/pathod.py b/libpathod/pathod.py index 31d821c2..cfedc934 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -6,7 +6,8 @@ import threading import urllib import time -from netlib import tcp, http, http2, wsgi, certutils, websockets, odict +from netlib import tcp, http, wsgi, certutils, websockets, odict +from netlib.http import http1, http2 from . import version, app, language, utils, log, protocols import language.http diff --git a/libpathod/protocols/http.py b/libpathod/protocols/http.py index bccdc786..363af0fe 100644 --- a/libpathod/protocols/http.py +++ b/libpathod/protocols/http.py @@ -1,4 +1,5 @@ -from netlib import tcp, http, http2, wsgi, certutils, websockets, odict +from netlib import tcp, http, wsgi +from netlib.http import http1 from .. import version, app, language, utils, log class HTTPProtocol: @@ -37,7 +38,7 @@ class HTTPProtocol: """ Handle a CONNECT request. """ - http.read_headers(self.pathod_handler.rfile) + http1.read_headers(self.pathod_handler.rfile) self.pathod_handler.wfile.write( 'HTTP/1.1 200 Connection established\r\n' + ('Proxy-agent: %s\r\n' % version.NAMEVERSION) + @@ -65,31 +66,31 @@ class HTTPProtocol: return self.pathod_handler.handle_http_request, None def read_request(self, lg): - line = http.get_request_line(self.pathod_handler.rfile) + line = http1.get_request_line(self.pathod_handler.rfile) if not line: # Normal termination return dict() m = utils.MemBool() - if m(http.parse_init_connect(line)): + if m(http1.parse_init_connect(line)): return dict(next_handle=self.handle_http_connect(m.v, lg)) - elif m(http.parse_init_proxy(line)): + elif m(http1.parse_init_proxy(line)): method, _, _, _, path, httpversion = m.v - elif m(http.parse_init_http(line)): + elif m(http1.parse_init_http(line)): method, path, httpversion = m.v else: s = "Invalid first line: %s" % repr(line) lg(s) return dict(errors=dict(type="error", msg=s)) - headers = http.read_headers(self.pathod_handler.rfile) + headers = http1.read_headers(self.pathod_handler.rfile) if headers is None: s = "Invalid headers" lg(s) return dict(errors=dict(type="error", msg=s)) try: - body = http.read_http_body( + body = http1.read_http_body( self.pathod_handler.rfile, headers, None, diff --git a/libpathod/protocols/http2.py b/libpathod/protocols/http2.py index 29c4e556..d2c269ed 100644 --- a/libpathod/protocols/http2.py +++ b/libpathod/protocols/http2.py @@ -1,4 +1,4 @@ -from netlib import tcp, http, http2, wsgi, certutils, websockets, odict +from netlib.http import http2 from .. import version, app, language, utils, log class HTTP2Protocol: diff --git a/libpathod/protocols/websockets.py b/libpathod/protocols/websockets.py index 334f9e9c..890f06c8 100644 --- a/libpathod/protocols/websockets.py +++ b/libpathod/protocols/websockets.py @@ -1,6 +1,6 @@ import time -from netlib import tcp, http, http2, wsgi, certutils, websockets, odict +from netlib import tcp, http, wsgi, certutils, websockets, odict from .. import version, app, language, utils, log class WebsocketsProtocol: -- cgit v1.2.3 From 032f8b2985a7c5f89d2dde0992e4f356bf58fcaa Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 16 Jul 2015 22:57:20 +0200 Subject: use common interface for HTTP/1 and HTTP/2 --- libpathod/pathoc.py | 15 +++------------ 1 file changed, 3 insertions(+), 12 deletions(-) (limited to 'libpathod') diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index 89b0e0c9..ba15e630 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -209,8 +209,7 @@ class Pathoc(tcp.TCPClient): ) self.protocol = http2.HTTP2Protocol(self, dump_frames=self.http2_framedump) else: - # TODO: create HTTP or Websockets protocol - self.protocol = None + self.protocol = http1.HTTP1Protocol(self) self.settings = language.Settings( is_client=True, @@ -409,16 +408,8 @@ class Pathoc(tcp.TCPClient): req = language.serve(r, self.wfile, self.settings) self.wfile.flush() - if self.use_http2: - status_code, headers, body = self.protocol.read_response() - resp = http.Response("HTTP/2", status_code, "", headers, body, self.sslinfo) - else: - resp = http.http1.read_response( - self.rfile, - req["method"], - None - ) - resp.sslinfo = self.sslinfo + resp = self.protocol.read_response(req["method"], None) + resp.sslinfo = self.sslinfo except http.HttpError as v: lg("Invalid server response: %s" % v) raise -- cgit v1.2.3 From caaac5cd5a48d2bd1e50f0ff88c9d73a20ec9aa6 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Thu, 16 Jul 2015 22:57:45 +0200 Subject: inline log method --- libpathod/pathoc.py | 9 +++------ 1 file changed, 3 insertions(+), 6 deletions(-) (limited to 'libpathod') diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index ba15e630..b7802daa 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -314,11 +314,6 @@ class Pathoc(tcp.TCPClient): if self.timeout: self.settimeout(self.timeout) - def _resp_summary(self, resp): - return "<< %s %s: %s bytes" % ( - resp.status_code, utils.xrepr(resp.msg), len(resp.content) - ) - def stop(self): if self.ws_framereader: self.ws_framereader.terminate.put(None) @@ -421,7 +416,9 @@ class Pathoc(tcp.TCPClient): raise finally: if resp: - lg(self._resp_summary(resp)) + lg("<< %s %s: %s bytes" % ( + resp.status_code, utils.xrepr(resp.msg), len(resp.content) + )) if resp.status_code in self.ignorecodes: lg.suppress() return resp -- cgit v1.2.3 From 6d5a3da9294d5bc6758ded173729042573c9fe5f Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sat, 18 Jul 2015 15:54:29 +0200 Subject: use new HTTP/1 protocol --- libpathod/pathoc.py | 4 ++-- libpathod/pathod.py | 11 ++++++----- libpathod/protocols/http.py | 18 ++++++++++-------- libpathod/protocols/http2.py | 2 +- 4 files changed, 19 insertions(+), 16 deletions(-) (limited to 'libpathod') diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index b7802daa..c1bca07f 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -228,12 +228,12 @@ class Pathoc(tcp.TCPClient): l = self.rfile.readline() if not l: raise PathocError("Proxy CONNECT failed") - parsed = http.http1.parse_response_line(l) + parsed = self.protocol.parse_response_line(l) if not parsed[1] == 200: raise PathocError( "Proxy CONNECT failed: %s - %s" % (parsed[1], parsed[2]) ) - http.http1.read_headers(self.rfile) + self.protocol.read_headers() def socks_connect(self, connect_to): try: diff --git a/libpathod/pathod.py b/libpathod/pathod.py index cfedc934..f8607cca 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -123,11 +123,12 @@ class PathodHandler(tcp.BaseHandler): """ with logger.ctx() as lg: if self.use_http2: - stream_id, headers, body = self.protocol.read_request() - method = headers[':method'] - path = headers[':path'] - headers = odict.ODict(headers) - httpversion = "" + req = self.protocol.read_request() + method = req.method + path = req.path + headers = odict.ODictCaseless(req.headers) + httpversion = req.httpversion + stream_id = req.stream_id else: req = self.protocol.read_request(lg) if 'next_handle' in req: diff --git a/libpathod/protocols/http.py b/libpathod/protocols/http.py index 363af0fe..43ef0799 100644 --- a/libpathod/protocols/http.py +++ b/libpathod/protocols/http.py @@ -6,6 +6,9 @@ class HTTPProtocol: def __init__(self, pathod_handler): self.pathod_handler = pathod_handler + self.wire_protocol = http1.HTTP1Protocol( + self.pathod_handler + ) def make_error_response(self, reason, body): return language.http.make_error_response(reason, body) @@ -38,7 +41,7 @@ class HTTPProtocol: """ Handle a CONNECT request. """ - http1.read_headers(self.pathod_handler.rfile) + self.wire_protocol.read_headers() self.pathod_handler.wfile.write( 'HTTP/1.1 200 Connection established\r\n' + ('Proxy-agent: %s\r\n' % version.NAMEVERSION) + @@ -66,32 +69,31 @@ class HTTPProtocol: return self.pathod_handler.handle_http_request, None def read_request(self, lg): - line = http1.get_request_line(self.pathod_handler.rfile) + line = self.wire_protocol.get_request_line() if not line: # Normal termination return dict() m = utils.MemBool() - if m(http1.parse_init_connect(line)): + if m(self.wire_protocol.parse_init_connect(line)): return dict(next_handle=self.handle_http_connect(m.v, lg)) - elif m(http1.parse_init_proxy(line)): + elif m(self.wire_protocol.parse_init_proxy(line)): method, _, _, _, path, httpversion = m.v - elif m(http1.parse_init_http(line)): + elif m(self.wire_protocol.parse_init_http(line)): method, path, httpversion = m.v else: s = "Invalid first line: %s" % repr(line) lg(s) return dict(errors=dict(type="error", msg=s)) - headers = http1.read_headers(self.pathod_handler.rfile) + headers = self.wire_protocol.read_headers() if headers is None: s = "Invalid headers" lg(s) return dict(errors=dict(type="error", msg=s)) try: - body = http1.read_http_body( - self.pathod_handler.rfile, + body = self.wire_protocol.read_http_body( headers, None, method, diff --git a/libpathod/protocols/http2.py b/libpathod/protocols/http2.py index d2c269ed..82ec5482 100644 --- a/libpathod/protocols/http2.py +++ b/libpathod/protocols/http2.py @@ -12,7 +12,7 @@ class HTTP2Protocol: def make_error_response(self, reason, body): return language.http2.make_error_response(reason, body) - def read_request(self): + def read_request(self, lg=None): self.wire_protocol.perform_server_connection_preface() return self.wire_protocol.read_request() -- cgit v1.2.3 From 11ac387df2a49d132a08648b6cb5121224924c9d Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 19 Jul 2015 18:17:09 +0200 Subject: rename content -> body --- libpathod/pathoc.py | 2 +- libpathod/pathod.py | 2 +- libpathod/protocols/http.py | 4 ++-- 3 files changed, 4 insertions(+), 4 deletions(-) (limited to 'libpathod') diff --git a/libpathod/pathoc.py b/libpathod/pathoc.py index c1bca07f..e93e6208 100644 --- a/libpathod/pathoc.py +++ b/libpathod/pathoc.py @@ -417,7 +417,7 @@ class Pathoc(tcp.TCPClient): finally: if resp: lg("<< %s %s: %s bytes" % ( - resp.status_code, utils.xrepr(resp.msg), len(resp.content) + resp.status_code, utils.xrepr(resp.msg), len(resp.body) )) if resp.status_code in self.ignorecodes: lg.suppress() diff --git a/libpathod/pathod.py b/libpathod/pathod.py index f8607cca..6f25d0c8 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -212,7 +212,7 @@ class PathodHandler(tcp.BaseHandler): spec = anchor_gen.next() if self.use_http2 and isinstance(spec, language.http2.Response): - spec.stream_id = stream_id + spec.stream_id = req.stream_id lg("crafting spec: %s" % spec) nexthandler, retlog["response"] = self.http_serve_crafted( diff --git a/libpathod/protocols/http.py b/libpathod/protocols/http.py index 43ef0799..d6017882 100644 --- a/libpathod/protocols/http.py +++ b/libpathod/protocols/http.py @@ -13,7 +13,7 @@ class HTTPProtocol: def make_error_response(self, reason, body): return language.http.make_error_response(reason, body) - def handle_http_app(self, method, path, headers, content, lg): + def handle_http_app(self, method, path, headers, body, lg): """ Handle a request to the built-in app. """ @@ -25,7 +25,7 @@ class HTTPProtocol: msg="Access denied: web interface disabled" ) lg("app: %s %s" % (method, path)) - req = wsgi.Request("http", method, path, headers, content) + req = wsgi.Request("http", method, path, headers, body) flow = wsgi.Flow(self.pathod_handler.address, req) sn = self.pathod_handler.connection.getsockname() a = wsgi.WSGIAdaptor( -- cgit v1.2.3 From 1a9c1a8c2d6ccedd2248a676a6b3235c8cf86147 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 19 Jul 2015 20:10:07 +0200 Subject: remove code duplication --- libpathod/pathod.py | 37 ++++++++++++++++++------------------- libpathod/protocols/http.py | 45 ++------------------------------------------- 2 files changed, 20 insertions(+), 62 deletions(-) (limited to 'libpathod') diff --git a/libpathod/pathod.py b/libpathod/pathod.py index 6f25d0c8..63dd1a15 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -122,26 +122,25 @@ class PathodHandler(tcp.BaseHandler): log: A dictionary, or None """ with logger.ctx() as lg: - if self.use_http2: + try: req = self.protocol.read_request() - method = req.method - path = req.path - headers = odict.ODictCaseless(req.headers) - httpversion = req.httpversion - stream_id = req.stream_id - else: - req = self.protocol.read_request(lg) - if 'next_handle' in req: - return req['next_handle'] - if 'errors' in req: - return None, req['errors'] - if 'method' not in req or 'path' not in req: - return None, None - method = req['method'] - path = req['path'] - headers = req['headers'] - body = req['body'] - httpversion = req['httpversion'] + except http.HttpError as s: + s = str(s) + lg(s) + return None, dict(type="error", msg=s) + + if isinstance(req, http.EmptyRequest): + return None, None + + if isinstance(req, http.ConnectRequest): + print([req.host, req.port, req.path]) + return self.protocol.handle_http_connect([req.host, req.port, req.path], lg) + + method = req.method + path = req.path + httpversion = req.httpversion + headers = odict.ODictCaseless(req.headers) + body = req.body clientcert = None if self.clientcert: diff --git a/libpathod/protocols/http.py b/libpathod/protocols/http.py index d6017882..439e0ba9 100644 --- a/libpathod/protocols/http.py +++ b/libpathod/protocols/http.py @@ -68,46 +68,5 @@ class HTTPProtocol: return None, dict(type="error", msg=s) return self.pathod_handler.handle_http_request, None - def read_request(self, lg): - line = self.wire_protocol.get_request_line() - if not line: - # Normal termination - return dict() - - m = utils.MemBool() - if m(self.wire_protocol.parse_init_connect(line)): - return dict(next_handle=self.handle_http_connect(m.v, lg)) - elif m(self.wire_protocol.parse_init_proxy(line)): - method, _, _, _, path, httpversion = m.v - elif m(self.wire_protocol.parse_init_http(line)): - method, path, httpversion = m.v - else: - s = "Invalid first line: %s" % repr(line) - lg(s) - return dict(errors=dict(type="error", msg=s)) - - headers = self.wire_protocol.read_headers() - if headers is None: - s = "Invalid headers" - lg(s) - return dict(errors=dict(type="error", msg=s)) - - try: - body = self.wire_protocol.read_http_body( - headers, - None, - method, - None, - True, - ) - except http.HttpError as s: - s = str(s) - lg(s) - return dict(errors=dict(type="error", msg=s)) - - return dict( - method=method, - path=path, - headers=headers, - body=body, - httpversion=httpversion) + def read_request(self, lg=None): + return self.wire_protocol.read_request(allow_empty=True) -- cgit v1.2.3 From 3cf6bd5fb61d0b84033f4c86b3fc4a99d5287ac1 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Sun, 19 Jul 2015 22:30:06 +0200 Subject: remove debug prints --- libpathod/pathod.py | 1 - 1 file changed, 1 deletion(-) (limited to 'libpathod') diff --git a/libpathod/pathod.py b/libpathod/pathod.py index 63dd1a15..e634d7c2 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -133,7 +133,6 @@ class PathodHandler(tcp.BaseHandler): return None, None if isinstance(req, http.ConnectRequest): - print([req.host, req.port, req.path]) return self.protocol.handle_http_connect([req.host, req.port, req.path], lg) method = req.method -- cgit v1.2.3 From 6101d11d6ce412a19b542817bc8a7080a1f36758 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Mon, 20 Jul 2015 16:33:12 +0200 Subject: http2: remove odict workaround --- libpathod/pathod.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libpathod') diff --git a/libpathod/pathod.py b/libpathod/pathod.py index e634d7c2..6f07ecc5 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -138,7 +138,7 @@ class PathodHandler(tcp.BaseHandler): method = req.method path = req.path httpversion = req.httpversion - headers = odict.ODictCaseless(req.headers) + headers = req.headers body = req.body clientcert = None -- cgit v1.2.3 From dc42e335058365ed7288463dcd8e5f61b03a87a2 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Wed, 22 Jul 2015 13:04:14 +0200 Subject: fix bugs --- libpathod/pathod.py | 8 ++++---- libpathod/protocols/http.py | 2 +- 2 files changed, 5 insertions(+), 5 deletions(-) (limited to 'libpathod') diff --git a/libpathod/pathod.py b/libpathod/pathod.py index 6f07ecc5..5c813cc5 100644 --- a/libpathod/pathod.py +++ b/libpathod/pathod.py @@ -85,7 +85,7 @@ class PathodHandler(tcp.BaseHandler): self.use_http2 = False self.http2_framedump = http2_framedump - def _handle_sni(self, connection): + def handle_sni(self, connection): self.sni = connection.get_servername() def http_serve_crafted(self, crafted, logctx): @@ -132,8 +132,8 @@ class PathodHandler(tcp.BaseHandler): if isinstance(req, http.EmptyRequest): return None, None - if isinstance(req, http.ConnectRequest): - return self.protocol.handle_http_connect([req.host, req.port, req.path], lg) + if req.method == 'CONNECT': + return self.protocol.handle_http_connect([req.host, req.port, req.httpversion], lg) method = req.method path = req.path @@ -239,7 +239,7 @@ class PathodHandler(tcp.BaseHandler): self.convert_to_ssl( cert, key, - handle_sni=self._handle_sni, + handle_sni=self.handle_sni, request_client_cert=self.server.ssloptions.request_client_cert, cipher_list=self.server.ssloptions.ciphers, method=self.server.ssloptions.ssl_version, diff --git a/libpathod/protocols/http.py b/libpathod/protocols/http.py index 439e0ba9..82595126 100644 --- a/libpathod/protocols/http.py +++ b/libpathod/protocols/http.py @@ -56,7 +56,7 @@ class HTTPProtocol: self.pathod_handler.convert_to_ssl( cert, key, - handle_sni=self.pathod_handler._handle_sni, + handle_sni=self.pathod_handler.handle_sni, request_client_cert=self.pathod_handler.server.ssloptions.request_client_cert, cipher_list=self.pathod_handler.server.ssloptions.ciphers, method=self.pathod_handler.server.ssloptions.ssl_version, -- cgit v1.2.3 From 96c9c4459f0bb9b76ab34f8c8d03d0e5d28621f4 Mon Sep 17 00:00:00 2001 From: Thomas Kriechbaumer Date: Fri, 24 Jul 2015 17:39:55 +0200 Subject: fix duplicated header reading --- libpathod/protocols/http.py | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) (limited to 'libpathod') diff --git a/libpathod/protocols/http.py b/libpathod/protocols/http.py index 82595126..ca2b28b4 100644 --- a/libpathod/protocols/http.py +++ b/libpathod/protocols/http.py @@ -41,7 +41,7 @@ class HTTPProtocol: """ Handle a CONNECT request. """ - self.wire_protocol.read_headers() + self.pathod_handler.wfile.write( 'HTTP/1.1 200 Connection established\r\n' + ('Proxy-agent: %s\r\n' % version.NAMEVERSION) + -- cgit v1.2.3