diff options
| -rw-r--r-- | examples/har_extractor.py | 2 | ||||
| -rw-r--r-- | libmproxy/cmdline.py | 8 | ||||
| -rw-r--r-- | libmproxy/console/common.py | 2 | ||||
| -rw-r--r-- | libmproxy/console/flowview.py | 2 | ||||
| -rw-r--r-- | libmproxy/dump.py | 2 | ||||
| -rw-r--r-- | libmproxy/exceptions.py | 8 | ||||
| -rw-r--r-- | libmproxy/flow.py | 12 | ||||
| -rw-r--r-- | libmproxy/models/http.py | 25 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 191 | ||||
| -rw-r--r-- | libmproxy/protocol/http_replay.py | 31 | ||||
| -rw-r--r-- | libmproxy/protocol/tls.py | 6 | ||||
| -rw-r--r-- | libmproxy/proxy/root_context.py | 7 | ||||
| -rw-r--r-- | libmproxy/proxy/server.py | 4 | ||||
| -rw-r--r-- | libmproxy/web/app.py | 6 | ||||
| -rw-r--r-- | test/test_dump.py | 10 | ||||
| -rw-r--r-- | test/test_filt.py | 5 | ||||
| -rw-r--r-- | test/test_flow.py | 8 | ||||
| -rw-r--r-- | test/test_fuzzing.py | 8 | ||||
| -rw-r--r-- | test/test_protocol_http.py | 52 | ||||
| -rw-r--r-- | test/test_proxy.py | 8 | ||||
| -rw-r--r-- | test/test_server.py | 52 | ||||
| -rw-r--r-- | test/tutils.py | 2 | ||||
| -rw-r--r-- | web/src/js/components/flowview/messages.js | 6 |
23 files changed, 206 insertions, 251 deletions
diff --git a/examples/har_extractor.py b/examples/har_extractor.py index bc784dc4..f16f1909 100644 --- a/examples/har_extractor.py +++ b/examples/har_extractor.py @@ -128,7 +128,7 @@ def response(context, flow): request_query_string = [{"name": k, "value": v} for k, v in flow.request.get_query()] - request_http_version = ".".join([str(v) for v in flow.request.httpversion]) + request_http_version = flow.request.httpversion # Cookies are shaped as tuples by MITMProxy. request_cookies = [{"name": k.strip(), "value": v[0]} for k, v in (flow.request.get_cookies() or {}).iteritems()] diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index 3779953f..16678486 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -103,11 +103,15 @@ def parse_setheader(s): def parse_server_spec(url): - p = netlib.utils.parse_url(url) - if not p or not p[1] or p[0] not in ("http", "https"): + try: + p = netlib.utils.parse_url(url) + if p[0] not in ("http", "https"): + raise ValueError() + except ValueError: raise configargparse.ArgumentTypeError( "Invalid server specification: %s" % url ) + address = Address(p[1:3]) scheme = p[0].lower() return config.ServerSpec(scheme, address) diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index ae3dd61e..13374b25 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -4,7 +4,7 @@ import urwid import urwid.util import os -from netlib.http.semantics import CONTENT_MISSING +from netlib.http import CONTENT_MISSING import netlib.utils from .. import utils diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 3e13fab4..8220c1e7 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -6,7 +6,7 @@ import sys import urwid from netlib import odict -from netlib.http.semantics import CONTENT_MISSING, Headers +from netlib.http import CONTENT_MISSING, Headers from . import common, grideditor, signals, searchable, tabs from . import flowdetailview from .. import utils, controller, contentviews diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 9fc9e1b8..3915d4c8 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -6,7 +6,7 @@ import traceback import click import itertools -from netlib.http.semantics import CONTENT_MISSING +from netlib.http import CONTENT_MISSING import netlib.utils from . import flow, filt, contentviews from .exceptions import ContentViewException diff --git a/libmproxy/exceptions.py b/libmproxy/exceptions.py index d916f457..b55201be 100644 --- a/libmproxy/exceptions.py +++ b/libmproxy/exceptions.py @@ -34,17 +34,17 @@ class Socks5Exception(ProtocolException): pass -class HttpException(ProtocolException): +class HttpProtocolException(ProtocolException): pass -class InvalidCredentials(HttpException): +class ServerException(ProxyException): pass -class ServerException(ProxyException): +class ContentViewException(ProxyException): pass -class ContentViewException(ProxyException): +class ReplayException(ProxyException): pass diff --git a/libmproxy/flow.py b/libmproxy/flow.py index d037d36e..d735b9ec 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -12,7 +12,8 @@ import urlparse from netlib import wsgi -from netlib.http.semantics import CONTENT_MISSING, Headers +from netlib.exceptions import HttpException +from netlib.http import CONTENT_MISSING, Headers, http1 import netlib.http from . import controller, tnetstring, filt, script, version from .onboarding import app @@ -161,9 +162,8 @@ class StreamLargeBodies(object): def run(self, flow, is_request): r = flow.request if is_request else flow.response - code = flow.response.code if flow.response else None - expected_size = netlib.http.http1.HTTP1Protocol.expected_http_body_size( - r.headers, is_request, flow.request.method, code + expected_size = http1.expected_http_body_size( + flow.request, flow.response if not is_request else None ) if not (0 <= expected_size <= self.max_size): # r.stream may already be a callable, which we want to preserve. @@ -842,7 +842,7 @@ class FlowMaster(controller.Master): host, port, path, - (1, 1), + b"HTTP/1.1", headers, None, None, @@ -1000,7 +1000,7 @@ class FlowMaster(controller.Master): try: if self.stream_large_bodies: self.stream_large_bodies.run(f, False) - except netlib.http.HttpError: + except HttpException: f.reply(Kill) return diff --git a/libmproxy/models/http.py b/libmproxy/models/http.py index 0d5e53b5..0769d2d0 100644 --- a/libmproxy/models/http.py +++ b/libmproxy/models/http.py @@ -6,18 +6,17 @@ import time from libmproxy import utils from netlib import encoding -from netlib.http import status_codes, Headers +from netlib.http import status_codes, Headers, Request, Response, CONTENT_MISSING from netlib.tcp import Address -from netlib.http.semantics import Request, Response, CONTENT_MISSING from .. import version, stateobject from .flow import Flow class MessageMixin(stateobject.StateObject): _stateobject_attributes = dict( - httpversion=tuple, + httpversion=bytes, headers=Headers, - body=str, + body=bytes, timestamp_start=float, timestamp_end=float ) @@ -120,7 +119,7 @@ class HTTPRequest(MessageMixin, Request): path: Path portion of the URL (not present in authority-form) - httpversion: HTTP version tuple, e.g. (1,1) + httpversion: HTTP version, e.g. "HTTP/1.1" headers: Headers object @@ -186,11 +185,11 @@ class HTTPRequest(MessageMixin, Request): _stateobject_attributes = MessageMixin._stateobject_attributes.copy() _stateobject_attributes.update( form_in=str, - method=str, - scheme=str, - host=str, + method=bytes, + scheme=bytes, + host=bytes, port=int, - path=str, + path=bytes, form_out=str, is_replay=bool ) @@ -267,7 +266,7 @@ class HTTPResponse(MessageMixin, Response): Exposes the following attributes: - httpversion: HTTP version tuple, e.g. (1, 0), (1, 1), or (2, 0) + httpversion: HTTP version, e.g. "HTTP/1.1" status_code: HTTP response status code @@ -312,7 +311,7 @@ class HTTPResponse(MessageMixin, Response): _stateobject_attributes = MessageMixin._stateobject_attributes.copy() _stateobject_attributes.update( status_code=int, - msg=str + msg=bytes ) @classmethod @@ -532,7 +531,7 @@ def make_error_response(status_code, message, headers=None): ) return HTTPResponse( - (1, 1), # FIXME: Should be a string. + b"HTTP/1.1", status_code, response, headers, @@ -543,7 +542,7 @@ def make_error_response(status_code, message, headers=None): def make_connect_request(address): address = Address.wrap(address) return HTTPRequest( - "authority", "CONNECT", None, address.host, address.port, None, (1, 1), + "authority", "CONNECT", None, address.host, address.port, None, b"HTTP/1.1", Headers(), "" ) diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 230f2be9..a876df41 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -6,14 +6,14 @@ import traceback import six from netlib import tcp -from netlib.http import http1, HttpErrorConnClosed, HttpError, Headers -from netlib.http.semantics import CONTENT_MISSING +from netlib.exceptions import HttpException, HttpReadDisconnect +from netlib.http import http1, Headers +from netlib.http import CONTENT_MISSING from netlib.tcp import NetLibError, Address -from netlib.http.http1 import HTTP1Protocol -from netlib.http.http2 import HTTP2Protocol +from netlib.http.http2.connections import HTTP2Protocol from netlib.http.http2.frame import GoAwayFrame, PriorityFrame, WindowUpdateFrame from .. import utils -from ..exceptions import InvalidCredentials, HttpException, ProtocolException +from ..exceptions import HttpProtocolException, ProtocolException from ..models import ( HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error ) @@ -45,14 +45,14 @@ class _StreamingHttpLayer(_HttpLayer): def read_response_headers(self): raise NotImplementedError - def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + def read_response_body(self, request, response): raise NotImplementedError() yield "this is a generator" # pragma: no cover - def read_response(self, request_method): + def read_response(self, request): response = self.read_response_headers() - response.body = "".join( - self.read_response_body(response.headers, request_method, response.code) + response.body = b"".join( + self.read_response_body(request, response) ) return response @@ -64,7 +64,7 @@ class _StreamingHttpLayer(_HttpLayer): def send_response(self, response): if response.body == CONTENT_MISSING: - raise HttpError(502, "Cannot assemble flow with CONTENT_MISSING") + raise HttpException("Cannot assemble flow with CONTENT_MISSING") self.send_response_headers(response) self.send_response_body(response, [response.body]) @@ -73,48 +73,31 @@ class Http1Layer(_StreamingHttpLayer): def __init__(self, ctx, mode): super(Http1Layer, self).__init__(ctx) self.mode = mode - self.client_protocol = HTTP1Protocol(self.client_conn) - self.server_protocol = HTTP1Protocol(self.server_conn) def read_request(self): - return HTTPRequest.from_protocol( - self.client_protocol, - body_size_limit=self.config.body_size_limit - ) + req = http1.read_request(self.client_conn.rfile, body_size_limit=self.config.body_size_limit) + return HTTPRequest.wrap(req) def send_request(self, request): - self.server_conn.send(self.server_protocol.assemble(request)) + self.server_conn.wfile.write(http1.assemble_request(request)) + self.server_conn.wfile.flush() def read_response_headers(self): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=None, # does not matter if we don't read the body. - body_size_limit=self.config.body_size_limit, - include_body=False - ) + resp = http1.read_response_head(self.server_conn.rfile) + return HTTPResponse.wrap(resp) - def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): - return self.server_protocol.read_http_body_chunked( - headers, - self.config.body_size_limit, - request_method, - response_code, - False, - max_chunk_size - ) + def read_response_body(self, request, response): + expected_size = http1.expected_http_body_size(request, response) + return http1.read_body(self.server_conn.rfile, expected_size, self.config.body_size_limit) def send_response_headers(self, response): - h = self.client_protocol._assemble_response_first_line(response) - self.client_conn.wfile.write(h + "\r\n") - h = self.client_protocol._assemble_response_headers( - response, - preserve_transfer_encoding=True - ) - self.client_conn.wfile.write(h + "\r\n") + raw = http1.assemble_response_head(response, preserve_transfer_encoding=True) + self.client_conn.wfile.write(raw) self.client_conn.wfile.flush() def send_response_body(self, response, chunks): - if self.client_protocol.has_chunked_encoding(response.headers): + if b"chunked" in response.headers.get(b"transfer-encoding", b"").lower(): + # TODO: Move this into netlib.http.http1 chunks = itertools.chain( ( "{:x}\r\n{}\r\n".format(len(chunk), chunk) @@ -127,36 +110,24 @@ class Http1Layer(_StreamingHttpLayer): self.client_conn.wfile.flush() def check_close_connection(self, flow): - close_connection = ( - http1.HTTP1Protocol.connection_close( - flow.request.httpversion, - flow.request.headers - ) or http1.HTTP1Protocol.connection_close( - flow.response.httpversion, - flow.response.headers - ) or http1.HTTP1Protocol.expected_http_body_size( - flow.response.headers, - False, - flow.request.method, - flow.response.code) == -1 + request_close = http1.connection_close( + flow.request.httpversion, + flow.request.headers ) + response_close = http1.connection_close( + flow.response.httpversion, + flow.response.headers + ) + read_until_eof = http1.expected_http_body_size(flow.request, flow.response) == -1 + close_connection = request_close or response_close or read_until_eof if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 + # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313: + # Charles Proxy sends a CONNECT response with HTTP/1.0 # and no Content-Length header return False return close_connection - def connect(self): - self.ctx.connect() - self.server_protocol = HTTP1Protocol(self.server_conn) - - def set_server(self, *args, **kwargs): - self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP1Protocol(self.server_conn) - def __call__(self): layer = HttpLayer(self, self.mode) layer() @@ -184,10 +155,10 @@ class Http2Layer(_HttpLayer): # TODO: implement flow control and WINDOW_UPDATE frames self.server_conn.send(self.server_protocol.assemble(message)) - def read_response(self, request_method): + def read_response(self, request): return HTTPResponse.from_protocol( self.server_protocol, - request_method=request_method, + request_method=request.method, body_size_limit=self.config.body_size_limit, include_body=True, stream_id=self._stream_id @@ -295,7 +266,7 @@ class UpstreamConnectLayer(Layer): def _send_connect_request(self): self.send_request(self.connect_request) - resp = self.read_response("CONNECT") + resp = self.read_response(self.connect_request) if resp.code != 200: raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") @@ -337,28 +308,31 @@ class HttpLayer(Layer): self.__original_server_conn = self.server_conn while True: try: - flow = HTTPFlow(self.client_conn, self.server_conn, live=self) + request = self.read_request() + self.log("request", "debug", [repr(request)]) - try: - request = self.read_request() - except tcp.NetLibError: - # don't throw an error for disconnects that happen - # before/between requests. + # Handle Proxy Authentication + if not self.authenticate(request): return - self.log("request", "debug", [repr(request)]) + # Make sure that the incoming request matches our expectations + self.validate_request(request) - # Handle Proxy Authentication - self.authenticate(request) + except HttpReadDisconnect: + # don't throw an error for disconnects that happen before/between requests. + return + except (HttpException, NetLibError) as e: + self.send_error_response(400, repr(e)) + six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) + + try: + flow = HTTPFlow(self.client_conn, self.server_conn, live=self) # Regular Proxy Mode: Handle CONNECT if self.mode == "regular" and request.form_in == "authority": self.handle_regular_mode_connect(request) return - # Make sure that the incoming request matches our expectations - self.validate_request(request) - flow.request = request self.process_request_hook(flow) @@ -384,30 +358,26 @@ class HttpLayer(Layer): self.handle_upstream_mode_connect(flow.request.copy()) return - except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: - error_propagated = False - if flow.request and not flow.response: + except (HttpException, NetLibError) as e: + self.send_error_response(502, repr(e)) + + if not flow.response: flow.error = Error(str(e)) self.channel.ask("error", flow) self.log(traceback.format_exc(), "debug") - error_propagated = True - - try: - self.send_response(make_error_response( - getattr(e, "code", 502), - repr(e) - )) - except NetLibError: - pass - - if not error_propagated: - if isinstance(e, ProtocolException): - six.reraise(ProtocolException, e, sys.exc_info()[2]) - else: - six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) + return + else: + six.reraise(ProtocolException, ProtocolException("Error in HTTP connection: %s" % repr(e)), sys.exc_info()[2]) finally: flow.live = False + def send_error_response(self, code, message): + try: + response = make_error_response(code, message) + self.send_response(response) + except NetLibError: + pass + def change_upstream_proxy_server(self, address): # Make set_upstream_proxy_server always available, # even if there's no UpstreamConnectLayer @@ -435,10 +405,8 @@ class HttpLayer(Layer): # First send the headers and then transfer the response incrementally self.send_response_headers(flow.response) chunks = self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code, - max_chunk_size=4096 + flow.request, + flow.response ) if callable(flow.response.stream): chunks = flow.response.stream(chunks) @@ -451,11 +419,11 @@ class HttpLayer(Layer): if self.supports_streaming: flow.response = self.read_response_headers() else: - flow.response = self.read_response(flow.request.method) + flow.response = self.read_response(flow.request) try: get_response() - except (tcp.NetLibError, HttpErrorConnClosed) as v: + except (tcp.NetLibError, HttpException) as v: self.log( "server communication error: %s" % repr(v), level="debug" @@ -485,10 +453,9 @@ class HttpLayer(Layer): if flow.response.stream: flow.response.content = CONTENT_MISSING else: - flow.response.content = "".join(self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code + flow.response.content = b"".join(self.read_response_body( + flow.request, + flow.response )) flow.response.timestamp_end = utils.timestamp() @@ -543,7 +510,7 @@ class HttpLayer(Layer): if not self.server_conn: self.connect() if tls: - raise HttpException("Cannot change scheme in upstream proxy mode.") + raise HttpProtocolException("Cannot change scheme in upstream proxy mode.") """ # This is a very ugly (untested) workaround to solve a very ugly problem. if self.server_conn and self.server_conn.tls_established and not ssl: @@ -561,12 +528,10 @@ class HttpLayer(Layer): def validate_request(self, request): if request.form_in == "absolute" and request.scheme != "http": - self.send_response( - make_error_response(400, "Invalid request scheme: %s" % request.scheme)) raise HttpException("Invalid request scheme: %s" % request.scheme) expected_request_forms = { - "regular": ("absolute",), # an authority request would already be handled. + "regular": ("authority", "absolute",), "upstream": ("authority", "absolute"), "transparent": ("relative",) } @@ -576,10 +541,9 @@ class HttpLayer(Layer): err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( " or ".join(allowed_request_forms), request.form_in ) - self.send_response(make_error_response(400, err_message)) raise HttpException(err_message) - if self.mode == "regular": + if self.mode == "regular" and request.form_in == "absolute": request.form_out = "relative" def authenticate(self, request): @@ -592,4 +556,5 @@ class HttpLayer(Layer): "Proxy Authentication Required", Headers(**self.config.authenticator.auth_challenge_headers()) )) - raise InvalidCredentials("Proxy Authentication Required") + return False + return True diff --git a/libmproxy/protocol/http_replay.py b/libmproxy/protocol/http_replay.py index a9ee5506..9d61d75c 100644 --- a/libmproxy/protocol/http_replay.py +++ b/libmproxy/protocol/http_replay.py @@ -1,8 +1,9 @@ from __future__ import (absolute_import, print_function, division) import threading +from libmproxy.exceptions import ReplayException +from netlib.exceptions import HttpException +from netlib.http import http1 -from netlib.http import HttpError -from netlib.http.http1 import HTTP1Protocol from netlib.tcp import NetLibError from ..controller import Channel from ..models import Error, HTTPResponse, ServerConnection, make_connect_request @@ -47,13 +48,17 @@ class RequestReplayThread(threading.Thread): server_address = self.config.upstream_server.address server = ServerConnection(server_address) server.connect() - protocol = HTTP1Protocol(server) if r.scheme == "https": connect_request = make_connect_request((r.host, r.port)) - server.send(protocol.assemble(connect_request)) - resp = protocol.read_response("CONNECT") + server.wfile.write(http1.assemble_request(connect_request)) + server.wfile.flush() + resp = http1.read_response( + server.rfile, + connect_request, + body_size_limit=self.config.body_size_limit + ) if resp.code != 200: - raise HttpError(502, "Upstream server refuses CONNECT request") + raise ReplayException("Upstream server refuses CONNECT request") server.establish_ssl( self.config.clientcerts, sni=self.flow.server_conn.sni @@ -65,7 +70,6 @@ class RequestReplayThread(threading.Thread): server_address = (r.host, r.port) server = ServerConnection(server_address) server.connect() - protocol = HTTP1Protocol(server) if r.scheme == "https": server.establish_ssl( self.config.clientcerts, @@ -73,18 +77,19 @@ class RequestReplayThread(threading.Thread): ) r.form_out = "relative" - server.send(protocol.assemble(r)) + server.wfile.write(http1.assemble_request(r)) + server.wfile.flush() self.flow.server_conn = server - self.flow.response = HTTPResponse.from_protocol( - protocol, - r.method, - body_size_limit=self.config.body_size_limit, + self.flow.response = http1.read_response( + server.rfile, + r, + body_size_limit=self.config.body_size_limit ) if self.channel: response_reply = self.channel.ask("response", self.flow) if response_reply == Kill: raise Kill() - except (HttpError, NetLibError) as v: + except (ReplayException, HttpException, NetLibError) as v: self.flow.error = Error(repr(v)) if self.channel: self.channel.ask("error", self.flow) diff --git a/libmproxy/protocol/tls.py b/libmproxy/protocol/tls.py index 2935ca9f..cf303ca1 100644 --- a/libmproxy/protocol/tls.py +++ b/libmproxy/protocol/tls.py @@ -7,7 +7,7 @@ from construct import ConstructError import six from netlib.tcp import NetLibError, NetLibInvalidCertificateError -from netlib.http.http1 import HTTP1Protocol +from netlib.http import ALPN_PROTO_HTTP1 from ..contrib.tls._constructs import ClientHello from ..exceptions import ProtocolException, TlsException, ClientHandshakeException from .base import Layer @@ -367,8 +367,8 @@ class TlsLayer(Layer): """ # This gets triggered if we haven't established an upstream connection yet. - default_alpn = HTTP1Protocol.ALPN_PROTO_HTTP1 - # alpn_preference = netlib.http.http2.HTTP2Protocol.ALPN_PROTO_H2 + default_alpn = ALPN_PROTO_HTTP1 + # alpn_preference = ALPN_PROTO_H2 if self.alpn_for_client_connection in options: choice = bytes(self.alpn_for_client_connection) diff --git a/libmproxy/proxy/root_context.py b/libmproxy/proxy/root_context.py index 54bea1db..72243c59 100644 --- a/libmproxy/proxy/root_context.py +++ b/libmproxy/proxy/root_context.py @@ -5,8 +5,7 @@ import sys import six from libmproxy.exceptions import ProtocolException -from netlib.http.http1 import HTTP1Protocol -from netlib.http.http2 import HTTP2Protocol +from netlib.http import ALPN_PROTO_H2, ALPN_PROTO_HTTP1 from netlib.tcp import NetLibError from ..protocol import ( RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin @@ -85,9 +84,9 @@ class RootContext(object): # 5. Check for TLS ALPN (HTTP1/HTTP2) if isinstance(top_layer, TlsLayer): alpn = top_layer.client_conn.get_alpn_proto_negotiated() - if alpn == HTTP2Protocol.ALPN_PROTO_H2: + if alpn == ALPN_PROTO_H2: return Http2Layer(top_layer, 'transparent') - if alpn == HTTP1Protocol.ALPN_PROTO_HTTP1: + if alpn == ALPN_PROTO_HTTP1: return Http1Layer(top_layer, 'transparent') # 6. Check for raw tcp mode diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 88448172..8b286458 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -6,7 +6,7 @@ import socket import six from netlib import tcp -from netlib.http.http1 import HTTP1Protocol +from netlib.http.http1 import assemble_response from netlib.tcp import NetLibError from ..exceptions import ProtocolException, ServerException, ClientHandshakeException from ..protocol import Kill @@ -138,7 +138,7 @@ class ConnectionHandler(object): # understandable by HTTP clients and humans. try: error_response = make_error_response(502, repr(e)) - self.client_conn.send(HTTP1Protocol().assemble(error_response)) + self.client_conn.send(assemble_response(error_response)) except NetLibError: pass except Exception: diff --git a/libmproxy/web/app.py b/libmproxy/web/app.py index 2517e7ad..8eee6dce 100644 --- a/libmproxy/web/app.py +++ b/libmproxy/web/app.py @@ -128,12 +128,10 @@ class FlowHandler(RequestHandler): if a == "request": request = flow.request for k, v in b.iteritems(): - if k in ["method", "scheme", "host", "path"]: + if k in ["method", "scheme", "host", "path", "httpversion"]: setattr(request, k, str(v)) elif k == "port": request.port = int(v) - elif k == "httpversion": - request.httpversion = tuple(int(x) for x in v) elif k == "headers": request.headers.load_state(v) else: @@ -147,7 +145,7 @@ class FlowHandler(RequestHandler): elif k == "code": response.code = int(v) elif k == "httpversion": - response.httpversion = tuple(int(x) for x in v) + response.httpversion = str(v) elif k == "headers": response.headers.load_state(v) else: diff --git a/test/test_dump.py b/test/test_dump.py index 29931759..9d177ea0 100644 --- a/test/test_dump.py +++ b/test/test_dump.py @@ -4,7 +4,7 @@ from libmproxy.exceptions import ContentViewException from libmproxy.models import HTTPResponse import netlib.tutils -from netlib.http.semantics import CONTENT_MISSING +from netlib.http import CONTENT_MISSING from libmproxy import dump, flow from libmproxy.proxy import Log @@ -38,13 +38,13 @@ def test_strfuncs(): flow.request.stickycookie = True flow.client_conn = mock.MagicMock() flow.client_conn.address.host = "foo" - flow.response = netlib.tutils.tresp(content=CONTENT_MISSING) + flow.response = netlib.tutils.tresp(body=CONTENT_MISSING) flow.response.is_replay = True flow.response.code = 300 m.echo_flow(flow) - flow = tutils.tflow(resp=netlib.tutils.tresp("{")) + flow = tutils.tflow(resp=netlib.tutils.tresp(body="{")) flow.response.headers["content-type"] = "application/json" flow.response.code = 400 m.echo_flow(flow) @@ -62,14 +62,14 @@ def test_contentview(get_content_view): class TestDumpMaster: def _cycle(self, m, content): - f = tutils.tflow(req=netlib.tutils.treq(content)) + f = tutils.tflow(req=netlib.tutils.treq(body=content)) l = Log("connect") l.reply = mock.MagicMock() m.handle_log(l) m.handle_clientconnect(f.client_conn) m.handle_serverconnect(f.server_conn) m.handle_request(f) - f.response = HTTPResponse.wrap(netlib.tutils.tresp(content)) + f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=content)) f = m.handle_response(f) m.handle_clientdisconnect(f.client_conn) return f diff --git a/test/test_filt.py b/test/test_filt.py index 76e10710..104849be 100644 --- a/test/test_filt.py +++ b/test/test_filt.py @@ -84,7 +84,7 @@ class TestMatching: "host", 80, "/path", - (1, 1), + b"HTTP/1.1", headers, "content_request", None, @@ -99,8 +99,7 @@ class TestMatching: headers = Headers([["header_response", "svalue"]]) f.response = http.HTTPResponse( - (1, - 1), + b"HTTP/1.1", 200, "OK", headers, diff --git a/test/test_flow.py b/test/test_flow.py index c93beca4..f73187c1 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -8,7 +8,7 @@ import mock import netlib.utils from netlib import odict -from netlib.http.semantics import CONTENT_MISSING, HDR_FORM_URLENCODED, Headers +from netlib.http import CONTENT_MISSING, HDR_FORM_URLENCODED, Headers from libmproxy import filt, protocol, controller, tnetstring, flow from libmproxy.models import Error, Flow, HTTPRequest, HTTPResponse, HTTPFlow, decoded from libmproxy.proxy.config import HostMatcher @@ -849,7 +849,7 @@ class TestFlowMaster: s = flow.State() f = tutils.tflow() - f.response = HTTPResponse.wrap(netlib.tutils.tresp(f.request)) + f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=f.request)) pb = [f] fm = flow.FlowMaster(None, s) @@ -903,7 +903,7 @@ class TestFlowMaster: def test_server_playback_kill(self): s = flow.State() f = tutils.tflow() - f.response = HTTPResponse.wrap(netlib.tutils.tresp(f.request)) + f.response = HTTPResponse.wrap(netlib.tutils.tresp(body=f.request)) pb = [f] fm = flow.FlowMaster(None, s) fm.refresh_server_playback = True @@ -1043,7 +1043,7 @@ class TestRequest: def test_getset_form_urlencoded(self): d = odict.ODict([("one", "two"), ("three", "four")]) - r = HTTPRequest.wrap(netlib.tutils.treq(content=netlib.utils.urlencode(d.lst))) + r = HTTPRequest.wrap(netlib.tutils.treq(body=netlib.utils.urlencode(d.lst))) r.headers["content-type"] = HDR_FORM_URLENCODED assert r.get_form_urlencoded() == d diff --git a/test/test_fuzzing.py b/test/test_fuzzing.py index 482495f3..eff8c573 100644 --- a/test/test_fuzzing.py +++ b/test/test_fuzzing.py @@ -17,15 +17,11 @@ class TestFuzzy(tservers.HTTPProxTest): p = self.pathoc() assert p.request(req % self.server.port).status_code == 400 - def test_invalid_ports(self): - req = 'get:"http://localhost:999999"' - p = self.pathoc() - assert p.request(req).status_code == 400 - def test_invalid_ipv6_url(self): req = 'get:"http://localhost:%s":i13,"["' p = self.pathoc() - assert p.request(req % self.server.port).status_code == 400 + resp = p.request(req % self.server.port) + assert resp.status_code == 400 # def test_invalid_upstream(self): # req = r"get:'http://localhost:%s/p/200:i10,\x27+\x27'" diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index f53d43cf..5ddb5b5b 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -1,46 +1,36 @@ -import cStringIO -from cStringIO import StringIO +from io import BytesIO +from netlib.exceptions import HttpSyntaxException -from mock import MagicMock - -from libmproxy.protocol.http import * -import netlib.http from netlib.http import http1 -from netlib.http.semantics import CONTENT_MISSING - +from netlib.tutils import treq, raises import tutils import tservers -def mock_protocol(data=''): - rfile = cStringIO.StringIO(data) - wfile = cStringIO.StringIO() - return http1.HTTP1Protocol(rfile=rfile, wfile=wfile) - class TestHTTPResponse: def test_read_from_stringio(self): - s = "HTTP/1.1 200 OK\r\n" \ - "Content-Length: 7\r\n" \ - "\r\n"\ - "content\r\n" \ - "HTTP/1.1 204 OK\r\n" \ - "\r\n" - - protocol = mock_protocol(s) - r = HTTPResponse.from_protocol(protocol, "GET") + s = ( + b"HTTP/1.1 200 OK\r\n" + b"Content-Length: 7\r\n" + b"\r\n" + b"content\r\n" + b"HTTP/1.1 204 OK\r\n" + b"\r\n" + ) + rfile = BytesIO(s) + r = http1.read_response(rfile, treq()) assert r.status_code == 200 - assert r.content == "content" - assert HTTPResponse.from_protocol(protocol, "GET").status_code == 204 + assert r.content == b"content" + assert http1.read_response(rfile, treq()).status_code == 204 - protocol = mock_protocol(s) + rfile = BytesIO(s) # HEAD must not have content by spec. We should leave it on the pipe. - r = HTTPResponse.from_protocol(protocol, "HEAD") + r = http1.read_response(rfile, treq(method=b"HEAD")) assert r.status_code == 200 - assert r.content == "" - tutils.raises( - "Invalid server response: 'content", - HTTPResponse.from_protocol, protocol, "GET" - ) + assert r.content == b"" + + with raises(HttpSyntaxException): + http1.read_response(rfile, treq()) class TestHTTPFlow(object): diff --git a/test/test_proxy.py b/test/test_proxy.py index 3707fabe..76d8758c 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -9,6 +9,7 @@ from libmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler import tutils from libpathod import test from netlib import http, tcp +from netlib.http import http1 class TestServerConnection: @@ -26,11 +27,10 @@ class TestServerConnection: f.request.path = "/p/200:da" # use this protocol just to assemble - not for actual sending - protocol = http.http1.HTTP1Protocol(rfile=sc.rfile) - sc.send(protocol.assemble(f.request)) + sc.wfile.write(http1.assemble_request(f.request)) + sc.wfile.flush() - protocol = http.http1.HTTP1Protocol(rfile=sc.rfile) - assert protocol.read_response(f.request.method, 1000) + assert http1.read_response(sc.rfile, f.request, 1000) assert self.d.last_log() sc.finish() diff --git a/test/test_server.py b/test/test_server.py index 4a5dd7c2..0e338368 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -1,13 +1,14 @@ import socket import time from OpenSSL import SSL +from netlib.exceptions import HttpReadDisconnect, HttpException from netlib.tcp import Address import netlib.tutils from netlib import tcp, http, socks from netlib.certutils import SSLCert -from netlib.http import authentication -from netlib.http.semantics import CONTENT_MISSING +from netlib.http import authentication, CONTENT_MISSING, http1 +from netlib.tutils import raises from libpathod import pathoc, pathod from libmproxy.proxy.config import HostMatcher @@ -143,10 +144,9 @@ class TcpMixin: # mitmproxy responds with bad gateway assert self.pathod(spec).status_code == 502 self._ignore_on() - tutils.raises( - "invalid server response", - self.pathod, - spec) # pathoc tries to parse answer as HTTP + with raises(HttpException): + self.pathod(spec) # pathoc tries to parse answer as HTTP + self._ignore_off() def _tcpproxy_on(self): @@ -250,11 +250,6 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin): assert p.request(req % self.server2.urlbase) assert switched(self.proxy.log) - def test_get_connection_err(self): - p = self.pathoc() - ret = p.request("get:'http://localhost:0'") - assert ret.status_code == 502 - def test_blank_leading_line(self): p = self.pathoc() req = "get:'%s/p/201':i0,'\r\n'" @@ -262,8 +257,8 @@ class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin): def test_invalid_headers(self): p = self.pathoc() - req = p.request("get:'http://foo':h':foo'='bar'") - assert req.status_code == 400 + resp = p.request("get:'http://foo':h':foo'='bar'") + assert resp.status_code == 400 def test_empty_chunked_content(self): """ @@ -570,17 +565,23 @@ class TestProxy(tservers.HTTPProxTest): connection = socket.socket(socket.AF_INET, socket.SOCK_STREAM) connection.connect(("localhost", self.proxy.port)) connection.send( - "GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n" % + "GET http://localhost:%d/p/200:b@1k HTTP/1.1\r\n" % self.server.port) connection.send("\r\n") - connection.recv(5000) + # a bit hacky: make sure that we don't just read the headers only. + recvd = 0 + while recvd < 1024: + recvd += len(connection.recv(5000)) connection.send( - "GET http://localhost:%d/p/304:b@1k HTTP/1.1\r\n" % + "GET http://localhost:%d/p/200:b@1k HTTP/1.1\r\n" % self.server.port) connection.send("\r\n") - connection.recv(5000) + recvd = 0 + while recvd < 1024: + recvd += len(connection.recv(5000)) connection.close() + print(self.master.state.view._list) first_flow = self.master.state.view[0] second_flow = self.master.state.view[1] assert first_flow.server_conn.timestamp_tcp_setup @@ -718,15 +719,12 @@ class TestStreamRequest(tservers.HTTPProxTest): (self.server.urlbase, spec)) connection.send("\r\n") - protocol = http.http1.HTTP1Protocol(rfile=fconn) - resp = protocol.read_response("GET", None, include_body=False) + resp = http1.read_response_head(fconn) assert resp.headers["Transfer-Encoding"] == 'chunked' assert resp.status_code == 200 - chunks = list(protocol.read_http_body_chunked( - resp.headers, None, "GET", 200, False - )) + chunks = list(http1.read_body(fconn, None)) assert chunks == ["this", "isatest__reachhex"] connection.close() @@ -743,7 +741,7 @@ class TestFakeResponse(tservers.HTTPProxTest): def test_fake(self): f = self.pathod("200") - assert "header_response" in f.headers + assert "header-response" in f.headers class TestServerConnect(tservers.HTTPProxTest): @@ -766,7 +764,8 @@ class TestKillRequest(tservers.HTTPProxTest): masterclass = MasterKillRequest def test_kill(self): - tutils.raises("server disconnect", self.pathod, "200") + with raises(HttpReadDisconnect): + self.pathod("200") # Nothing should have hit the server assert not self.server.last_log() @@ -780,7 +779,8 @@ class TestKillResponse(tservers.HTTPProxTest): masterclass = MasterKillResponse def test_kill(self): - tutils.raises("server disconnect", self.pathod, "200") + with raises(HttpReadDisconnect): + self.pathod("200") # The server should have seen a request assert self.server.last_log() @@ -907,7 +907,7 @@ class TestUpstreamProxySSL( """ def handle_request(f): - f.request.httpversion = (1, 0) + f.request.httpversion = b"HTTP/1.1" del f.request.headers["Content-Length"] f.reply() diff --git a/test/tutils.py b/test/tutils.py index d64388f3..229b51a8 100644 --- a/test/tutils.py +++ b/test/tutils.py @@ -102,7 +102,7 @@ def tflowview(request_contents=None): if request_contents is None: flow = tflow() else: - flow = tflow(req=netlib.tutils.treq(request_contents)) + flow = tflow(req=netlib.tutils.treq(body=request_contents)) fv = FlowView(m, cs, flow) return fv diff --git a/web/src/js/components/flowview/messages.js b/web/src/js/components/flowview/messages.js index fa75efbe..8879e753 100644 --- a/web/src/js/components/flowview/messages.js +++ b/web/src/js/components/flowview/messages.js @@ -121,7 +121,7 @@ var RequestLine = React.createClass({ render: function () { var flow = this.props.flow; var url = flowutils.RequestUtils.pretty_url(flow.request); - var httpver = "HTTP/" + flow.request.httpversion.join("."); + var httpver = flow.request.httpversion; return <div className="first-line request-line"> <ValueEditor @@ -175,7 +175,7 @@ var RequestLine = React.createClass({ var ResponseLine = React.createClass({ render: function () { var flow = this.props.flow; - var httpver = "HTTP/" + flow.response.httpversion.join("."); + var httpver = flow.response.httpversion; return <div className="first-line response-line"> <ValueEditor ref="httpVersion" @@ -323,4 +323,4 @@ module.exports = { Request: Request, Response: Response, Error: Error -};
\ No newline at end of file +}; |