diff options
| -rw-r--r-- | libmproxy/console/common.py | 2 | ||||
| -rw-r--r-- | libmproxy/console/flowview.py | 2 | ||||
| -rw-r--r-- | libmproxy/filt.py | 2 | ||||
| -rw-r--r-- | libmproxy/flow.py | 23 | ||||
| -rw-r--r-- | libmproxy/models/__init__.py | 16 | ||||
| -rw-r--r-- | libmproxy/models/connections.py (renamed from libmproxy/proxy/connection.py) | 1 | ||||
| -rw-r--r-- | libmproxy/models/flow.py (renamed from libmproxy/protocol/primitives.py) | 8 | ||||
| -rw-r--r-- | libmproxy/models/http.py (renamed from libmproxy/protocol/http_wrappers.py) | 211 | ||||
| -rw-r--r-- | libmproxy/protocol/__init__.py | 13 | ||||
| -rw-r--r-- | libmproxy/protocol/base.py (renamed from libmproxy/protocol2/layer.py) | 28 | ||||
| -rw-r--r-- | libmproxy/protocol/http.py | 602 | ||||
| -rw-r--r-- | libmproxy/protocol/http_replay.py (renamed from libmproxy/protocol2/http_replay.py) | 16 | ||||
| -rw-r--r-- | libmproxy/protocol/rawtcp.py (renamed from libmproxy/protocol2/rawtcp.py) | 6 | ||||
| -rw-r--r-- | libmproxy/protocol/tls.py (renamed from libmproxy/protocol2/tls.py) | 2 | ||||
| -rw-r--r-- | libmproxy/protocol2/http.py | 588 | ||||
| -rw-r--r-- | libmproxy/proxy/__init__.py | 8 | ||||
| -rw-r--r-- | libmproxy/proxy/config.py | 2 | ||||
| -rw-r--r-- | libmproxy/proxy/modes/__init__.py (renamed from libmproxy/protocol2/__init__.py) | 13 | ||||
| -rw-r--r-- | libmproxy/proxy/modes/http_proxy.py (renamed from libmproxy/protocol2/http_proxy.py) | 2 | ||||
| -rw-r--r-- | libmproxy/proxy/modes/reverse_proxy.py (renamed from libmproxy/protocol2/reverse_proxy.py) | 2 | ||||
| -rw-r--r-- | libmproxy/proxy/modes/socks_proxy.py (renamed from libmproxy/protocol2/socks_proxy.py) | 5 | ||||
| -rw-r--r-- | libmproxy/proxy/modes/transparent_proxy.py (renamed from libmproxy/protocol2/transparent_proxy.py) | 6 | ||||
| -rw-r--r-- | libmproxy/proxy/primitives.py | 15 | ||||
| -rw-r--r-- | libmproxy/proxy/root_context.py (renamed from libmproxy/protocol2/root_context.py) | 14 | ||||
| -rw-r--r-- | libmproxy/proxy/server.py | 23 | ||||
| -rw-r--r-- | test/test_dump.py | 10 | ||||
| -rw-r--r-- | test/test_filt.py | 2 | ||||
| -rw-r--r-- | test/test_flow.py | 83 | ||||
| -rw-r--r-- | test/test_proxy.py | 8 | ||||
| -rw-r--r-- | test/test_server.py | 14 | ||||
| -rw-r--r-- | test/tutils.py | 23 |
31 files changed, 884 insertions, 866 deletions
diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 1940e390..c25f7267 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -8,7 +8,7 @@ from netlib.http.semantics import CONTENT_MISSING import netlib.utils from .. import utils -from ..protocol.http import decoded +from ..models import decoded from . import signals diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 1e0f0c17..8b828653 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -9,7 +9,7 @@ from netlib.http.semantics import CONTENT_MISSING from . import common, grideditor, contentview, signals, searchable, tabs from . import flowdetailview from .. import utils, controller -from ..protocol.http import HTTPRequest, HTTPResponse, decoded +from ..models import HTTPRequest, HTTPResponse, decoded class SearchError(Exception): diff --git a/libmproxy/filt.py b/libmproxy/filt.py index 25747bc6..cfd3a1bc 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -35,7 +35,7 @@ from __future__ import absolute_import import re import sys import pyparsing as pp -from .protocol.http import decoded +from .models import decoded class _Token: diff --git a/libmproxy/flow.py b/libmproxy/flow.py index a2f57512..00ec83d2 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -8,19 +8,18 @@ import Cookie import cookielib import os import re -from libmproxy.protocol.http import HTTPFlow -from libmproxy.protocol2.http_replay import RequestReplayThread +import urlparse + from netlib import odict, wsgi from netlib.http.semantics import CONTENT_MISSING import netlib.http - -from . import controller, protocol, tnetstring, filt, script, version +from . import controller, tnetstring, filt, script, version from .onboarding import app -from .protocol import http from .proxy.config import HostMatcher -from .proxy.connection import ClientConnection, ServerConnection -import urlparse +from .protocol.http_replay import RequestReplayThread +from .protocol import Kill +from .models import ClientConnection, ServerConnection, HTTPResponse, HTTPFlow, HTTPRequest class AppRegistry: @@ -790,7 +789,7 @@ class FlowMaster(controller.Master): rflow = self.server_playback.next_flow(flow) if not rflow: return None - response = http.HTTPResponse.from_state(rflow.response.get_state()) + response = HTTPResponse.from_state(rflow.response.get_state()) response.is_replay = True if self.refresh_server_playback: response.refresh() @@ -836,10 +835,10 @@ class FlowMaster(controller.Master): sni=host, ssl_established=True )) - f = http.HTTPFlow(c, s) + f = HTTPFlow(c, s) headers = odict.ODictCaseless() - req = http.HTTPRequest( + req = HTTPRequest( "absolute", method, scheme, @@ -981,7 +980,7 @@ class FlowMaster(controller.Master): ) if err: self.add_event("Error in wsgi app. %s" % err, "error") - f.reply(protocol.KILL) + f.reply(Kill) return if f not in self.state.flows: # don't add again on replay self.state.add_flow(f) @@ -998,7 +997,7 @@ class FlowMaster(controller.Master): if self.stream_large_bodies: self.stream_large_bodies.run(f, False) except netlib.http.HttpError: - f.reply(protocol.KILL) + f.reply(Kill) return f.reply() diff --git a/libmproxy/models/__init__.py b/libmproxy/models/__init__.py new file mode 100644 index 00000000..3947847c --- /dev/null +++ b/libmproxy/models/__init__.py @@ -0,0 +1,16 @@ +from __future__ import (absolute_import, print_function, division) + +from .http import ( + HTTPFlow, HTTPRequest, HTTPResponse, decoded, + make_error_response, make_connect_request, make_connect_response +) +from .connections import ClientConnection, ServerConnection +from .flow import Flow, Error + +__all__ = [ + "HTTPFlow", "HTTPRequest", "HTTPResponse", "decoded" + "make_error_response", "make_connect_request", + "make_connect_response", + "ClientConnection", "ServerConnection", + "Flow", "Error", +] diff --git a/libmproxy/proxy/connection.py b/libmproxy/models/connections.py index 94f318f6..98bae3cc 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/models/connections.py @@ -190,4 +190,5 @@ class ServerConnection(tcp.TCPClient, stateobject.StateObject): tcp.TCPClient.finish(self) self.timestamp_end = utils.timestamp() + ServerConnection._stateobject_attributes["via"] = ServerConnection diff --git a/libmproxy/protocol/primitives.py b/libmproxy/models/flow.py index c663f0c5..58287e5b 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/models/flow.py @@ -3,9 +3,7 @@ import copy import uuid from .. import stateobject, utils, version -from ..proxy.connection import ClientConnection, ServerConnection - -KILL = 0 # const for killed requests +from .connections import ClientConnection, ServerConnection class Error(stateobject.StateObject): @@ -140,9 +138,11 @@ class Flow(stateobject.StateObject): """ Kill this request. """ + from ..protocol import Kill + self.error = Error("Connection killed") self.intercepted = False - self.reply(KILL) + self.reply(Kill) master.handle_error(self) def intercept(self, master): diff --git a/libmproxy/protocol/http_wrappers.py b/libmproxy/models/http.py index a26ddbb4..fb2f305b 100644 --- a/libmproxy/protocol/http_wrappers.py +++ b/libmproxy/models/http.py @@ -1,39 +1,16 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) import Cookie import copy -import time from email.utils import parsedate_tz, formatdate, mktime_tz +import time +from libmproxy import utils from netlib import odict, encoding -from netlib.http import semantics, CONTENT_MISSING -from .. import utils, stateobject - - -class decoded(object): - """ - A context manager that decodes a request or response, and then - re-encodes it with the same encoding after execution of the block. - - Example: - with decoded(request): - request.content = request.content.replace("foo", "bar") - """ - - def __init__(self, o): - self.o = o - ce = o.headers.get_first("content-encoding") - if ce in encoding.ENCODINGS: - self.ce = ce - else: - self.ce = None - - def __enter__(self): - if self.ce: - self.o.decode() - - def __exit__(self, type, value, tb): - if self.ce: - self.o.encode(self.ce) +from netlib.http import status_codes +from netlib.tcp import Address +from netlib.http.semantics import Request, Response, CONTENT_MISSING +from .. import version, stateobject +from .flow import Flow class MessageMixin(stateobject.StateObject): @@ -116,7 +93,7 @@ class MessageMixin(stateobject.StateObject): return c -class HTTPRequest(MessageMixin, semantics.Request): +class HTTPRequest(MessageMixin, Request): """ An HTTP request. @@ -176,7 +153,7 @@ class HTTPRequest(MessageMixin, semantics.Request): timestamp_end=None, form_out=None, ): - semantics.Request.__init__( + Request.__init__( self, form_in, method, @@ -277,7 +254,7 @@ class HTTPRequest(MessageMixin, semantics.Request): return c -class HTTPResponse(MessageMixin, semantics.Response): +class HTTPResponse(MessageMixin, Response): """ An HTTP response. @@ -310,7 +287,7 @@ class HTTPResponse(MessageMixin, semantics.Response): timestamp_start=None, timestamp_end=None, ): - semantics.Response.__init__( + Response.__init__( self, httpversion, status_code, @@ -411,3 +388,167 @@ class HTTPResponse(MessageMixin, semantics.Response): c.append(self._refresh_cookie(i, delta)) if c: self.headers["set-cookie"] = c + + +class HTTPFlow(Flow): + """ + A HTTPFlow is a collection of objects representing a single HTTP + transaction. The main attributes are: + + request: HTTPRequest object + response: HTTPResponse object + error: Error object + server_conn: ServerConnection object + client_conn: ClientConnection object + + Note that it's possible for a Flow to have both a response and an error + object. This might happen, for instance, when a response was received + from the server, but there was an error sending it back to the client. + + The following additional attributes are exposed: + + intercepted: Is this flow currently being intercepted? + live: Does this flow have a live client connection? + """ + + def __init__(self, client_conn, server_conn, live=None): + super(HTTPFlow, self).__init__("http", client_conn, server_conn, live) + self.request = None + """@type: HTTPRequest""" + self.response = None + """@type: HTTPResponse""" + + _stateobject_attributes = Flow._stateobject_attributes.copy() + _stateobject_attributes.update( + request=HTTPRequest, + response=HTTPResponse + ) + + @classmethod + def from_state(cls, state): + f = cls(None, None) + f.load_state(state) + return f + + def __repr__(self): + s = "<HTTPFlow" + for a in ("request", "response", "error", "client_conn", "server_conn"): + if getattr(self, a, False): + s += "\r\n %s = {flow.%s}" % (a, a) + s += ">" + return s.format(flow=self) + + def copy(self): + f = super(HTTPFlow, self).copy() + if self.request: + f.request = self.request.copy() + if self.response: + f.response = self.response.copy() + return f + + def match(self, f): + """ + Match this flow against a compiled filter expression. Returns True + if matched, False if not. + + If f is a string, it will be compiled as a filter expression. If + the expression is invalid, ValueError is raised. + """ + if isinstance(f, basestring): + from .. import filt + + f = filt.parse(f) + if not f: + raise ValueError("Invalid filter expression.") + if f: + return f(self) + return True + + def replace(self, pattern, repl, *args, **kwargs): + """ + Replaces a regular expression pattern with repl in both request and + response of the flow. Encoded content will be decoded before + replacement, and re-encoded afterwards. + + Returns the number of replacements made. + """ + c = self.request.replace(pattern, repl, *args, **kwargs) + if self.response: + c += self.response.replace(pattern, repl, *args, **kwargs) + return c + + +class decoded(object): + """ + A context manager that decodes a request or response, and then + re-encodes it with the same encoding after execution of the block. + + Example: + with decoded(request): + request.content = request.content.replace("foo", "bar") + """ + + def __init__(self, o): + self.o = o + ce = o.headers.get_first("content-encoding") + if ce in encoding.ENCODINGS: + self.ce = ce + else: + self.ce = None + + def __enter__(self): + if self.ce: + self.o.decode() + + def __exit__(self, type, value, tb): + if self.ce: + self.o.encode(self.ce) + + +def make_error_response(status_code, message, headers=None): + response = status_codes.RESPONSES.get(status_code, "Unknown") + body = """ + <html> + <head> + <title>%d %s</title> + </head> + <body>%s</body> + </html> + """.strip() % (status_code, response, message) + + if not headers: + headers = odict.ODictCaseless() + headers["Server"] = [version.NAMEVERSION] + headers["Connection"] = ["close"] + headers["Content-Length"] = [len(body)] + headers["Content-Type"] = ["text/html"] + + return HTTPResponse( + (1, 1), # FIXME: Should be a string. + status_code, + response, + headers, + body, + ) + + +def make_connect_request(address): + address = Address.wrap(address) + return HTTPRequest( + "authority", "CONNECT", None, address.host, address.port, None, (1, 1), + odict.ODictCaseless(), "" + ) + + +def make_connect_response(httpversion): + headers = odict.ODictCaseless([ + ["Content-Length", "0"], + ["Proxy-Agent", version.NAMEVERSION] + ]) + return HTTPResponse( + httpversion, + 200, + "Connection established", + headers, + "", + ) diff --git a/libmproxy/protocol/__init__.py b/libmproxy/protocol/__init__.py index bbc20dba..c582592b 100644 --- a/libmproxy/protocol/__init__.py +++ b/libmproxy/protocol/__init__.py @@ -1 +1,12 @@ -from .primitives import * +from __future__ import (absolute_import, print_function, division) +from .base import Layer, ServerConnectionMixin, Log, Kill +from .http import Http1Layer, Http2Layer +from .tls import TlsLayer, is_tls_record_magic +from .rawtcp import RawTCPLayer + +__all__ = [ + "Layer", "ServerConnectionMixin", "Log", "Kill", + "Http1Layer", "Http2Layer", + "TlsLayer", "is_tls_record_magic", + "RawTCPLayer" +] diff --git a/libmproxy/protocol2/layer.py b/libmproxy/protocol/base.py index 2b47cc26..d22a71c6 100644 --- a/libmproxy/protocol2/layer.py +++ b/libmproxy/protocol/base.py @@ -2,22 +2,25 @@ mitmproxy protocol architecture In mitmproxy, protocols are implemented as a set of layers, which are composed on top each other. -For example, the following scenarios depict possible scenarios (lowest layer first): +For example, the following scenarios depict possible settings (lowest layer first): Transparent HTTP proxy, no SSL: - TransparentModeLayer + TransparentProxy + Http1Layer HttpLayer Regular proxy, CONNECT request with WebSockets over SSL: - RegularModeLayer + HttpProxy + Http1Layer HttpLayer SslLayer WebsocketLayer (or TcpLayer) Automated protocol detection by peeking into the buffer: - TransparentModeLayer - SslLayer + TransparentProxy + TLSLayer Http2Layer + HttpLayer Communication between layers is done as follows: - lower layers provide context information to higher layers @@ -31,8 +34,7 @@ Further goals: """ from __future__ import (absolute_import, print_function, division) from netlib import tcp -from ..proxy import Log -from ..proxy.connection import ServerConnection +from ..models import ServerConnection from ..exceptions import ProtocolException @@ -136,3 +138,15 @@ class ServerConnectionMixin(object): except tcp.NetLibError as e: raise ProtocolException( "Server connection to '%s' failed: %s" % (self.server_conn.address, e), e) + + +class Log(object): + def __init__(self, msg, level="info"): + self.msg = msg + self.level = level + + +class Kill(Exception): + """ + Kill a connection. + """ diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index bde7b088..fc57f6df 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -1,92 +1,538 @@ -from __future__ import absolute_import +from __future__ import (absolute_import, print_function, division) -from .primitives import Flow +from netlib import tcp +from netlib.http import http1, HttpErrorConnClosed, HttpError +from netlib.http.semantics import CONTENT_MISSING +from netlib import odict +from netlib.tcp import NetLibError, Address +from netlib.http.http1 import HTTP1Protocol +from netlib.http.http2 import HTTP2Protocol -from .http_wrappers import decoded, HTTPRequest, HTTPResponse +from .. import utils +from ..exceptions import InvalidCredentials, HttpException, ProtocolException +from ..models import ( + HTTPFlow, HTTPRequest, HTTPResponse, make_error_response, make_connect_response, Error +) +from .base import Layer, Kill -class HTTPFlow(Flow): - """ - A HTTPFlow is a collection of objects representing a single HTTP - transaction. The main attributes are: - request: HTTPRequest object - response: HTTPResponse object - error: Error object - server_conn: ServerConnection object - client_conn: ClientConnection object +class _HttpLayer(Layer): + supports_streaming = False + + def read_request(self): + raise NotImplementedError() + + def send_request(self, request): + raise NotImplementedError() + + def read_response(self, request_method): + raise NotImplementedError() + + def send_response(self, response): + raise NotImplementedError() + + +class _StreamingHttpLayer(_HttpLayer): + supports_streaming = True + + def read_response_headers(self): + raise NotImplementedError + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + raise NotImplementedError() + yield "this is a generator" + + def send_response_headers(self, response): + raise NotImplementedError + + def send_response_body(self, response, chunks): + raise NotImplementedError() + + +class Http1Layer(_StreamingHttpLayer): + def __init__(self, ctx, mode): + super(Http1Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP1Protocol(self.client_conn) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def read_request(self): + return HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + + def send_request(self, request): + self.server_conn.send(self.server_protocol.assemble(request)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True + ) + + def send_response(self, response): + self.client_conn.send(self.client_protocol.assemble(response)) + + def read_response_headers(self): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=None, # does not matter if we don't read the body. + body_size_limit=self.config.body_size_limit, + include_body=False + ) + + def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): + return self.server_protocol.read_http_body_chunked( + headers, + self.config.body_size_limit, + request_method, + response_code, + False, + max_chunk_size + ) + + def send_response_headers(self, response): + h = self.client_protocol._assemble_response_first_line(response) + self.client_conn.wfile.write(h + "\r\n") + h = self.client_protocol._assemble_response_headers( + response, + preserve_transfer_encoding=True + ) + self.client_conn.send(h + "\r\n") + + def send_response_body(self, response, chunks): + if self.client_protocol.has_chunked_encoding(response.headers): + chunks = ( + "%d\r\n%s\r\n" % (len(chunk), chunk) + for chunk in chunks + ) + for chunk in chunks: + self.client_conn.send(chunk) - Note that it's possible for a Flow to have both a response and an error - object. This might happen, for instance, when a response was received - from the server, but there was an error sending it back to the client. + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP1Protocol(self.server_conn) - The following additional attributes are exposed: + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP1Protocol(self.server_conn) - intercepted: Is this flow currently being intercepted? - live: Does this flow have a live client connection? + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP1Protocol(self.server_conn) + + def __call__(self): + layer = HttpLayer(self, self.mode) + layer() + + +# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. +class Http2Layer(_HttpLayer): + def __init__(self, ctx, mode): + super(Http2Layer, self).__init__(ctx) + self.mode = mode + self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + + def read_request(self): + request = HTTPRequest.from_protocol( + self.client_protocol, + body_size_limit=self.config.body_size_limit + ) + self._stream_id = request.stream_id + return request + + def send_request(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.server_conn.send(self.server_protocol.assemble(message)) + + def read_response(self, request_method): + return HTTPResponse.from_protocol( + self.server_protocol, + request_method=request_method, + body_size_limit=self.config.body_size_limit, + include_body=True, + stream_id=self._stream_id + ) + + def send_response(self, message): + # TODO: implement flow control and WINDOW_UPDATE frames + self.client_conn.send(self.client_protocol.assemble(message)) + + def connect(self): + self.ctx.connect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def reconnect(self): + self.ctx.reconnect() + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def set_server(self, *args, **kwargs): + self.ctx.set_server(*args, **kwargs) + self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, + unhandled_frame_cb=self.handle_unexpected_frame) + self.server_protocol.perform_connection_preface() + + def __call__(self): + self.server_protocol.perform_connection_preface() + layer = HttpLayer(self, self.mode) + layer() + + def handle_unexpected_frame(self, frm): + self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") + + +class ConnectServerConnection(object): + """ + "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. """ - def __init__(self, client_conn, server_conn, live=None): - super(HTTPFlow, self).__init__("http", client_conn, server_conn, live) - self.request = None - """@type: HTTPRequest""" - self.response = None - """@type: HTTPResponse""" - - _stateobject_attributes = Flow._stateobject_attributes.copy() - _stateobject_attributes.update( - request=HTTPRequest, - response=HTTPResponse - ) - - @classmethod - def from_state(cls, state): - f = cls(None, None) - f.load_state(state) - return f - - def __repr__(self): - s = "<HTTPFlow" - for a in ("request", "response", "error", "client_conn", "server_conn"): - if getattr(self, a, False): - s += "\r\n %s = {flow.%s}" % (a, a) - s += ">" - return s.format(flow=self) - - def copy(self): - f = super(HTTPFlow, self).copy() - if self.request: - f.request = self.request.copy() - if self.response: - f.response = self.response.copy() - return f - - def match(self, f): - """ - Match this flow against a compiled filter expression. Returns True - if matched, False if not. + def __init__(self, address, ctx): + self.address = tcp.Address.wrap(address) + self._ctx = ctx - If f is a string, it will be compiled as a filter expression. If - the expression is invalid, ValueError is raised. - """ - if isinstance(f, basestring): - from .. import filt + @property + def via(self): + return self._ctx.server_conn - f = filt.parse(f) - if not f: - raise ValueError("Invalid filter expression.") - if f: - return f(self) - return True + def __getattr__(self, item): + return getattr(self.via, item) - def replace(self, pattern, repl, *args, **kwargs): - """ - Replaces a regular expression pattern with repl in both request and - response of the flow. Encoded content will be decoded before - replacement, and re-encoded afterwards. - Returns the number of replacements made. +class UpstreamConnectLayer(Layer): + def __init__(self, ctx, connect_request): + super(UpstreamConnectLayer, self).__init__(ctx) + self.connect_request = connect_request + self.server_conn = ConnectServerConnection( + (connect_request.host, connect_request.port), + self.ctx + ) + + def __call__(self): + layer = self.ctx.next_layer(self) + layer() + + def connect(self): + if not self.server_conn: + self.ctx.connect() + self.send_request(self.connect_request) + else: + pass # swallow the message + + def reconnect(self): + self.ctx.reconnect() + self.send_request(self.connect_request) + resp = self.read_response("CONNECT") + if resp.code != 200: + raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") + + def set_server(self, address, server_tls=None, sni=None, depth=1): + if depth == 1: + if self.ctx.server_conn: + self.ctx.reconnect() + address = Address.wrap(address) + self.connect_request.host = address.host + self.connect_request.port = address.port + self.server_conn.address = address + else: + self.ctx.set_server(address, server_tls, sni, depth - 1) + + +class HttpLayer(Layer): + def __init__(self, ctx, mode): + super(HttpLayer, self).__init__(ctx) + self.mode = mode + self.__original_server_conn = None + "Contains the original destination in transparent mode, which needs to be restored" + "if an inline script modified the target server for a single http request" + + def __call__(self): + if self.mode == "transparent": + self.__original_server_conn = self.server_conn + while True: + try: + flow = HTTPFlow(self.client_conn, self.server_conn, live=self) + + try: + request = self.read_request() + except tcp.NetLibError: + # don't throw an error for disconnects that happen + # before/between requests. + return + + self.log("request", "debug", [repr(request)]) + + # Handle Proxy Authentication + self.authenticate(request) + + # Regular Proxy Mode: Handle CONNECT + if self.mode == "regular" and request.form_in == "authority": + self.handle_regular_mode_connect(request) + return + + # Make sure that the incoming request matches our expectations + self.validate_request(request) + + flow.request = request + self.process_request_hook(flow) + + if not flow.response: + self.establish_server_connection(flow) + self.get_response_from_server(flow) + + self.send_response_to_client(flow) + + if self.check_close_connection(flow): + return + + # TODO: Implement HTTP Upgrade + + # Upstream Proxy Mode: Handle CONNECT + if flow.request.form_in == "authority" and flow.response.code == 200: + self.handle_upstream_mode_connect(flow.request.copy()) + return + + except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: + if flow.request and not flow.response: + flow.error = Error(repr(e)) + self.channel.ask("error", flow) + try: + self.send_response(make_error_response( + getattr(e, "code", 502), + repr(e) + )) + except NetLibError: + pass + if isinstance(e, ProtocolException): + raise e + else: + raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) + finally: + flow.live = False + + def handle_regular_mode_connect(self, request): + self.set_server((request.host, request.port)) + self.send_response(make_connect_response(request.httpversion)) + layer = self.ctx.next_layer(self) + layer() + + def handle_upstream_mode_connect(self, connect_request): + layer = UpstreamConnectLayer(self, connect_request) + layer() + + def check_close_connection(self, flow): """ - c = self.request.replace(pattern, repl, *args, **kwargs) - if self.response: - c += self.response.replace(pattern, repl, *args, **kwargs) - return c + Checks if the connection should be closed depending on the HTTP + semantics. Returns True, if so. + """ + + # TODO: add logic for HTTP/2 + + close_connection = ( + http1.HTTP1Protocol.connection_close( + flow.request.httpversion, + flow.request.headers + ) or http1.HTTP1Protocol.connection_close( + flow.response.httpversion, + flow.response.headers + ) or http1.HTTP1Protocol.expected_http_body_size( + flow.response.headers, + False, + flow.request.method, + flow.response.code) == -1 + ) + if flow.request.form_in == "authority" and flow.response.code == 200: + # Workaround for + # https://github.com/mitmproxy/mitmproxy/issues/313: Some + # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 + # and no Content-Length header + + return False + return close_connection + + def send_response_to_client(self, flow): + if not (self.supports_streaming and flow.response.stream): + # no streaming: + # we already received the full response from the server and can + # send it to the client straight away. + self.send_response(flow.response) + else: + # streaming: + # First send the headers and then transfer the response incrementally + self.send_response_headers(flow.response) + chunks = self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code, + max_chunk_size=4096 + ) + if callable(flow.response.stream): + chunks = flow.response.stream(chunks) + self.send_response_body(flow.response, chunks) + flow.response.timestamp_end = utils.timestamp() + + def get_response_from_server(self, flow): + def get_response(): + self.send_request(flow.request) + if self.supports_streaming: + flow.response = self.read_response_headers() + else: + flow.response = self.read_response() + + try: + get_response() + except (tcp.NetLibError, HttpErrorConnClosed) as v: + self.log( + "server communication error: %s" % repr(v), + level="debug" + ) + # In any case, we try to reconnect at least once. This is + # necessary because it might be possible that we already + # initiated an upstream connection after clientconnect that + # has already been expired, e.g consider the following event + # log: + # > clientconnect (transparent mode destination known) + # > serverconnect (required for client tls handshake) + # > read n% of large request + # > server detects timeout, disconnects + # > read (100-n)% of large request + # > send large request upstream + self.reconnect() + get_response() + + # call the appropriate script hook - this is an opportunity for an + # inline script to set flow.stream = True + flow = self.channel.ask("responseheaders", flow) + if flow is None or flow == Kill: + raise Kill() + + if self.supports_streaming: + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = "".join(self.read_response_body( + flow.response.headers, + flow.request.method, + flow.response.code + )) + flow.response.timestamp_end = utils.timestamp() + + # no further manipulation of self.server_conn beyond this point + # we can safely set it as the final attribute value here. + flow.server_conn = self.server_conn + + self.log( + "response", + "debug", + [repr(flow.response)] + ) + response_reply = self.channel.ask("response", flow) + if response_reply is None or response_reply == Kill: + raise Kill() + + def process_request_hook(self, flow): + # Determine .scheme, .host and .port attributes for inline scripts. + # For absolute-form requests, they are directly given in the request. + # For authority-form requests, we only need to determine the request scheme. + # For relative-form requests, we need to determine host and port as + # well. + if self.mode == "regular": + pass # only absolute-form at this point, nothing to do here. + elif self.mode == "upstream": + if flow.request.form_in == "authority": + flow.request.scheme = "http" # pseudo value + else: + flow.request.host = self.__original_server_conn.address.host + flow.request.port = self.__original_server_conn.address.port + flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" + + request_reply = self.channel.ask("request", flow) + if request_reply is None or request_reply == Kill: + raise Kill() + if isinstance(request_reply, HTTPResponse): + flow.response = request_reply + return + + def establish_server_connection(self, flow): + address = tcp.Address((flow.request.host, flow.request.port)) + tls = (flow.request.scheme == "https") + + if self.mode == "regular" or self.mode == "transparent": + # If there's an existing connection that doesn't match our expectations, kill it. + if address != self.server_conn.address or tls != self.server_conn.ssl_established: + self.set_server(address, tls, address.host) + # Establish connection is neccessary. + if not self.server_conn: + self.connect() + + # SetServer is not guaranteed to work with TLS: + # If there's not TlsLayer below which could catch the exception, + # TLS will not be established. + if tls and not self.server_conn.tls_established: + raise ProtocolException( + "Cannot upgrade to SSL, no TLS layer on the protocol stack.") + else: + if not self.server_conn: + self.connect() + if tls: + raise HttpException("Cannot change scheme in upstream proxy mode.") + """ + # This is a very ugly (untested) workaround to solve a very ugly problem. + if self.server_conn and self.server_conn.tls_established and not ssl: + self.reconnect() + elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: + if self.server_conn.tls_established: + self.reconnect() + + self.send_request(make_connect_request(address)) + tls_layer = TlsLayer(self, False, True) + tls_layer._establish_tls_with_server() + """ + + def validate_request(self, request): + if request.form_in == "absolute" and request.scheme != "http": + self.send_response( + make_error_response(400, "Invalid request scheme: %s" % request.scheme)) + raise HttpException("Invalid request scheme: %s" % request.scheme) + + expected_request_forms = { + "regular": ("absolute",), # an authority request would already be handled. + "upstream": ("authority", "absolute"), + "transparent": ("relative",) + } + + allowed_request_forms = expected_request_forms[self.mode] + if request.form_in not in allowed_request_forms: + err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( + " or ".join(allowed_request_forms), request.form_in + ) + self.send_response(make_error_response(400, err_message)) + raise HttpException(err_message) + + if self.mode == "regular": + request.form_out = "relative" + + def authenticate(self, request): + if self.config.authenticator: + if self.config.authenticator.authenticate(request.headers): + self.config.authenticator.clean(request.headers) + else: + self.send_response(make_error_response( + 407, + "Proxy Authentication Required", + odict.ODictCaseless( + [ + [k, v] for k, v in + self.config.authenticator.auth_challenge_headers().items() + ]) + )) + raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/protocol2/http_replay.py b/libmproxy/protocol/http_replay.py index 872ef9cd..e0144c93 100644 --- a/libmproxy/protocol2/http_replay.py +++ b/libmproxy/protocol/http_replay.py @@ -1,14 +1,14 @@ import threading + from netlib.http import HttpError from netlib.http.http1 import HTTP1Protocol from netlib.tcp import NetLibError - from ..controller import Channel -from ..protocol import KILL, Error -from ..protocol.http_wrappers import HTTPResponse -from ..proxy import Log, Kill -from ..proxy.connection import ServerConnection -from .http import make_connect_request +from ..models import Error, HTTPResponse, ServerConnection, make_connect_request +from .base import Log, Kill + + +# TODO: Doesn't really belong into libmproxy.protocol... class RequestReplayThread(threading.Thread): @@ -35,7 +35,7 @@ class RequestReplayThread(threading.Thread): # If we have a channel, run script hooks. if self.channel: request_reply = self.channel.ask("request", self.flow) - if request_reply is None or request_reply == KILL: + if request_reply is None or request_reply == Kill: raise Kill() elif isinstance(request_reply, HTTPResponse): self.flow.response = request_reply @@ -81,7 +81,7 @@ class RequestReplayThread(threading.Thread): ) if self.channel: response_reply = self.channel.ask("response", self.flow) - if response_reply is None or response_reply == KILL: + if response_reply is None or response_reply == Kill: raise Kill() except (HttpError, NetLibError) as v: self.flow.error = Error(repr(v)) diff --git a/libmproxy/protocol2/rawtcp.py b/libmproxy/protocol/rawtcp.py index b10217f1..86468773 100644 --- a/libmproxy/protocol2/rawtcp.py +++ b/libmproxy/protocol/rawtcp.py @@ -7,15 +7,15 @@ from OpenSSL import SSL from netlib.tcp import NetLibError from netlib.utils import cleanBin from ..exceptions import ProtocolException -from .layer import Layer +from .base import Layer -class RawTcpLayer(Layer): +class RawTCPLayer(Layer): chunk_size = 4096 def __init__(self, ctx, logging=True): self.logging = logging - super(RawTcpLayer, self).__init__(ctx) + super(RawTCPLayer, self).__init__(ctx) def __call__(self): self.connect() diff --git a/libmproxy/protocol2/tls.py b/libmproxy/protocol/tls.py index 73bb12f3..b85a6595 100644 --- a/libmproxy/protocol2/tls.py +++ b/libmproxy/protocol/tls.py @@ -8,7 +8,7 @@ from netlib.tcp import NetLibError, NetLibInvalidCertificateError from netlib.http.http1 import HTTP1Protocol from ..contrib.tls._constructs import ClientHello from ..exceptions import ProtocolException -from .layer import Layer +from .base import Layer def is_tls_record_magic(d): diff --git a/libmproxy/protocol2/http.py b/libmproxy/protocol2/http.py deleted file mode 100644 index a508ae8b..00000000 --- a/libmproxy/protocol2/http.py +++ /dev/null @@ -1,588 +0,0 @@ -from __future__ import (absolute_import, print_function, division) - -from netlib import tcp -from netlib.http import status_codes, http1, HttpErrorConnClosed, HttpError -from netlib.http.semantics import CONTENT_MISSING -from netlib import odict -from netlib.tcp import NetLibError, Address -from netlib.http.http1 import HTTP1Protocol -from netlib.http.http2 import HTTP2Protocol - -from .. import version, utils -from ..exceptions import InvalidCredentials, HttpException, ProtocolException -from .layer import Layer -from ..proxy import Kill -from libmproxy.protocol import KILL, Error -from libmproxy.protocol.http import HTTPFlow -from libmproxy.protocol.http_wrappers import HTTPResponse, HTTPRequest - - -class _HttpLayer(Layer): - supports_streaming = False - - def read_request(self): - raise NotImplementedError() - - def send_request(self, request): - raise NotImplementedError() - - def read_response(self, request_method): - raise NotImplementedError() - - def send_response(self, response): - raise NotImplementedError() - - -class _StreamingHttpLayer(_HttpLayer): - supports_streaming = True - - def read_response_headers(self): - raise NotImplementedError - - def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): - raise NotImplementedError() - yield "this is a generator" - - def send_response_headers(self, response): - raise NotImplementedError - - def send_response_body(self, response, chunks): - raise NotImplementedError() - - -class Http1Layer(_StreamingHttpLayer): - def __init__(self, ctx, mode): - super(Http1Layer, self).__init__(ctx) - self.mode = mode - self.client_protocol = HTTP1Protocol(self.client_conn) - self.server_protocol = HTTP1Protocol(self.server_conn) - - def read_request(self): - return HTTPRequest.from_protocol( - self.client_protocol, - body_size_limit=self.config.body_size_limit - ) - - def send_request(self, request): - self.server_conn.send(self.server_protocol.assemble(request)) - - def read_response(self, request_method): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=request_method, - body_size_limit=self.config.body_size_limit, - include_body=True - ) - - def send_response(self, response): - self.client_conn.send(self.client_protocol.assemble(response)) - - def read_response_headers(self): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=None, # does not matter if we don't read the body. - body_size_limit=self.config.body_size_limit, - include_body=False - ) - - def read_response_body(self, headers, request_method, response_code, max_chunk_size=None): - return self.server_protocol.read_http_body_chunked( - headers, - self.config.body_size_limit, - request_method, - response_code, - False, - max_chunk_size - ) - - def send_response_headers(self, response): - h = self.client_protocol._assemble_response_first_line(response) - self.client_conn.wfile.write(h + "\r\n") - h = self.client_protocol._assemble_response_headers( - response, - preserve_transfer_encoding=True - ) - self.client_conn.send(h + "\r\n") - - def send_response_body(self, response, chunks): - if self.client_protocol.has_chunked_encoding(response.headers): - chunks = ( - "%d\r\n%s\r\n" % (len(chunk), chunk) - for chunk in chunks - ) - for chunk in chunks: - self.client_conn.send(chunk) - - def connect(self): - self.ctx.connect() - self.server_protocol = HTTP1Protocol(self.server_conn) - - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP1Protocol(self.server_conn) - - def set_server(self, *args, **kwargs): - self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP1Protocol(self.server_conn) - - def __call__(self): - layer = HttpLayer(self, self.mode) - layer() - - -# TODO: The HTTP2 layer is missing multiplexing, which requires a major rewrite. -class Http2Layer(_HttpLayer): - def __init__(self, ctx, mode): - super(Http2Layer, self).__init__(ctx) - self.mode = mode - self.client_protocol = HTTP2Protocol(self.client_conn, is_server=True, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - - def read_request(self): - request = HTTPRequest.from_protocol( - self.client_protocol, - body_size_limit=self.config.body_size_limit - ) - self._stream_id = request.stream_id - return request - - def send_request(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames - self.server_conn.send(self.server_protocol.assemble(message)) - - def read_response(self, request_method): - return HTTPResponse.from_protocol( - self.server_protocol, - request_method=request_method, - body_size_limit=self.config.body_size_limit, - include_body=True, - stream_id=self._stream_id - ) - - def send_response(self, message): - # TODO: implement flow control and WINDOW_UPDATE frames - self.client_conn.send(self.client_protocol.assemble(message)) - - def connect(self): - self.ctx.connect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def reconnect(self): - self.ctx.reconnect() - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def set_server(self, *args, **kwargs): - self.ctx.set_server(*args, **kwargs) - self.server_protocol = HTTP2Protocol(self.server_conn, is_server=False, - unhandled_frame_cb=self.handle_unexpected_frame) - self.server_protocol.perform_connection_preface() - - def __call__(self): - self.server_protocol.perform_connection_preface() - layer = HttpLayer(self, self.mode) - layer() - - def handle_unexpected_frame(self, frm): - self.log("Unexpected HTTP2 Frame: %s" % frm.human_readable(), "info") - - -def make_error_response(status_code, message, headers=None): - response = status_codes.RESPONSES.get(status_code, "Unknown") - body = """ - <html> - <head> - <title>%d %s</title> - </head> - <body>%s</body> - </html> - """.strip() % (status_code, response, message) - - if not headers: - headers = odict.ODictCaseless() - headers["Server"] = [version.NAMEVERSION] - headers["Connection"] = ["close"] - headers["Content-Length"] = [len(body)] - headers["Content-Type"] = ["text/html"] - - return HTTPResponse( - (1, 1), # FIXME: Should be a string. - status_code, - response, - headers, - body, - ) - - -def make_connect_request(address): - address = Address.wrap(address) - return HTTPRequest( - "authority", "CONNECT", None, address.host, address.port, None, (1, 1), - odict.ODictCaseless(), "" - ) - - -def make_connect_response(httpversion): - headers = odict.ODictCaseless([ - ["Content-Length", "0"], - ["Proxy-Agent", version.NAMEVERSION] - ]) - return HTTPResponse( - httpversion, - 200, - "Connection established", - headers, - "", - ) - - -class ConnectServerConnection(object): - """ - "Fake" ServerConnection to represent state after a CONNECT request to an upstream proxy. - """ - - def __init__(self, address, ctx): - self.address = tcp.Address.wrap(address) - self._ctx = ctx - - @property - def via(self): - return self._ctx.server_conn - - def __getattr__(self, item): - return getattr(self.via, item) - - -class UpstreamConnectLayer(Layer): - def __init__(self, ctx, connect_request): - super(UpstreamConnectLayer, self).__init__(ctx) - self.connect_request = connect_request - self.server_conn = ConnectServerConnection( - (connect_request.host, connect_request.port), - self.ctx - ) - - def __call__(self): - layer = self.ctx.next_layer(self) - layer() - - def connect(self): - if not self.server_conn: - self.ctx.connect() - self.send_request(self.connect_request) - else: - pass # swallow the message - - def reconnect(self): - self.ctx.reconnect() - self.send_request(self.connect_request) - resp = self.read_response("CONNECT") - if resp.code != 200: - raise ProtocolException("Reconnect: Upstream server refuses CONNECT request") - - def set_server(self, address, server_tls=None, sni=None, depth=1): - if depth == 1: - if self.ctx.server_conn: - self.ctx.reconnect() - address = Address.wrap(address) - self.connect_request.host = address.host - self.connect_request.port = address.port - self.server_conn.address = address - else: - self.ctx.set_server(address, server_tls, sni, depth - 1) - - -class HttpLayer(Layer): - def __init__(self, ctx, mode): - super(HttpLayer, self).__init__(ctx) - self.mode = mode - self.__original_server_conn = None - "Contains the original destination in transparent mode, which needs to be restored" - "if an inline script modified the target server for a single http request" - - def __call__(self): - if self.mode == "transparent": - self.__original_server_conn = self.server_conn - while True: - try: - flow = HTTPFlow(self.client_conn, self.server_conn, live=self) - - try: - request = self.read_request() - except tcp.NetLibError: - # don't throw an error for disconnects that happen - # before/between requests. - return - - self.log("request", "debug", [repr(request)]) - - # Handle Proxy Authentication - self.authenticate(request) - - # Regular Proxy Mode: Handle CONNECT - if self.mode == "regular" and request.form_in == "authority": - self.handle_regular_mode_connect(request) - return - - # Make sure that the incoming request matches our expectations - self.validate_request(request) - - flow.request = request - self.process_request_hook(flow) - - if not flow.response: - self.establish_server_connection(flow) - self.get_response_from_server(flow) - - self.send_response_to_client(flow) - - if self.check_close_connection(flow): - return - - # TODO: Implement HTTP Upgrade - - # Upstream Proxy Mode: Handle CONNECT - if flow.request.form_in == "authority" and flow.response.code == 200: - self.handle_upstream_mode_connect(flow.request.copy()) - return - - except (HttpErrorConnClosed, NetLibError, HttpError, ProtocolException) as e: - if flow.request and not flow.response: - flow.error = Error(repr(e)) - self.channel.ask("error", flow) - try: - self.send_response(make_error_response( - getattr(e, "code", 502), - repr(e) - )) - except NetLibError: - pass - if isinstance(e, ProtocolException): - raise e - else: - raise ProtocolException("Error in HTTP connection: %s" % repr(e), e) - finally: - flow.live = False - - def handle_regular_mode_connect(self, request): - self.set_server((request.host, request.port)) - self.send_response(make_connect_response(request.httpversion)) - layer = self.ctx.next_layer(self) - layer() - - def handle_upstream_mode_connect(self, connect_request): - layer = UpstreamConnectLayer(self, connect_request) - layer() - - def check_close_connection(self, flow): - """ - Checks if the connection should be closed depending on the HTTP - semantics. Returns True, if so. - """ - - # TODO: add logic for HTTP/2 - - close_connection = ( - http1.HTTP1Protocol.connection_close( - flow.request.httpversion, - flow.request.headers - ) or http1.HTTP1Protocol.connection_close( - flow.response.httpversion, - flow.response.headers - ) or http1.HTTP1Protocol.expected_http_body_size( - flow.response.headers, - False, - flow.request.method, - flow.response.code) == -1 - ) - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for - # https://github.com/mitmproxy/mitmproxy/issues/313: Some - # proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 - # and no Content-Length header - - return False - return close_connection - - def send_response_to_client(self, flow): - if not (self.supports_streaming and flow.response.stream): - # no streaming: - # we already received the full response from the server and can - # send it to the client straight away. - self.send_response(flow.response) - else: - # streaming: - # First send the headers and then transfer the response incrementally - self.send_response_headers(flow.response) - chunks = self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code, - max_chunk_size=4096 - ) - if callable(flow.response.stream): - chunks = flow.response.stream(chunks) - self.send_response_body(flow.response, chunks) - flow.response.timestamp_end = utils.timestamp() - - def get_response_from_server(self, flow): - def get_response(): - self.send_request(flow.request) - if self.supports_streaming: - flow.response = self.read_response_headers() - else: - flow.response = self.read_response() - - try: - get_response() - except (tcp.NetLibError, HttpErrorConnClosed) as v: - self.log( - "server communication error: %s" % repr(v), - level="debug" - ) - # In any case, we try to reconnect at least once. This is - # necessary because it might be possible that we already - # initiated an upstream connection after clientconnect that - # has already been expired, e.g consider the following event - # log: - # > clientconnect (transparent mode destination known) - # > serverconnect (required for client tls handshake) - # > read n% of large request - # > server detects timeout, disconnects - # > read (100-n)% of large request - # > send large request upstream - self.reconnect() - get_response() - - # call the appropriate script hook - this is an opportunity for an - # inline script to set flow.stream = True - flow = self.channel.ask("responseheaders", flow) - if flow is None or flow == KILL: - raise Kill() - - if self.supports_streaming: - if flow.response.stream: - flow.response.content = CONTENT_MISSING - else: - flow.response.content = "".join(self.read_response_body( - flow.response.headers, - flow.request.method, - flow.response.code - )) - flow.response.timestamp_end = utils.timestamp() - - # no further manipulation of self.server_conn beyond this point - # we can safely set it as the final attribute value here. - flow.server_conn = self.server_conn - - self.log( - "response", - "debug", - [repr(flow.response)] - ) - response_reply = self.channel.ask("response", flow) - if response_reply is None or response_reply == KILL: - raise Kill() - - def process_request_hook(self, flow): - # Determine .scheme, .host and .port attributes for inline scripts. - # For absolute-form requests, they are directly given in the request. - # For authority-form requests, we only need to determine the request scheme. - # For relative-form requests, we need to determine host and port as - # well. - if self.mode == "regular": - pass # only absolute-form at this point, nothing to do here. - elif self.mode == "upstream": - if flow.request.form_in == "authority": - flow.request.scheme = "http" # pseudo value - else: - flow.request.host = self.__original_server_conn.address.host - flow.request.port = self.__original_server_conn.address.port - flow.request.scheme = "https" if self.__original_server_conn.tls_established else "http" - - request_reply = self.channel.ask("request", flow) - if request_reply is None or request_reply == KILL: - raise Kill() - if isinstance(request_reply, HTTPResponse): - flow.response = request_reply - return - - def establish_server_connection(self, flow): - address = tcp.Address((flow.request.host, flow.request.port)) - tls = (flow.request.scheme == "https") - - if self.mode == "regular" or self.mode == "transparent": - # If there's an existing connection that doesn't match our expectations, kill it. - if address != self.server_conn.address or tls != self.server_conn.ssl_established: - self.set_server(address, tls, address.host) - # Establish connection is neccessary. - if not self.server_conn: - self.connect() - - # SetServer is not guaranteed to work with TLS: - # If there's not TlsLayer below which could catch the exception, - # TLS will not be established. - if tls and not self.server_conn.tls_established: - raise ProtocolException( - "Cannot upgrade to SSL, no TLS layer on the protocol stack.") - else: - if not self.server_conn: - self.connect() - if tls: - raise HttpException("Cannot change scheme in upstream proxy mode.") - """ - # This is a very ugly (untested) workaround to solve a very ugly problem. - if self.server_conn and self.server_conn.tls_established and not ssl: - self.reconnect() - elif ssl and not hasattr(self, "connected_to") or self.connected_to != address: - if self.server_conn.tls_established: - self.reconnect() - - self.send_request(make_connect_request(address)) - tls_layer = TlsLayer(self, False, True) - tls_layer._establish_tls_with_server() - """ - - def validate_request(self, request): - if request.form_in == "absolute" and request.scheme != "http": - self.send_response( - make_error_response(400, "Invalid request scheme: %s" % request.scheme)) - raise HttpException("Invalid request scheme: %s" % request.scheme) - - expected_request_forms = { - "regular": ("absolute",), # an authority request would already be handled. - "upstream": ("authority", "absolute"), - "transparent": ("relative",) - } - - allowed_request_forms = expected_request_forms[self.mode] - if request.form_in not in allowed_request_forms: - err_message = "Invalid HTTP request form (expected: %s, got: %s)" % ( - " or ".join(allowed_request_forms), request.form_in - ) - self.send_response(make_error_response(400, err_message)) - raise HttpException(err_message) - - if self.mode == "regular": - request.form_out = "relative" - - def authenticate(self, request): - if self.config.authenticator: - if self.config.authenticator.authenticate(request.headers): - self.config.authenticator.clean(request.headers) - else: - self.send_response(make_error_response( - 407, - "Proxy Authentication Required", - odict.ODictCaseless( - [ - [k, v] for k, v in - self.config.authenticator.auth_challenge_headers().items() - ]) - )) - raise InvalidCredentials("Proxy Authentication Required") diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index 709654cb..d5297cb1 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1,11 +1,9 @@ from __future__ import (absolute_import, print_function, division) -from .primitives import Log, Kill +from .server import ProxyServer, DummyServer from .config import ProxyConfig -from .connection import ClientConnection, ServerConnection __all__ = [ - "Log", "Kill", + "ProxyServer", "DummyServer", "ProxyConfig", - "ClientConnection", "ServerConnection" -]
\ No newline at end of file +] diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index b360abbd..65029087 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -6,9 +6,9 @@ from OpenSSL import SSL from netlib import certutils, tcp from netlib.http import authentication +from netlib.tcp import Address, sslversion_choices from .. import utils, platform -from netlib.tcp import Address, sslversion_choices CONF_BASENAME = "mitmproxy" CA_DIR = "~/.mitmproxy" diff --git a/libmproxy/protocol2/__init__.py b/libmproxy/proxy/modes/__init__.py index 61b9a77e..f014ed98 100644 --- a/libmproxy/protocol2/__init__.py +++ b/libmproxy/proxy/modes/__init__.py @@ -1,13 +1,12 @@ from __future__ import (absolute_import, print_function, division) -from .root_context import RootContext -from .socks_proxy import Socks5Proxy -from .reverse_proxy import ReverseProxy from .http_proxy import HttpProxy, HttpUpstreamProxy +from .reverse_proxy import ReverseProxy +from .socks_proxy import Socks5Proxy from .transparent_proxy import TransparentProxy -from .http import make_error_response __all__ = [ - "RootContext", - "Socks5Proxy", "ReverseProxy", "HttpProxy", "HttpUpstreamProxy", "TransparentProxy", - "make_error_response" + "HttpProxy", "HttpUpstreamProxy", + "ReverseProxy", + "Socks5Proxy", + "TransparentProxy" ] diff --git a/libmproxy/protocol2/http_proxy.py b/libmproxy/proxy/modes/http_proxy.py index 2876c022..90c54cc6 100644 --- a/libmproxy/protocol2/http_proxy.py +++ b/libmproxy/proxy/modes/http_proxy.py @@ -1,6 +1,6 @@ from __future__ import (absolute_import, print_function, division) -from .layer import Layer, ServerConnectionMixin +from ...protocol import Layer, ServerConnectionMixin class HttpProxy(Layer, ServerConnectionMixin): diff --git a/libmproxy/protocol2/reverse_proxy.py b/libmproxy/proxy/modes/reverse_proxy.py index 3ca998d5..b57ac5eb 100644 --- a/libmproxy/protocol2/reverse_proxy.py +++ b/libmproxy/proxy/modes/reverse_proxy.py @@ -1,6 +1,6 @@ from __future__ import (absolute_import, print_function, division) -from .layer import Layer, ServerConnectionMixin +from ...protocol import Layer, ServerConnectionMixin class ReverseProxy(Layer, ServerConnectionMixin): diff --git a/libmproxy/protocol2/socks_proxy.py b/libmproxy/proxy/modes/socks_proxy.py index 525520e8..ebaf939e 100644 --- a/libmproxy/protocol2/socks_proxy.py +++ b/libmproxy/proxy/modes/socks_proxy.py @@ -2,8 +2,9 @@ from __future__ import (absolute_import, print_function, division) from netlib import socks from netlib.tcp import NetLibError -from ..exceptions import Socks5Exception -from .layer import Layer, ServerConnectionMixin + +from ...exceptions import Socks5Exception +from ...protocol import Layer, ServerConnectionMixin class Socks5Proxy(Layer, ServerConnectionMixin): diff --git a/libmproxy/protocol2/transparent_proxy.py b/libmproxy/proxy/modes/transparent_proxy.py index e6ebf115..96ad86c4 100644 --- a/libmproxy/protocol2/transparent_proxy.py +++ b/libmproxy/proxy/modes/transparent_proxy.py @@ -1,8 +1,8 @@ from __future__ import (absolute_import, print_function, division) -from ..exceptions import ProtocolException -from .. import platform -from .layer import Layer, ServerConnectionMixin +from ... import platform +from ...exceptions import ProtocolException +from ...protocol import Layer, ServerConnectionMixin class TransparentProxy(Layer, ServerConnectionMixin): diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py deleted file mode 100644 index 2e440fe8..00000000 --- a/libmproxy/proxy/primitives.py +++ /dev/null @@ -1,15 +0,0 @@ -from __future__ import absolute_import -import collections -from netlib import socks, tcp - - -class Log(object): - def __init__(self, msg, level="info"): - self.msg = msg - self.level = level - - -class Kill(Exception): - """ - Kill a connection. - """
\ No newline at end of file diff --git a/libmproxy/protocol2/root_context.py b/libmproxy/proxy/root_context.py index daea54bd..35909612 100644 --- a/libmproxy/protocol2/root_context.py +++ b/libmproxy/proxy/root_context.py @@ -3,12 +3,10 @@ from __future__ import (absolute_import, print_function, division) from netlib.http.http1 import HTTP1Protocol from netlib.http.http2 import HTTP2Protocol -from .rawtcp import RawTcpLayer -from .tls import TlsLayer, is_tls_record_magic -from .http import Http1Layer, Http2Layer -from .layer import ServerConnectionMixin -from .http_proxy import HttpProxy, HttpUpstreamProxy -from .reverse_proxy import ReverseProxy +from ..protocol import ( + RawTCPLayer, TlsLayer, Http1Layer, Http2Layer, is_tls_record_magic, ServerConnectionMixin +) +from .modes import HttpProxy, HttpUpstreamProxy, ReverseProxy class RootContext(object): @@ -35,7 +33,7 @@ class RootContext(object): # 1. Check for --ignore. if self.config.check_ignore(top_layer.server_conn.address): - return RawTcpLayer(top_layer, logging=False) + return RawTCPLayer(top_layer, logging=False) d = top_layer.client_conn.rfile.peek(3) client_tls = is_tls_record_magic(d) @@ -61,7 +59,7 @@ class RootContext(object): # 4. Check for --tcp if self.config.check_tcp(top_layer.server_conn.address): - return RawTcpLayer(top_layer) + return RawTCPLayer(top_layer) # 5. Check for TLS ALPN (HTTP1/HTTP2) if isinstance(top_layer, TlsLayer): diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 5abd0877..2a451ba1 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -3,14 +3,15 @@ from __future__ import absolute_import, print_function import traceback import sys import socket + from netlib import tcp from netlib.http.http1 import HTTP1Protocol from netlib.tcp import NetLibError - -from .. import protocol2 from ..exceptions import ProtocolException, ServerException -from .primitives import Log, Kill -from .connection import ClientConnection +from ..protocol import Log, Kill +from ..models import ClientConnection, make_error_response +from .modes import HttpUpstreamProxy, HttpProxy, ReverseProxy, TransparentProxy, Socks5Proxy +from .root_context import RootContext class DummyServer: @@ -71,7 +72,7 @@ class ConnectionHandler(object): """@type: libmproxy.controller.Channel""" def _create_root_layer(self): - root_context = protocol2.RootContext( + root_context = RootContext( self.client_conn, self.config, self.channel @@ -79,23 +80,23 @@ class ConnectionHandler(object): mode = self.config.mode if mode == "upstream": - return protocol2.HttpUpstreamProxy( + return HttpUpstreamProxy( root_context, self.config.upstream_server.address ) elif mode == "transparent": - return protocol2.TransparentProxy(root_context) + return TransparentProxy(root_context) elif mode == "reverse": server_tls = self.config.upstream_server.scheme == "https" - return protocol2.ReverseProxy( + return ReverseProxy( root_context, self.config.upstream_server.address, server_tls ) elif mode == "socks5": - return protocol2.Socks5Proxy(root_context) + return Socks5Proxy(root_context) elif mode == "regular": - return protocol2.HttpProxy(root_context) + return HttpProxy(root_context) elif callable(mode): # pragma: nocover return mode(root_context) else: # pragma: nocover @@ -116,7 +117,7 @@ class ConnectionHandler(object): # we send an HTTP error response, which is both # understandable by HTTP clients and humans. try: - error_response = protocol2.make_error_response(502, repr(e)) + error_response = make_error_response(502, repr(e)) self.client_conn.send(HTTP1Protocol().assemble(error_response)) except NetLibError: pass diff --git a/test/test_dump.py b/test/test_dump.py index b05f6a0f..a0ad6cb4 100644 --- a/test/test_dump.py +++ b/test/test_dump.py @@ -1,18 +1,18 @@ import os from cStringIO import StringIO +from libmproxy.models import HTTPResponse import netlib.tutils from netlib.http.semantics import CONTENT_MISSING from libmproxy import dump, flow -from libmproxy.protocol import http_wrappers -from libmproxy.proxy import Log +from libmproxy.protocol import Log import tutils import mock def test_strfuncs(): - t = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + t = HTTPResponse.wrap(netlib.tutils.tresp()) t.is_replay = True dump.str_response(t) @@ -34,7 +34,7 @@ class TestDumpMaster: m.handle_clientconnect(f.client_conn) m.handle_serverconnect(f.server_conn) m.handle_request(f) - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp(content)) + f.response = HTTPResponse.wrap(netlib.tutils.tresp(content)) f = m.handle_response(f) m.handle_clientdisconnect(f.client_conn) return f @@ -71,7 +71,7 @@ class TestDumpMaster: f = tutils.tflow() f.request.content = CONTENT_MISSING m.handle_request(f) - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = HTTPResponse.wrap(netlib.tutils.tresp()) f.response.content = CONTENT_MISSING m.handle_response(f) assert "content missing" in cs.getvalue() diff --git a/test/test_filt.py b/test/test_filt.py index bcdf6e4c..aeec2485 100644 --- a/test/test_filt.py +++ b/test/test_filt.py @@ -2,7 +2,7 @@ import cStringIO from netlib import odict from libmproxy import filt, flow from libmproxy.protocol import http -from libmproxy.protocol.primitives import Error +from libmproxy.models import Error import tutils diff --git a/test/test_flow.py b/test/test_flow.py index 5c49deed..9cce26b3 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -3,21 +3,18 @@ import time import os.path from cStringIO import StringIO import email.utils + import mock -from libmproxy.cmdline import parse_server_spec import netlib.utils from netlib import odict -from netlib.http.semantics import CONTENT_MISSING, HDR_FORM_URLENCODED, HDR_FORM_MULTIPART - -from libmproxy import filt, protocol, controller, utils, tnetstring, flow -from libmproxy.protocol import http_wrappers -from libmproxy.protocol.primitives import Error, Flow -from libmproxy.protocol.http import decoded +from netlib.http.semantics import CONTENT_MISSING, HDR_FORM_URLENCODED +from libmproxy import filt, protocol, controller, tnetstring, flow +from libmproxy.models import Error, Flow, HTTPRequest, HTTPResponse, HTTPFlow, decoded from libmproxy.proxy.config import HostMatcher from libmproxy.proxy import ProxyConfig from libmproxy.proxy.server import DummyServer -from libmproxy.proxy.connection import ClientConnection +from libmproxy.models.connections import ClientConnection import tutils @@ -25,7 +22,7 @@ def test_app_registry(): ar = flow.AppRegistry() ar.add("foo", "domain", 80) - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.host = "domain" r.port = 80 assert ar.get(r) @@ -33,7 +30,7 @@ def test_app_registry(): r.port = 81 assert not ar.get(r) - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.host = "domain2" r.port = 80 assert not ar.get(r) @@ -386,7 +383,7 @@ class TestFlow: def test_backup(self): f = tutils.tflow() - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = HTTPResponse.wrap(netlib.tutils.tresp()) f.request.content = "foo" assert not f.modified() f.backup() @@ -405,13 +402,13 @@ class TestFlow: def test_getset_state(self): f = tutils.tflow(resp=True) state = f.get_state() - assert f.get_state() == protocol.http.HTTPFlow.from_state( + assert f.get_state() == HTTPFlow.from_state( state).get_state() f.response = None f.error = Error("error") state = f.get_state() - assert f.get_state() == protocol.http.HTTPFlow.from_state( + assert f.get_state() == HTTPFlow.from_state( state).get_state() f2 = f.copy() @@ -519,16 +516,16 @@ class TestState: assert c.add_flow(newf) assert c.active_flow_count() == 2 - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = HTTPResponse.wrap(netlib.tutils.tresp()) assert c.update_flow(f) assert c.flow_count() == 2 assert c.active_flow_count() == 1 - _ = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + _ = HTTPResponse.wrap(netlib.tutils.tresp()) assert not c.update_flow(None) assert c.active_flow_count() == 1 - newf.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + newf.response = HTTPResponse.wrap(netlib.tutils.tresp()) assert c.update_flow(newf) assert c.active_flow_count() == 0 @@ -560,7 +557,7 @@ class TestState: c.set_limit("~s") assert c.limit_txt == "~s" assert len(c.view) == 0 - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = HTTPResponse.wrap(netlib.tutils.tresp()) c.update_flow(f) assert len(c.view) == 1 c.set_limit(None) @@ -592,7 +589,7 @@ class TestState: def _add_response(self, state): f = tutils.tflow() state.add_flow(f) - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = HTTPResponse.wrap(netlib.tutils.tresp()) state.update_flow(f) def _add_error(self, state): @@ -807,11 +804,11 @@ class TestFlowMaster: fm.anticomp = True f = tutils.tflow(req=None) fm.handle_clientconnect(f.client_conn) - f.request = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + f.request = HTTPRequest.wrap(netlib.tutils.treq()) fm.handle_request(f) assert s.flow_count() == 1 - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + f.response = HTTPResponse.wrap(netlib.tutils.tresp()) fm.handle_response(f) assert not fm.handle_response(None) assert s.flow_count() == 1 @@ -856,7 +853,7 @@ class TestFlowMaster: s = flow.State() f = tutils.tflow() - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp(f.request)) + f.response = HTTPResponse.wrap(netlib.tutils.tresp(f.request)) pb = [f] fm = flow.FlowMaster(None, s) @@ -910,7 +907,7 @@ class TestFlowMaster: def test_server_playback_kill(self): s = flow.State() f = tutils.tflow() - f.response = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp(f.request)) + f.response = HTTPResponse.wrap(netlib.tutils.tresp(f.request)) pb = [f] fm = flow.FlowMaster(None, s) fm.refresh_server_playback = True @@ -1009,7 +1006,7 @@ class TestRequest: assert r.get_state() == r2.get_state() def test_get_url(self): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) assert r.url == "http://address:22/path" @@ -1030,7 +1027,7 @@ class TestRequest: assert r.pretty_url(True) == "https://foo.com:22/path" def test_path_components(self): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.path = "/" assert r.get_path_components() == [] r.path = "/foo/bar" @@ -1050,7 +1047,7 @@ class TestRequest: def test_getset_form_urlencoded(self): d = odict.ODict([("one", "two"), ("three", "four")]) - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq(content=netlib.utils.urlencode(d.lst))) + r = HTTPRequest.wrap(netlib.tutils.treq(content=netlib.utils.urlencode(d.lst))) r.headers["content-type"] = [HDR_FORM_URLENCODED] assert r.get_form_urlencoded() == d @@ -1064,7 +1061,7 @@ class TestRequest: def test_getset_query(self): h = odict.ODictCaseless() - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.path = "/foo?x=y&a=b" q = r.get_query() assert q.lst == [("x", "y"), ("a", "b")] @@ -1087,7 +1084,7 @@ class TestRequest: def test_anticache(self): h = odict.ODictCaseless() - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.headers = h h["if-modified-since"] = ["test"] h["if-none-match"] = ["test"] @@ -1096,7 +1093,7 @@ class TestRequest: assert not "if-none-match" in r.headers def test_replace(self): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.path = "path/foo" r.headers["Foo"] = ["fOo"] r.content = "afoob" @@ -1106,31 +1103,31 @@ class TestRequest: assert r.headers["boo"] == ["boo"] def test_constrain_encoding(self): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.headers["accept-encoding"] = ["gzip", "oink"] r.constrain_encoding() assert "oink" not in r.headers["accept-encoding"] def test_decodeencode(self): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" r.decode() assert not r.headers["content-encoding"] assert r.content == "falafel" - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.content = "falafel" assert not r.decode() - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" r.encode("identity") assert r.headers["content-encoding"] == ["identity"] assert r.content == "falafel" - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" r.encode("gzip") @@ -1141,7 +1138,7 @@ class TestRequest: assert r.content == "falafel" def test_get_decoded_content(self): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) r.content = None r.headers["content-encoding"] = ["identity"] assert r.get_decoded_content() == None @@ -1153,7 +1150,7 @@ class TestRequest: def test_get_content_type(self): h = odict.ODictCaseless() h["Content-Type"] = ["text/plain"] - resp = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + resp = HTTPResponse.wrap(netlib.tutils.tresp()) resp.headers = h assert resp.headers.get_first("content-type") == "text/plain" @@ -1166,7 +1163,7 @@ class TestResponse: assert resp2.get_state() == resp.get_state() def test_refresh(self): - r = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + r = HTTPResponse.wrap(netlib.tutils.tresp()) n = time.time() r.headers["date"] = [email.utils.formatdate(n)] pre = r.headers["date"] @@ -1184,7 +1181,7 @@ class TestResponse: r.refresh() def test_refresh_cookie(self): - r = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + r = HTTPResponse.wrap(netlib.tutils.tresp()) # Invalid expires format, sent to us by Reddit. c = "rfoo=bar; Domain=reddit.com; expires=Thu, 31 Dec 2037 23:59:59 GMT; Path=/" @@ -1194,7 +1191,7 @@ class TestResponse: assert "00:21:38" in r._refresh_cookie(c, 60) def test_replace(self): - r = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["Foo"] = ["fOo"] r.content = "afoob" assert r.replace("foo(?i)", "boo") == 3 @@ -1202,21 +1199,21 @@ class TestResponse: assert r.headers["boo"] == ["boo"] def test_decodeencode(self): - r = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" assert r.decode() assert not r.headers["content-encoding"] assert r.content == "falafel" - r = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" r.encode("identity") assert r.headers["content-encoding"] == ["identity"] assert r.content == "falafel" - r = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + r = HTTPResponse.wrap(netlib.tutils.tresp()) r.headers["content-encoding"] = ["identity"] r.content = "falafel" r.encode("gzip") @@ -1233,7 +1230,7 @@ class TestResponse: def test_get_content_type(self): h = odict.ODictCaseless() h["Content-Type"] = ["text/plain"] - resp = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + resp = HTTPResponse.wrap(netlib.tutils.tresp()) resp.headers = h assert resp.headers.get_first("content-type") == "text/plain" @@ -1277,7 +1274,7 @@ class TestClientConnection: def test_decoded(): - r = http_wrappers.HTTPRequest.wrap(netlib.tutils.treq()) + r = HTTPRequest.wrap(netlib.tutils.treq()) assert r.content == "content" assert not r.headers["content-encoding"] r.encode("gzip") diff --git a/test/test_proxy.py b/test/test_proxy.py index 301ce2ca..b9ca2cce 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -1,14 +1,14 @@ +import mock +from OpenSSL import SSL + from libmproxy import cmdline from libmproxy.proxy import ProxyConfig from libmproxy.proxy.config import process_proxy_options -from libmproxy.proxy.connection import ServerConnection +from libmproxy.models.connections import ServerConnection from libmproxy.proxy.server import DummyServer, ProxyServer, ConnectionHandler import tutils from libpathod import test from netlib import http, tcp -import mock - -from OpenSSL import SSL class TestServerConnection: diff --git a/test/test_server.py b/test/test_server.py index 66c3a0ae..23d802ca 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -11,7 +11,9 @@ from netlib.http.semantics import CONTENT_MISSING from libpathod import pathoc, pathod from libmproxy.proxy.config import HostMatcher -from libmproxy.protocol import KILL, Error, http_wrappers +from libmproxy.protocol import Kill +from libmproxy.models import Error, HTTPResponse + import tutils import tservers @@ -734,7 +736,7 @@ class TestStreamRequest(tservers.HTTPProxTest): class MasterFakeResponse(tservers.TestMaster): def handle_request(self, f): - resp = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + resp = HTTPResponse.wrap(netlib.tutils.tresp()) f.reply(resp) @@ -759,7 +761,7 @@ class TestServerConnect(tservers.HTTPProxTest): class MasterKillRequest(tservers.TestMaster): def handle_request(self, f): - f.reply(KILL) + f.reply(Kill) class TestKillRequest(tservers.HTTPProxTest): @@ -773,7 +775,7 @@ class TestKillRequest(tservers.HTTPProxTest): class MasterKillResponse(tservers.TestMaster): def handle_response(self, f): - f.reply(KILL) + f.reply(Kill) class TestKillResponse(tservers.HTTPProxTest): @@ -799,7 +801,7 @@ class TestTransparentResolveError(tservers.TransparentProxTest): class MasterIncomplete(tservers.TestMaster): def handle_request(self, f): - resp = http_wrappers.HTTPResponse.wrap(netlib.tutils.tresp()) + resp = HTTPResponse.wrap(netlib.tutils.tresp()) resp.content = CONTENT_MISSING f.reply(resp) @@ -938,7 +940,7 @@ class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxTest): if not (k[0] in exclude): f.client_conn.finish() f.error = Error("terminated") - f.reply(KILL) + f.reply(Kill) return _func(f) setattr(master, attr, handler) diff --git a/test/tutils.py b/test/tutils.py index 61b1154c..d64388f3 100644 --- a/test/tutils.py +++ b/test/tutils.py @@ -3,22 +3,19 @@ import shutil import tempfile import argparse import sys -import mock_urwid from cStringIO import StringIO from contextlib import contextmanager + from nose.plugins.skip import SkipTest from mock import Mock -from time import time -from netlib import certutils, odict import netlib.tutils - -from libmproxy import flow, utils, controller -from libmproxy.protocol import http, http_wrappers -from libmproxy.proxy.connection import ClientConnection, ServerConnection +from libmproxy import utils, controller +from libmproxy.models import ( + ClientConnection, ServerConnection, Error, HTTPRequest, HTTPResponse, HTTPFlow +) from libmproxy.console.flowview import FlowView from libmproxy.console import ConsoleState -from libmproxy.protocol.primitives import Error def _SkipWindows(): @@ -53,11 +50,11 @@ def tflow(client_conn=True, server_conn=True, req=True, resp=None, err=None): err = terr() if req: - req = http_wrappers.HTTPRequest.wrap(req) + req = HTTPRequest.wrap(req) if resp: - resp = http_wrappers.HTTPResponse.wrap(resp) + resp = HTTPResponse.wrap(resp) - f = http.HTTPFlow(client_conn, server_conn) + f = HTTPFlow(client_conn, server_conn) f.request = req f.response = resp f.error = err @@ -91,7 +88,6 @@ def tserver_conn(): return c - def terr(content="error"): """ @return: libmproxy.protocol.primitives.Error @@ -106,7 +102,7 @@ def tflowview(request_contents=None): if request_contents is None: flow = tflow() else: - flow = tflow(req=treq(request_contents)) + flow = tflow(req=netlib.tutils.treq(request_contents)) fv = FlowView(m, cs, flow) return fv @@ -184,4 +180,5 @@ def capture_stderr(command, *args, **kwargs): yield sys.stderr.getvalue() sys.stderr = out + test_data = utils.Data(__name__) |