diff options
| author | Aldo Cortesi <aldo@corte.si> | 2014-09-07 12:59:35 +1200 |
|---|---|---|
| committer | Aldo Cortesi <aldo@corte.si> | 2014-09-07 12:59:35 +1200 |
| commit | 3d62e90dbf7ea05283e16752531a261e53a4bb47 (patch) | |
| tree | c3f5aed62bcf13378522968a1c77375dc8102b53 | |
| parent | 0e0cff638c1e055275e77e2af0ae540542f77197 (diff) | |
| parent | fdd7b2f108717900e39e3d0ab220ee65b79304ef (diff) | |
| download | mitmproxy-3d62e90dbf7ea05283e16752531a261e53a4bb47.tar.gz mitmproxy-3d62e90dbf7ea05283e16752531a261e53a4bb47.tar.bz2 mitmproxy-3d62e90dbf7ea05283e16752531a261e53a4bb47.zip | |
Merge pull request #342 from mitmproxy/server_change_api
Server change api
34 files changed, 779 insertions, 711 deletions
diff --git a/doc-src/scripting/inlinescripts.html b/doc-src/scripting/inlinescripts.html index 738f8dc3..eef4e440 100644 --- a/doc-src/scripting/inlinescripts.html +++ b/doc-src/scripting/inlinescripts.html @@ -29,46 +29,45 @@ The new header will be added to all responses passing through the proxy. Called once on startup, before any other events. -### clientconnect(ScriptContext, ClientConnect) +### clientconnect(ScriptContext, ConnectionHandler) Called when a client initiates a connection to the proxy. Note that a connection can correspond to multiple HTTP requests. - -### serverconnect(ScriptContext, ServerConnection) +### serverconnect(ScriptContext, ConnectionHandler) Called when the proxy initiates a connection to the target server. Note that a connection can correspond to multiple HTTP requests. -### request(ScriptContext, Flow) +### request(ScriptContext, HTTPFlow) -Called when a client request has been received. The __Flow__ object is +Called when a client request has been received. The __HTTPFlow__ object is guaranteed to have a non-None __request__ attribute. -### responseheaders(ScriptContext, Flow) +### responseheaders(ScriptContext, HTTPFlow) Called when the headers of a server response have been received. This will always be called before the response hook. -The __Flow__ object is guaranteed to have non-None __request__ and -__response__ attributes. __response.content__ will not be valid, +The __HTTPFlow__ object is guaranteed to have non-None __request__ and +__response__ attributes. __response.content__ will be None, as the response body has not been read yet. -### response(ScriptContext, Flow) +### response(ScriptContext, HTTPFlow) -Called when a server response has been received. The __Flow__ object is +Called when a server response has been received. The __HTTPFlow__ object is guaranteed to have non-None __request__ and __response__ attributes. Note that if response streaming is enabled for this response, __response.content__ will not contain the response body. -### error(ScriptContext, Flow) +### error(ScriptContext, HTTPFlow) Called when a flow error has occurred, e.g. invalid server responses, or interrupted connections. This is distinct from a valid server HTTP error -response, which is simply a response with an HTTP error code. The __Flow__ +response, which is simply a response with an HTTP error code. The __HTTPFlow__ object is guaranteed to have non-None __request__ and __error__ attributes. -### clientdisconnect(ScriptContext, ClientDisconnect) +### clientdisconnect(ScriptContext, ConnectionHandler) Called when a client disconnects from the proxy. @@ -97,22 +96,10 @@ The main classes you will deal with in writing mitmproxy scripts are: <td>Describes a server connection.</td> </tr> <tr> - <th>libmproxy.protocol.primitives.Error</th> - <td>A communications error.</td> - </tr> - <tr> <th>libmproxy.protocol.http.HTTPFlow</th> <td>A collection of objects representing a single HTTP transaction.</td> </tr> <tr> - <th>libmproxy.flow.ODict</th> - - <td>A dictionary-like object for managing sets of key/value data. There - is also a variant called CaselessODict that ignores key case for some - calls (used mainly for headers). - </td> - </tr> - <tr> <th>libmproxy.protocol.http.HTTPResponse</th> <td>An HTTP response.</td> </tr> @@ -121,10 +108,22 @@ The main classes you will deal with in writing mitmproxy scripts are: <td>An HTTP request.</td> </tr> <tr> + <th>libmproxy.protocol.primitives.Error</th> + <td>A communications error.</td> + </tr> + <tr> <th>libmproxy.script.ScriptContext</th> <td> A handle for interacting with mitmproxy's from within scripts.</td> </tr> <tr> + <th>libmproxy.flow.ODict</th> + + <td>A dictionary-like object for managing sets of key/value data. There + is also a variant called CaselessODict that ignores key case for some + calls (used mainly for headers). + </td> + </tr> + <tr> <th>libmproxy.certutils.SSLCert</th> <td>Exposes information SSL certificates.</td> </tr> @@ -161,9 +160,9 @@ flows from a file (see the "scripted data transformation" example on the one-shot script on a single flow through the _|_ (pipe) shortcut in mitmproxy. In this case, there are no client connections, and the events are run in the -following order: __start__, __request__, __response__, __error__, __done__. If +following order: __start__, __request__, __responseheaders__, __response__, __error__, __done__. If the flow doesn't have a __response__ or __error__ associated with it, the -matching event will be skipped. +matching events will be skipped. ## Spaces in the script path By default, spaces are interpreted as separator between the inline script and its arguments (e.g. <code>-s "foo.py diff --git a/examples/add_header.py b/examples/add_header.py index 0c0593d1..b9c8c1c6 100644 --- a/examples/add_header.py +++ b/examples/add_header.py @@ -1,2 +1,2 @@ -def response(context, flow): - flow.response.headers["newheader"] = ["foo"] +def response(ctx, flow): + flow.response.headers["newheader"] = ["foo"]
\ No newline at end of file diff --git a/examples/dup_and_replay.py b/examples/dup_and_replay.py index 9c58d3a4..b38c2b7e 100644 --- a/examples/dup_and_replay.py +++ b/examples/dup_and_replay.py @@ -1,4 +1,4 @@ def request(ctx, flow): - f = ctx.duplicate_flow(flow) - f.request.path = "/changed" - ctx.replay_request(f) + f = ctx.duplicate_flow(flow) + f.request.path = "/changed" + ctx.replay_request(f)
\ No newline at end of file diff --git a/examples/flowbasic b/examples/flowbasic index 8dbe2f28..2b44be3f 100755 --- a/examples/flowbasic +++ b/examples/flowbasic @@ -3,11 +3,14 @@ This example shows how to build a proxy based on mitmproxy's Flow primitives. + Heads Up: In the majority of cases, you want to use inline scripts. + Note that request and response messages are not automatically replied to, so we need to implement handlers to do this. """ import os -from libmproxy import proxy, flow +from libmproxy import flow, proxy +from libmproxy.proxy.server import ProxyServer class MyMaster(flow.FlowMaster): def run(self): @@ -31,9 +34,9 @@ class MyMaster(flow.FlowMaster): config = proxy.ProxyConfig( - cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem") + ca_file = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem") ) state = flow.State() -server = proxy.ProxyServer(config, 8080) +server = ProxyServer(config, 8080) m = MyMaster(server, state) m.run() diff --git a/examples/iframe_injector b/examples/iframe_injector deleted file mode 100755 index 8b1e02f1..00000000 --- a/examples/iframe_injector +++ /dev/null @@ -1,50 +0,0 @@ -#!/usr/bin/env python -""" - Zap encoding in requests and inject iframe after body tag in html responses. - Usage: - iframe_injector http://someurl/somefile.html -""" -from libmproxy import controller, proxy -import os -import sys - - -class InjectingMaster(controller.Master): - def __init__(self, server, iframe_url): - controller.Master.__init__(self, server) - self._iframe_url = iframe_url - - def run(self): - try: - return controller.Master.run(self) - except KeyboardInterrupt: - self.shutdown() - - def handle_request(self, msg): - if 'Accept-Encoding' in msg.headers: - msg.headers["Accept-Encoding"] = 'none' - msg.reply() - - def handle_response(self, msg): - if msg.content: - c = msg.replace('<body>', '<body><iframe src="%s" frameborder="0" height="0" width="0"></iframe>' % self._iframe_url) - if c > 0: - print 'Iframe injected!' - msg.reply() - - -def main(argv): - if len(argv) != 2: - print "Usage: %s IFRAME_URL" % argv[0] - sys.exit(1) - iframe_url = argv[1] - config = proxy.ProxyConfig( - cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem") - ) - server = proxy.ProxyServer(config, 8080) - print 'Starting proxy...' - m = InjectingMaster(server, iframe_url) - m.run() - -if __name__ == '__main__': - main(sys.argv) diff --git a/examples/iframe_injector.py b/examples/iframe_injector.py new file mode 100644 index 00000000..7042dbab --- /dev/null +++ b/examples/iframe_injector.py @@ -0,0 +1,18 @@ +# Usage: mitmdump -s "iframe_injector.py url" +# (this script works best with --anticache) +from libmproxy.protocol.http import decoded + + +def start(ctx, argv): + if len(argv) != 2: + raise ValueError('Usage: -s "iframe_injector.py url"') + ctx.iframe_url = argv[1] + + +def handle_response(ctx, flow): + with decoded(flow.response): # Remove content encoding (gzip, ...) + c = flow.response.replace( + '<body>', + '<body><iframe src="%s" frameborder="0" height="0" width="0"></iframe>' % ctx.iframe_url) + if c > 0: + ctx.log("Iframe injected!")
\ No newline at end of file diff --git a/examples/modify_form.py b/examples/modify_form.py index 2d839aed..6d651b19 100644 --- a/examples/modify_form.py +++ b/examples/modify_form.py @@ -1,8 +1,6 @@ -def request(context, flow): +def request(ctx, flow): if "application/x-www-form-urlencoded" in flow.request.headers["content-type"]: - frm = flow.request.get_form_urlencoded() - frm["mitmproxy"] = ["rocks"] - flow.request.set_form_urlencoded(frm) - - + form = flow.request.get_form_urlencoded() + form["mitmproxy"] = ["rocks"] + flow.request.set_form_urlencoded(form)
\ No newline at end of file diff --git a/examples/modify_querystring.py b/examples/modify_querystring.py index b1abcc1e..56fbbb32 100644 --- a/examples/modify_querystring.py +++ b/examples/modify_querystring.py @@ -1,7 +1,6 @@ -def request(context, flow): +def request(ctx, flow): q = flow.request.get_query() if q: q["mitmproxy"] = ["rocks"] - flow.request.set_query(q) - + flow.request.set_query(q)
\ No newline at end of file diff --git a/examples/nonblocking.py b/examples/nonblocking.py index 9a131b32..1396742a 100644 --- a/examples/nonblocking.py +++ b/examples/nonblocking.py @@ -1,8 +1,9 @@ import time from libmproxy.script import concurrent + @concurrent def request(context, flow): print "handle request: %s%s" % (flow.request.host, flow.request.path) time.sleep(5) - print "start request: %s%s" % (flow.request.host, flow.request.path) + print "start request: %s%s" % (flow.request.host, flow.request.path)
\ No newline at end of file diff --git a/examples/redirect_requests.py b/examples/redirect_requests.py index 530da200..c5561839 100644 --- a/examples/redirect_requests.py +++ b/examples/redirect_requests.py @@ -6,13 +6,15 @@ This example shows two ways to redirect flows to other destinations. """ -def request(context, flow): - if flow.request.get_host(hostheader=True).endswith("example.com"): +def request(ctx, flow): + # pretty_host(hostheader=True) takes the Host: header of the request into account, + # which is useful in transparent mode where we usually only have the IP otherwise. + if flow.request.pretty_host(hostheader=True).endswith("example.com"): resp = HTTPResponse( [1, 1], 200, "OK", ODictCaseless([["Content-Type", "text/html"]]), "helloworld") flow.reply(resp) - if flow.request.get_host(hostheader=True).endswith("example.org"): + if flow.request.pretty_host(hostheader=True).endswith("example.org"): flow.request.host = "mitmproxy.org" - flow.request.headers["Host"] = ["mitmproxy.org"] + flow.request.update_host_header() diff --git a/examples/stickycookies b/examples/stickycookies index 17cd6019..2aab31d6 100755 --- a/examples/stickycookies +++ b/examples/stickycookies @@ -5,8 +5,10 @@ implement functionality similar to the "sticky cookies" option. This is at a lower level than the Flow mechanism, so we're dealing directly with request and response objects. """ -from libmproxy import controller, proxy import os +from libmproxy import controller, proxy +from libmproxy.proxy.server import ProxyServer + class StickyMaster(controller.Master): def __init__(self, server): @@ -35,8 +37,8 @@ class StickyMaster(controller.Master): config = proxy.ProxyConfig( - cacert = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem") + ca_file = os.path.expanduser("~/.mitmproxy/mitmproxy-ca.pem") ) -server = proxy.ProxyServer(config, 8080) +server = ProxyServer(config, 8080) m = StickyMaster(server) m.run() diff --git a/examples/stub.py b/examples/stub.py index 0cf67db7..5976dd76 100644 --- a/examples/stub.py +++ b/examples/stub.py @@ -7,14 +7,14 @@ def start(ctx, argv): """ ctx.log("start") -def clientconnect(ctx, client_connect): +def clientconnect(ctx, conn_handler): """ Called when a client initiates a connection to the proxy. Note that a connection can correspond to multiple HTTP requests """ ctx.log("clientconnect") -def serverconnect(ctx, server_connection): +def serverconnect(ctx, conn_handler): """ Called when the proxy initiates a connection to the target server. Note that a connection can correspond to multiple HTTP requests @@ -50,7 +50,7 @@ def error(ctx, flow): """ ctx.log("error") -def clientdisconnect(ctx, client_disconnect): +def clientdisconnect(ctx, conn_handler): """ Called when a client disconnects from the proxy. """ diff --git a/examples/upsidedownternet.py b/examples/upsidedownternet.py index 181a40c2..a52b6d30 100644 --- a/examples/upsidedownternet.py +++ b/examples/upsidedownternet.py @@ -1,7 +1,7 @@ import cStringIO from PIL import Image -def response(context, flow): +def response(ctx, flow): if flow.response.headers["content-type"] == ["image/png"]: s = cStringIO.StringIO(flow.response.content) img = Image.open(s).rotate(180) diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 5cb3dd2a..104b7216 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -177,7 +177,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2): req_timestamp = f.request.timestamp_start, req_is_replay = f.request.is_replay, req_method = f.request.method, - req_url = f.request.get_url(hostheader=hostheader), + req_url = f.request.pretty_url(hostheader=hostheader), err_msg = f.error.msg if f.error else None, resp_code = f.response.code if f.response else None, diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 3c63ac29..014d44c0 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -528,7 +528,9 @@ class FlowView(common.WWrap): def set_url(self, url): request = self.flow.request - if not request.set_url(str(url)): + try: + request.url = str(url) + except ValueError: return "Invalid URL." self.master.refresh_flow(self.flow) @@ -608,7 +610,7 @@ class FlowView(common.WWrap): elif part == "q": self.master.view_grideditor(grideditor.QueryEditor(self.master, conn.get_query().lst, self.set_query, conn)) elif part == "u" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST: - self.master.prompt_edit("URL", conn.get_url(), self.set_url) + self.master.prompt_edit("URL", conn.url, self.set_url) elif part == "m" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST: self.master.prompt_onekey("Method", self.method_options, self.edit_method) elif part == "c" and self.state.view_flow_mode == common.VIEW_FLOW_RESPONSE: diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 8ecd56e7..72ab58a3 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -55,7 +55,7 @@ def str_request(f, showhost): c = f.client_conn.address.host else: c = "[replay]" - r = "%s %s %s"%(c, f.request.method, f.request.get_url(showhost, f)) + r = "%s %s %s"%(c, f.request.method, f.request.pretty_url(showhost)) if f.request.stickycookie: r = "[stickycookie] " + r return r diff --git a/libmproxy/filt.py b/libmproxy/filt.py index 925dbfbb..7d2bd737 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -208,7 +208,7 @@ class FDomain(_Rex): code = "d" help = "Domain" def __call__(self, f): - return bool(re.search(self.expr, f.request.get_host(False, f), re.IGNORECASE)) + return bool(re.search(self.expr, f.request.host, re.IGNORECASE)) class FUrl(_Rex): @@ -222,7 +222,7 @@ class FUrl(_Rex): return klass(*toks) def __call__(self, f): - return re.search(self.expr, f.request.get_url(False, f)) + return re.search(self.expr, f.request.url) class _Int(_Action): diff --git a/libmproxy/flow.py b/libmproxy/flow.py index df72878f..086710bc 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -259,8 +259,8 @@ class StickyCookieState: Returns a (domain, port, path) tuple. """ return ( - m["domain"] or f.request.get_host(False, f), - f.request.get_port(f), + m["domain"] or f.request.host, + f.request.port, m["path"] or "/" ) @@ -278,7 +278,7 @@ class StickyCookieState: c = Cookie.SimpleCookie(str(i)) m = c.values()[0] k = self.ckey(m, f) - if self.domain_match(f.request.get_host(False, f), k[0]): + if self.domain_match(f.request.host, k[0]): self.jar[self.ckey(m, f)] = m def handle_request(self, f): @@ -286,8 +286,8 @@ class StickyCookieState: if f.match(self.flt): for i in self.jar.keys(): match = [ - self.domain_match(f.request.get_host(False, f), i[0]), - f.request.get_port(f) == i[1], + self.domain_match(f.request.host, i[0]), + f.request.port == i[1], f.request.path.startswith(i[2]) ] if all(match): @@ -306,7 +306,7 @@ class StickyAuthState: self.hosts = {} def handle_request(self, f): - host = f.request.get_host(False, f) + host = f.request.host if "authorization" in f.request.headers: self.hosts[host] = f.request.headers["authorization"] elif f.match(self.flt): @@ -612,6 +612,7 @@ class FlowMaster(controller.Master): if f.request: self.handle_request(f) if f.response: + self.handle_responseheaders(f) self.handle_response(f) if f.error: self.handle_error(f) @@ -668,7 +669,7 @@ class FlowMaster(controller.Master): self.masterq, self.should_exit ) - rt.start() # pragma: no cover + rt.start() # pragma: no cover if block: rt.join() diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 9699c78a..c67cb471 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -26,16 +26,22 @@ def get_line(fp): return line -def send_connect_request(conn, host, port): +def send_connect_request(conn, host, port, update_state=True): upstream_request = HTTPRequest("authority", "CONNECT", None, host, port, None, (1, 1), ODictCaseless(), "") - conn.send(upstream_request._assemble()) + conn.send(upstream_request.assemble()) resp = HTTPResponse.from_stream(conn.rfile, upstream_request.method) if resp.code != 200: raise proxy.ProxyError(resp.code, "Cannot establish SSL " + "connection with upstream proxy: \r\n" + - str(resp._assemble())) + str(resp.assemble())) + if update_state: + conn.state.append(("http", { + "state": "connect", + "host": host, + "port": port} + )) return resp @@ -67,6 +73,9 @@ class decoded(object): class HTTPMessage(stateobject.SimpleStateObject): + """ + Base class for HTTPRequest and HTTPResponse + """ def __init__(self, httpversion, headers, content, timestamp_start=None, timestamp_end=None): self.httpversion = httpversion @@ -151,36 +160,29 @@ class HTTPMessage(stateobject.SimpleStateObject): c += self.headers.replace(pattern, repl, *args, **kwargs) return c - @classmethod - def from_stream(cls, rfile, include_body=True, body_size_limit=None): - """ - Parse an HTTP message from a file stream - """ - raise NotImplementedError # pragma: nocover - def _assemble_first_line(self): """ Returns the assembled request/response line """ - raise NotImplementedError # pragma: nocover + raise NotImplementedError() # pragma: nocover def _assemble_headers(self): """ Returns the assembled headers """ - raise NotImplementedError # pragma: nocover + raise NotImplementedError() # pragma: nocover def _assemble_head(self): """ Returns the assembled request/response line plus headers """ - raise NotImplementedError # pragma: nocover + raise NotImplementedError() # pragma: nocover - def _assemble(self): + def assemble(self): """ Returns the assembled request/response """ - raise NotImplementedError # pragma: nocover + raise NotImplementedError() # pragma: nocover class HTTPRequest(HTTPMessage): @@ -189,7 +191,17 @@ class HTTPRequest(HTTPMessage): Exposes the following attributes: - flow: Flow object the request belongs to + method: HTTP method + + scheme: URL scheme (http/https) (absolute-form only) + + host: Host portion of the URL (absolute-form and authority-form only) + + port: Destination port (absolute-form and authority-form only) + + path: Path portion of the URL (not present in authority-form) + + httpversion: HTTP version tuple, e.g. (1,1) headers: ODictCaseless object @@ -205,18 +217,6 @@ class HTTPRequest(HTTPMessage): form_out: The request form which mitmproxy has send out to the destination - method: HTTP method - - scheme: URL scheme (http/https) (absolute-form only) - - host: Host portion of the URL (absolute-form and authority-form only) - - port: Destination port (absolute-form and authority-form only) - - path: Path portion of the URL (not present in authority-form) - - httpversion: HTTP version tuple - timestamp_start: Timestamp indicating when request transmission started timestamp_end: Timestamp indicating when request transmission ended @@ -358,7 +358,7 @@ class HTTPRequest(HTTPMessage): def _assemble_head(self, form=None): return "%s\r\n%s\r\n" % (self._assemble_first_line(form), self._assemble_headers()) - def _assemble(self, form=None): + def assemble(self, form=None): """ Assembles the request for transmission to the server. We make some modifications to make sure interception works properly. @@ -405,6 +405,12 @@ class HTTPRequest(HTTPMessage): e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0] )] + def update_host_header(self): + """ + Update the host header to reflect the current target. + """ + self.headers["Host"] = [self.host] + def get_form_urlencoded(self): """ Retrieves the URL-encoded form data, returning an ODict object. @@ -426,16 +432,16 @@ class HTTPRequest(HTTPMessage): self.headers["Content-Type"] = [HDR_FORM_URLENCODED] self.content = utils.urlencode(odict.lst) - def get_path_components(self, f): + def get_path_components(self): """ Returns the path components of the URL as a list of strings. Components are unquoted. """ - _, _, path, _, _, _ = urlparse.urlparse(self.get_url(False, f)) + _, _, path, _, _, _ = urlparse.urlparse(self.url) return [urllib.unquote(i) for i in path.split("/") if i] - def set_path_components(self, lst, f): + def set_path_components(self, lst): """ Takes a list of strings, and sets the path component of the URL. @@ -443,32 +449,32 @@ class HTTPRequest(HTTPMessage): """ lst = [urllib.quote(i, safe="") for i in lst] path = "/" + "/".join(lst) - scheme, netloc, _, params, query, fragment = urlparse.urlparse(self.get_url(False, f)) - self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment]), f) + scheme, netloc, _, params, query, fragment = urlparse.urlparse(self.url) + self.url = urlparse.urlunparse([scheme, netloc, path, params, query, fragment]) - def get_query(self, f): + def get_query(self): """ Gets the request query string. Returns an ODict object. """ - _, _, _, _, query, _ = urlparse.urlparse(self.get_url(False, f)) + _, _, _, _, query, _ = urlparse.urlparse(self.url) if query: return ODict(utils.urldecode(query)) return ODict([]) - def set_query(self, odict, f): + def set_query(self, odict): """ Takes an ODict object, and sets the request query string. """ - scheme, netloc, path, params, _, fragment = urlparse.urlparse(self.get_url(False, f)) + scheme, netloc, path, params, _, fragment = urlparse.urlparse(self.url) query = utils.urlencode(odict.lst) - self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment]), f) + self.url = urlparse.urlunparse([scheme, netloc, path, params, query, fragment]) - def get_host(self, hostheader, flow): + def pretty_host(self, hostheader): """ Heuristic to get the host of the request. - Note that get_host() does not always return the TCP destination of the request, - e.g. on a transparently intercepted request to an unrelated HTTP proxy. + Note that pretty_host() does not always return the TCP destination of the request, + e.g. if an upstream proxy is in place If hostheader is set to True, the Host: header will be used as additional (and preferred) data source. This is handy in transparent mode, where only the ip of the destination is known, but not the @@ -478,54 +484,27 @@ class HTTPRequest(HTTPMessage): if hostheader: host = self.headers.get_first("host") if not host: - if self.host: - host = self.host - else: - for s in flow.server_conn.state: - if s[0] == "http" and s[1]["state"] == "connect": - host = s[1]["host"] - break - if not host: - host = flow.server_conn.address.host + host = self.host host = host.encode("idna") return host - def get_scheme(self, flow): - """ - Returns the request port, either from the request itself or from the flow's server connection - """ - if self.scheme: - return self.scheme - if self.form_out == "authority": # On SSLed connections, the original CONNECT request is still unencrypted. - return "http" - return "https" if flow.server_conn.ssl_established else "http" - - def get_port(self, flow): - """ - Returns the request port, either from the request itself or from the flow's server connection - """ - if self.port: - return self.port - for s in flow.server_conn.state: - if s[0] == "http" and s[1].get("state") == "connect": - return s[1]["port"] - return flow.server_conn.address.port + def pretty_url(self, hostheader): + if self.form_out == "authority": # upstream proxy mode + return "%s:%s" % (self.pretty_host(hostheader), self.port) + return utils.unparse_url(self.scheme, + self.pretty_host(hostheader), + self.port, + self.path).encode('ascii') - def get_url(self, hostheader, flow): + @property + def url(self): """ Returns a URL string, constructed from the Request's URL components. - - If hostheader is True, we use the value specified in the request - Host header to construct the URL. """ - if self.form_out == "authority": # upstream proxy mode - return "%s:%s" % (self.get_host(hostheader, flow), self.get_port(flow)) - return utils.unparse_url(self.get_scheme(flow), - self.get_host(hostheader, flow), - self.get_port(flow), - self.path).encode('ascii') + return self.pretty_url(False) - def set_url(self, url, flow): + @url.setter + def url(self, url): """ Parses a URL specification, and updates the Request's information accordingly. @@ -534,30 +513,8 @@ class HTTPRequest(HTTPMessage): """ parts = http.parse_url(url) if not parts: - return False - scheme, host, port, path = parts - is_ssl = (True if scheme == "https" else False) - - self.path = path - - if host != self.get_host(False, flow) or port != self.get_port(flow): - if flow.live: - flow.live.change_server((host, port), ssl=is_ssl) - else: - # There's not live server connection, we're just changing the attributes here. - flow.server_conn = ServerConnection((host, port), - proxy.AddressPriority.MANUALLY_CHANGED) - flow.server_conn.ssl_established = is_ssl - - # If this is an absolute request, replace the attributes on the request object as well. - if self.host: - self.host = host - if self.port: - self.port = port - if self.scheme: - self.scheme = scheme - - return True + raise ValueError("Invalid URL: %s" % url) + self.scheme, self.host, self.port, self.path = parts def get_cookies(self): cookie_headers = self.headers.get("cookie") @@ -590,7 +547,7 @@ class HTTPResponse(HTTPMessage): Exposes the following attributes: - flow: Flow object the request belongs to + httpversion: HTTP version tuple, e.g. (1,1) code: HTTP response code @@ -602,8 +559,6 @@ class HTTPResponse(HTTPMessage): is content associated, but not present. CONTENT_MISSING evaluates to False to make checking for the presence of content natural. - httpversion: HTTP version tuple - timestamp_start: Timestamp indicating when request transmission started timestamp_end: Timestamp indicating when request transmission ended @@ -682,7 +637,8 @@ class HTTPResponse(HTTPMessage): if self.content: headers["Content-Length"] = [str(len(self.content))] - elif not preserve_transfer_encoding and 'Transfer-Encoding' in self.headers: # add content-length for chuncked transfer-encoding with no content + # add content-length for chuncked transfer-encoding with no content + elif not preserve_transfer_encoding and 'Transfer-Encoding' in self.headers: headers["Content-Length"] = ["0"] return str(headers) @@ -691,7 +647,7 @@ class HTTPResponse(HTTPMessage): return '%s\r\n%s\r\n' % ( self._assemble_first_line(), self._assemble_headers(preserve_transfer_encoding=preserve_transfer_encoding)) - def _assemble(self): + def assemble(self): """ Assembles the response for transmission to the client. We make some modifications to make sure interception works properly. @@ -774,12 +730,14 @@ class HTTPResponse(HTTPMessage): class HTTPFlow(Flow): """ - A Flow is a collection of objects representing a single HTTP + A HTTPFlow is a collection of objects representing a single HTTP transaction. The main attributes are: request: HTTPRequest object response: HTTPResponse object error: Error object + server_conn: ServerConnection object + client_conn: ClientConnection object Note that it's possible for a Flow to have both a response and an error object. This might happen, for instance, when a response was received @@ -816,7 +774,7 @@ class HTTPFlow(Flow): s = "<HTTPFlow" for a in ("request", "response", "error", "client_conn", "server_conn"): if getattr(self, a, False): - s += "\r\n %s = {flow.%s}" % (a,a) + s += "\r\n %s = {flow.%s}" % (a, a) s += ">" return s.format(flow=self) @@ -895,6 +853,10 @@ class HttpAuthenticationError(Exception): class HTTPHandler(ProtocolHandler): + """ + HTTPHandler implements mitmproxys understanding of the HTTP protocol. + + """ def __init__(self, c): super(HTTPHandler, self).__init__(c) self.expected_form_in = c.config.http_form_in @@ -905,19 +867,21 @@ class HTTPHandler(ProtocolHandler): while self.handle_flow(): pass - def get_response_from_server(self, request, include_body=True): + def get_response_from_server(self, flow): self.c.establish_server_connection() - request_raw = request._assemble() + request_raw = flow.request.assemble() - for i in range(2): + for attempt in (0, 1): try: self.c.server_conn.send(request_raw) - res = HTTPResponse.from_stream(self.c.server_conn.rfile, request.method, - body_size_limit=self.c.config.body_size_limit, include_body=include_body) - return res + # Only get the headers at first... + flow.response = HTTPResponse.from_stream(self.c.server_conn.rfile, flow.request.method, + body_size_limit=self.c.config.body_size_limit, + include_body=False) + break except (tcp.NetLibDisconnect, http.HttpErrorConnClosed), v: self.c.log("error in server communication: %s" % repr(v), level="debug") - if i < 1: + if attempt == 0: # In any case, we try to reconnect at least once. # This is necessary because it might be possible that we already initiated an upstream connection # after clientconnect that has already been expired, e.g consider the following event log: @@ -931,13 +895,24 @@ class HTTPHandler(ProtocolHandler): else: raise + # call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True + self.c.channel.ask("responseheaders", flow) + + # now get the rest of the request body, if body still needs to be read but not streaming this response + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = http.read_http_body(self.c.server_conn.rfile, flow.response.headers, + self.c.config.body_size_limit, + flow.request.method, flow.response.code, False) + def handle_flow(self): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live) try: try: req = HTTPRequest.from_stream(self.c.client_conn.rfile, body_size_limit=self.c.config.body_size_limit) - except tcp.NetLibDisconnect: # specifically ignore disconnects that happen before/between requests. + except tcp.NetLibDisconnect: # don't throw an error for disconnects that happen before/between requests. return False self.c.log("request", "debug", [req._assemble_first_line(req.form_in)]) ret = self.process_request(flow, req) @@ -951,8 +926,7 @@ class HTTPHandler(ProtocolHandler): # sent through to the Master. flow.request = req request_reply = self.c.channel.ask("request", flow) - self.determine_server_address(flow, flow.request) # The inline script may have changed request.host - flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow + self.process_server_address(flow) # The inline script may have changed request.host if request_reply is None or request_reply == KILL: return False @@ -960,20 +934,7 @@ class HTTPHandler(ProtocolHandler): if isinstance(request_reply, HTTPResponse): flow.response = request_reply else: - - # read initially in "stream" mode, so we can get the headers separately - flow.response = self.get_response_from_server(flow.request, include_body=False) - - # call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True - self.c.channel.ask("responseheaders", flow) - - # now get the rest of the request body, if body still needs to be read but not streaming this response - if flow.response.stream: - flow.response.content = CONTENT_MISSING - else: - flow.response.content = http.read_http_body(self.c.server_conn.rfile, flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, flow.response.code, False) + self.get_response_from_server(flow) # no further manipulation of self.c.server_conn beyond this point # we can safely set it as the final attribute value here. @@ -984,72 +945,39 @@ class HTTPHandler(ProtocolHandler): if response_reply is None or response_reply == KILL: return False - if not flow.response.stream: - # no streaming: - # we already received the full response from the server and can send it to the client straight away. - self.c.client_conn.send(flow.response._assemble()) - else: - # streaming: - # First send the body and then transfer the response incrementally: - h = flow.response._assemble_head(preserve_transfer_encoding=True) - self.c.client_conn.send(h) - for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, - flow.response.headers, - self.c.config.body_size_limit, flow.request.method, - flow.response.code, False, 4096): - for part in chunk: - self.c.client_conn.wfile.write(part) - self.c.client_conn.wfile.flush() - flow.response.timestamp_end = utils.timestamp() - - flow.timestamp_end = utils.timestamp() + self.send_response_to_client(flow) - close_connection = ( - http.connection_close(flow.request.httpversion, flow.request.headers) or - http.connection_close(flow.response.httpversion, flow.response.headers) or - http.expected_http_body_size(flow.response.headers, False, flow.request.method, - flow.response.code) == -1) - if close_connection: - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313: - # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header - pass - else: - return False + if self.check_close_connection(flow): + return False # We sent a CONNECT request to an upstream proxy. if flow.request.form_in == "authority" and flow.response.code == 200: - # TODO: Eventually add headers (space/usefulness tradeoff) - # Make sure to add state info before the actual upgrade happens. - # During the upgrade, we may receive an SNI indication from the client, + # TODO: Possibly add headers (memory consumption/usefulness tradeoff) + # Make sure to add state info before the actual processing of the CONNECT request happens. + # During an SSL upgrade, we may receive an SNI indication from the client, # which resets the upstream connection. If this is the case, we must # already re-issue the CONNECT request at this point. self.c.server_conn.state.append(("http", {"state": "connect", "host": flow.request.host, "port": flow.request.port})) - - if self.c.check_ignore_address((flow.request.host, flow.request.port)): - self.c.log("Ignore host: %s:%s" % self.c.server_conn.address(), "info") - TCPHandler(self.c).handle_messages() + if not self.process_connect_request((flow.request.host, flow.request.port)): return False - else: - if flow.request.port in self.c.config.ssl_ports: - self.ssl_upgrade() - self.skip_authentication = True # If the user has changed the target server on this connection, # restore the original target server flow.live.restore_server() - flow.live = None - return True + return True # Next flow please. except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: self.handle_error(e, flow) + finally: + flow.timestamp_end = utils.timestamp() + flow.live = None # Connection is not live anymore. return False def handle_server_reconnect(self, state): if state["state"] == "connect": - send_connect_request(self.c.server_conn, state["host"], state["port"]) + send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False) else: # pragma: nocover raise RuntimeError("Unknown State: %s" % state["state"]) @@ -1093,16 +1021,6 @@ class HTTPHandler(ProtocolHandler): self.c.client_conn.wfile.write(html_content) self.c.client_conn.wfile.flush() - def ssl_upgrade(self): - """ - Upgrade the connection to SSL after an authority (CONNECT) request has been made. - """ - self.c.log("Received CONNECT request. Upgrading to SSL...", "debug") - self.expected_form_in = "relative" - self.expected_form_out = "relative" - self.c.establish_ssl(server=True, client=True) - self.c.log("Upgrade to SSL completed.", "debug") - def process_request(self, flow, request): """ @returns: @@ -1115,14 +1033,30 @@ class HTTPHandler(ProtocolHandler): if not self.skip_authentication: self.authenticate(request) + # Determine .scheme, .host and .port attributes + # For absolute-form requests, they are directly given in the request. + # For authority-form requests, we only need to determine the request scheme. + # For relative-form requests, we need to determine host and port as well. + if not request.scheme: + request.scheme = "https" if flow.server_conn and flow.server_conn.ssl_established else "http" + if not request.host: + # Host/Port Complication: In upstream mode, use the server we CONNECTed to, + # not the upstream proxy. + if flow.server_conn: + for s in flow.server_conn.state: + if s[0] == "http" and s[1]["state"] == "connect": + request.host, request.port = s[1]["host"], s[1]["port"] + if not request.host and flow.server_conn: + request.host, request.port = flow.server_conn.address.host, flow.server_conn.address.port + + # Now we can process the request. if request.form_in == "authority": if self.c.client_conn.ssl_established: raise http.HttpError(400, "Must not CONNECT on already encrypted connection") if self.expected_form_in == "absolute": - if not self.c.config.get_upstream_server: - self.c.set_server_address((request.host, request.port), - proxy.AddressPriority.FROM_PROTOCOL) + if not self.c.config.get_upstream_server: # Regular mode + self.c.set_server_address((request.host, request.port)) flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow self.c.establish_server_connection() self.c.client_conn.send( @@ -1131,34 +1065,123 @@ class HTTPHandler(ProtocolHandler): ('Proxy-agent: %s\r\n' % self.c.server_version) + '\r\n' ) - - if self.c.check_ignore_address(self.c.server_conn.address): - self.c.log("Ignore host: %s:%s" % self.c.server_conn.address(), "info") - TCPHandler(self.c).handle_messages() - return False - else: - if self.c.server_conn.address.port in self.c.config.ssl_ports: - self.ssl_upgrade() - self.skip_authentication = True - return True - else: + return self.process_connect_request(self.c.server_conn.address) + else: # upstream proxy mode return None + else: + pass # CONNECT should never occur if we don't expect absolute-form requests + elif request.form_in == self.expected_form_in: + + request.form_out = self.expected_form_out + if request.form_in == "absolute": if request.scheme != "http": raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme) - self.determine_server_address(flow, request) - request.form_out = self.expected_form_out + if request.form_out == "relative": + self.c.set_server_address((request.host, request.port)) + flow.server_conn = self.c.server_conn + return None raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" % (self.expected_form_in, request.form_in)) - def determine_server_address(self, flow, request): - if request.form_in == "absolute": - self.c.set_server_address((request.host, request.port), - proxy.AddressPriority.FROM_PROTOCOL) - flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow + def process_server_address(self, flow): + # Depending on the proxy mode, server handling is entirely different + # We provide a mostly unified API to the user, which needs to be unfiddled here + # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 ) + address = netlib.tcp.Address((flow.request.host, flow.request.port)) + + ssl = (flow.request.scheme == "https") + + if self.c.config.http_form_in == self.c.config.http_form_out == "absolute": # Upstream Proxy mode + + # The connection to the upstream proxy may have a state we may need to take into account. + connected_to = None + for s in flow.server_conn.state: + if s[0] == "http" and s[1]["state"] == "connect": + connected_to = tcp.Address((s[1]["host"], s[1]["port"])) + + # We need to reconnect if the current flow either requires a (possibly impossible) + # change to the connection state, e.g. the host has changed but we already CONNECTed somewhere else. + needs_server_change = ( + ssl != self.c.server_conn.ssl_established + or + (connected_to and address != connected_to) # HTTP proxying is "stateless", CONNECT isn't. + ) + + if needs_server_change: + # force create new connection to the proxy server to reset state + self.live.change_server(self.c.server_conn.address, force=True) + if ssl: + send_connect_request(self.c.server_conn, address.host, address.port) + self.c.establish_ssl(server=True) + else: + # If we're not in upstream mode, we just want to update the host and possibly establish TLS. + self.live.change_server(address, ssl=ssl) # this is a no op if the addresses match. + + flow.server_conn = self.c.server_conn + + def send_response_to_client(self, flow): + if not flow.response.stream: + # no streaming: + # we already received the full response from the server and can send it to the client straight away. + self.c.client_conn.send(flow.response.assemble()) + else: + # streaming: + # First send the body and then transfer the response incrementally: + h = flow.response._assemble_head(preserve_transfer_encoding=True) + self.c.client_conn.send(h) + for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, + flow.response.headers, + self.c.config.body_size_limit, flow.request.method, + flow.response.code, False, 4096): + for part in chunk: + self.c.client_conn.wfile.write(part) + self.c.client_conn.wfile.flush() + flow.response.timestamp_end = utils.timestamp() + + def check_close_connection(self, flow): + """ + Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so. + """ + close_connection = ( + http.connection_close(flow.request.httpversion, flow.request.headers) or + http.connection_close(flow.response.httpversion, flow.response.headers) or + http.expected_http_body_size(flow.response.headers, False, flow.request.method, + flow.response.code) == -1) + if close_connection: + if flow.request.form_in == "authority" and flow.response.code == 200: + # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313: + # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header + pass + else: + return True + return False + + def process_connect_request(self, address): + """ + Process a CONNECT request. + Returns True if the CONNECT request has been processed successfully. + Returns False, if the connection should be closed immediately. + """ + address = tcp.Address.wrap(address) + if self.c.check_ignore_address(address): + self.c.log("Ignore host: %s:%s" % address(), "info") + TCPHandler(self.c).handle_messages() + return False + else: + self.expected_form_in = "relative" + self.expected_form_out = "relative" + self.skip_authentication = True + + if address.port in self.c.config.ssl_ports: + self.c.log("Received CONNECT request to SSL port. Upgrading to SSL...", "debug") + self.c.establish_ssl(server=True, client=True) + self.c.log("Upgrade to SSL completed.", "debug") + + return True def authenticate(self, request): if self.c.config.authenticator: @@ -1178,33 +1201,29 @@ class RequestReplayThread(threading.Thread): threading.Thread.__init__(self) def run(self): + r = self.flow.request + form_out_backup = r.form_out try: - r = self.flow.request - form_out_backup = r.form_out - - r.form_out = self.config.http_form_out - server_address, server_ssl = False, False - if self.config.get_upstream_server: - try: - # this will fail in transparent mode - upstream_info = self.config.get_upstream_server(self.flow.client_conn) - server_ssl = upstream_info[1] - server_address = upstream_info[2:] - except proxy.ProxyError: - pass - if not server_address: - server_address = (r.get_host(False, self.flow), r.get_port(self.flow)) - - server = ServerConnection(server_address, None) - server.connect() - - if server_ssl or r.get_scheme(self.flow) == "https": - if self.config.http_form_out == "absolute": # form_out == absolute -> forward mode -> send CONNECT - send_connect_request(server, r.get_host(), r.get_port()) + # In all modes, we directly connect to the server displayed + if self.config.http_form_out == "absolute": # form_out == absolute -> forward mode + server_address = self.config.get_upstream_server(self.flow.client_conn)[2:] + server = ServerConnection(server_address) + server.connect() + if r.scheme == "https": + send_connect_request(server, r.host, r.port) + server.establish_ssl(self.config.clientcerts, sni=r.host) r.form_out = "relative" - server.establish_ssl(self.config.clientcerts, - self.flow.server_conn.sni) - server.send(r._assemble()) + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + if r.scheme == "https": + server.establish_ssl(self.config.clientcerts, sni=r.host) + r.form_out = "relative" + + server.send(r.assemble()) self.flow.response = HTTPResponse.from_stream(server.rfile, r.method, body_size_limit=self.config.body_size_limit) self.channel.ask("response", self.flow) diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py index a84b4061..ecad9d9e 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/protocol/primitives.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import copy import netlib.tcp from .. import stateobject, utils, version -from ..proxy.primitives import AddressPriority from ..proxy.connection import ClientConnection, ServerConnection @@ -13,9 +12,9 @@ class Error(stateobject.SimpleStateObject): """ An Error. - This is distinct from an HTTP error response (say, a code 500), which - is represented by a normal Response object. This class is responsible - for indicating errors that fall outside of normal HTTP communications, + This is distinct from an protocol error response (say, a HTTP code 500), which + is represented by a normal HTTPResponse object. This class is responsible + for indicating errors that fall outside of normal protocol communications, like interrupted connections, timeouts, protocol errors. Exposes the following attributes: @@ -53,13 +52,17 @@ class Error(stateobject.SimpleStateObject): class Flow(stateobject.SimpleStateObject): + """ + A Flow is a collection of objects representing a single transaction. + This class is usually subclassed for each protocol, e.g. HTTPFlow. + """ def __init__(self, conntype, client_conn, server_conn, live=None): self.conntype = conntype self.client_conn = client_conn """@type: ClientConnection""" self.server_conn = server_conn """@type: ServerConnection""" - self.live = live # Used by flow.request.set_url to change the server address + self.live = live """@type: LiveConnection""" self.error = None @@ -118,6 +121,10 @@ class Flow(stateobject.SimpleStateObject): class ProtocolHandler(object): + """ + A ProtocolHandler implements an application-layer protocol, e.g. HTTP. + See: libmproxy.protocol.http.HTTPHandler + """ def __init__(self, c): self.c = c """@type: libmproxy.proxy.server.ConnectionHandler""" @@ -149,48 +156,53 @@ class ProtocolHandler(object): class LiveConnection(object): """ - This facade allows protocol handlers to interface with a live connection, - without requiring the expose the ConnectionHandler. + This facade allows interested parties (FlowMaster, inline scripts) to interface with a live connection, + without requiring to expose the internals of the ConnectionHandler. """ def __init__(self, c): - self._c = c + self.c = c """@type: libmproxy.proxy.server.ConnectionHandler""" + self._backup_server_conn = None + """@type: libmproxy.proxy.connection.ServerConnection""" - def change_server(self, address, ssl, persistent_change=False): + def change_server(self, address, ssl=False, force=False, persistent_change=False): address = netlib.tcp.Address.wrap(address) - if address != self._c.server_conn.address: + if force or address != self.c.server_conn.address or ssl != self.c.server_conn.ssl_established: - self._c.log("Change server connection: %s:%s -> %s:%s" % ( - self._c.server_conn.address.host, - self._c.server_conn.address.port, + self.c.log("Change server connection: %s:%s -> %s:%s [persistent: %s]" % ( + self.c.server_conn.address.host, + self.c.server_conn.address.port, address.host, - address.port + address.port, + persistent_change ), "debug") - if not hasattr(self, "_backup_server_conn"): - self._backup_server_conn = self._c.server_conn - self._c.server_conn = None + if not self._backup_server_conn: + self._backup_server_conn = self.c.server_conn + self.c.server_conn = None else: # This is at least the second temporary change. We can kill the current connection. - self._c.del_server_connection() + self.c.del_server_connection() - self._c.set_server_address(address, AddressPriority.MANUALLY_CHANGED) - self._c.establish_server_connection(ask=False) + self.c.set_server_address(address) + self.c.establish_server_connection(ask=False) if ssl: - self._c.establish_ssl(server=True) - if hasattr(self, "_backup_server_conn") and persistent_change: - del self._backup_server_conn + self.c.establish_ssl(server=True) + if persistent_change: + self._backup_server_conn = None def restore_server(self): - if not hasattr(self, "_backup_server_conn"): + # TODO: Similar to _backup_server_conn, introduce _cache_server_conn, which keeps the changed connection open + # This may be beneficial if a user is rewriting all requests from http to https or similar. + if not self._backup_server_conn: return - self._c.log("Restore original server connection: %s:%s -> %s:%s" % ( - self._c.server_conn.address.host, - self._c.server_conn.address.port, + self.c.log("Restore original server connection: %s:%s -> %s:%s" % ( + self.c.server_conn.address.host, + self.c.server_conn.address.port, self._backup_server_conn.address.host, self._backup_server_conn.address.port ), "debug") - self._c.del_server_connection() - self._c.server_conn = self._backup_server_conn - del self._backup_server_conn
\ No newline at end of file + self.c.del_server_connection() + self.c.server_conn = self._backup_server_conn + self._backup_server_conn = None
\ No newline at end of file diff --git a/libmproxy/protocol/tcp.py b/libmproxy/protocol/tcp.py index 57a48ab9..990c502a 100644 --- a/libmproxy/protocol/tcp.py +++ b/libmproxy/protocol/tcp.py @@ -59,11 +59,11 @@ class TCPHandler(ProtocolHandler): # if one of the peers is over SSL, we need to send bytes/strings if not src.ssl_established: # only ssl to dst, i.e. we revc'd into buf but need bytes/string now. contents = buf[:size].tobytes() - # self.c.log("%s %s\r\n%s" % (direction, dst_str, cleanBin(contents)), "debug") + self.c.log("%s %s\r\n%s" % (direction, dst_str, cleanBin(contents)), "debug") dst.connection.send(contents) else: # socket.socket.send supports raw bytearrays/memoryviews - # self.c.log("%s %s\r\n%s" % (direction, dst_str, cleanBin(buf.tobytes())), "debug") + self.c.log("%s %s\r\n%s" % (direction, dst_str, cleanBin(buf.tobytes())), "debug") dst.connection.send(buf[:size]) except socket.error as e: self.c.log("TCP connection closed unexpectedly.", "debug") diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index f5d6a2d0..e4c20030 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1 +1,2 @@ -from .primitives import *
\ No newline at end of file +from .primitives import * +from .config import ProxyConfig diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 6d4c078b..ea815c69 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,8 +1,8 @@ from __future__ import absolute_import import os -from .. import utils, platform import re from netlib import http_auth, certutils +from .. import utils, platform from .primitives import ConstUpstreamServerResolver, TransparentUpstreamServerResolver TRANSPARENT_SSL_PORTS = [443, 8443] @@ -11,7 +11,7 @@ CONF_DIR = "~/.mitmproxy" class ProxyConfig: - def __init__(self, confdir=CONF_DIR, clientcerts=None, + def __init__(self, confdir=CONF_DIR, ca_file=None, clientcerts=None, no_upstream_cert=False, body_size_limit=None, mode=None, upstream_server=None, http_form_in=None, http_form_out=None, authenticator=None, ignore=[], @@ -44,7 +44,7 @@ class ProxyConfig: self.ignore = [re.compile(i, re.IGNORECASE) for i in ignore] self.authenticator = authenticator self.confdir = os.path.expanduser(confdir) - self.ca_file = os.path.join(self.confdir, CONF_BASENAME + "-ca.pem") + self.ca_file = ca_file or os.path.join(self.confdir, CONF_BASENAME + "-ca.pem") self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) for spec, cert in certs: self.certstore.add_cert_file(spec, cert) diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index d99ffa9b..de8e20d8 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -72,13 +72,10 @@ class ClientConnection(tcp.BaseHandler, stateobject.SimpleStateObject): class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject): - def __init__(self, address, priority): + def __init__(self, address): tcp.TCPClient.__init__(self, address) - self.priority = priority self.state = [] # a list containing (conntype, state) tuples - self.peername = None - self.sockname = None self.timestamp_start = None self.timestamp_end = None self.timestamp_tcp_setup = None @@ -99,8 +96,6 @@ class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject): _stateobject_attributes = dict( state=list, - peername=tuple, - sockname=tuple, timestamp_start=float, timestamp_end=float, timestamp_tcp_setup=float, @@ -115,9 +110,10 @@ class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject): def _get_state(self): d = super(ServerConnection, self)._get_state() d.update( - address={"address": self.address(), "use_ipv6": self.address.use_ipv6}, - source_address= {"address": self.source_address(), - "use_ipv6": self.source_address.use_ipv6} if self.source_address else None, + address={"address": self.address(), + "use_ipv6": self.address.use_ipv6}, + source_address= ({"address": self.source_address(), + "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), cert=self.cert.to_pem() if self.cert else None ) return d @@ -131,7 +127,7 @@ class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject): @classmethod def _from_state(cls, state): - f = cls(tuple(), None) + f = cls(tuple()) f._load_state(state) return f @@ -141,8 +137,6 @@ class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject): def connect(self): self.timestamp_start = utils.timestamp() tcp.TCPClient.connect(self) - self.peername = self.connection.getpeername() - self.sockname = self.connection.getsockname() self.timestamp_tcp_setup = utils.timestamp() def send(self, message): diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index e09f23e4..8c674381 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -45,19 +45,6 @@ class TransparentUpstreamServerResolver(UpstreamServerResolver): return [ssl, ssl] + list(dst) -class AddressPriority(object): - """ - Enum that signifies the priority of the given address when choosing the destination host. - Higher is better (None < i) - """ - MANUALLY_CHANGED = 3 - """user changed the target address in the ui""" - FROM_SETTINGS = 2 - """upstream server from arguments (reverse proxy, upstream proxy or from transparent resolver)""" - FROM_PROTOCOL = 1 - """derived from protocol (e.g. absolute-form http requests)""" - - class Log: def __init__(self, msg, level="info"): self.msg = msg diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 092eae54..aacc908e 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -1,11 +1,10 @@ from __future__ import absolute_import -import re import socket from OpenSSL import SSL from netlib import tcp -from .primitives import ProxyServerError, Log, ProxyError, AddressPriority +from .primitives import ProxyServerError, Log, ProxyError from .connection import ClientConnection, ServerConnection from ..protocol.handle import protocol_handler from .. import version @@ -76,7 +75,7 @@ class ConnectionHandler: client_ssl, server_ssl = False, False if self.config.get_upstream_server: upstream_info = self.config.get_upstream_server(self.client_conn.connection) - self.set_server_address(upstream_info[2:], AddressPriority.FROM_SETTINGS) + self.set_server_address(upstream_info[2:]) client_ssl, server_ssl = upstream_info[:2] if self.check_ignore_address(self.server_conn.address): self.log("Ignore host: %s:%s" % self.server_conn.address(), "info") @@ -129,27 +128,22 @@ class ConnectionHandler: else: return False - def set_server_address(self, address, priority): + def set_server_address(self, address): """ Sets a new server address with the given priority. Does not re-establish either connection or SSL handshake. """ address = tcp.Address.wrap(address) - if self.server_conn: - if self.server_conn.priority > priority: - self.log("Attempt to change server address, " - "but priority is too low (is: %s, got: %s)" % ( - self.server_conn.priority, priority), "debug") - return - if self.server_conn.address == address: - self.server_conn.priority = priority # Possibly increase priority - return + # Don't reconnect to the same destination. + if self.server_conn and self.server_conn.address == address: + return + if self.server_conn: self.del_server_connection() self.log("Set new server address: %s:%s" % (address.host, address.port), "debug") - self.server_conn = ServerConnection(address, priority) + self.server_conn = ServerConnection(address) def establish_server_connection(self, ask=True): """ @@ -212,12 +206,11 @@ class ConnectionHandler: def server_reconnect(self): address = self.server_conn.address had_ssl = self.server_conn.ssl_established - priority = self.server_conn.priority state = self.server_conn.state sni = self.sni self.log("(server reconnect follows)", "debug") self.del_server_connection() - self.set_server_address(address, priority) + self.set_server_address(address) self.establish_server_connection() for s in state: diff --git a/libmproxy/stateobject.py b/libmproxy/stateobject.py index 3437b90e..6fb73c24 100644 --- a/libmproxy/stateobject.py +++ b/libmproxy/stateobject.py @@ -21,6 +21,9 @@ class StateObject(object): except AttributeError: # we may compare with something that's not a StateObject return False + def __ne__(self, other): + return not self.__eq__(other) + class SimpleStateObject(StateObject): """ diff --git a/test/test_examples.py b/test/test_examples.py index d18b5862..d557080e 100644 --- a/test/test_examples.py +++ b/test/test_examples.py @@ -12,6 +12,8 @@ def test_load_scripts(): tmaster = tservers.TestMaster(config.ProxyConfig()) for f in scripts: + if "iframe_injector" in f: + f += " foo" # one argument required if "modify_response_body" in f: f += " foo bar" # two arguments required script.Script(f, tmaster) # Loads the script file.
\ No newline at end of file diff --git a/test/test_flow.py b/test/test_flow.py index 6e9464e7..914138c9 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -481,7 +481,7 @@ class TestSerialize: f2 = l[0] assert f2._get_state() == f._get_state() - assert f2.request._assemble() == f.request._assemble() + assert f2.request.assemble() == f.request.assemble() def test_load_flows(self): r = self._treader() @@ -753,63 +753,61 @@ class TestRequest: def test_simple(self): f = tutils.tflow() r = f.request - u = r.get_url(False, f) - assert r.set_url(u, f) - assert not r.set_url("", f) - assert r.get_url(False, f) == u - assert r._assemble() - assert r.size() == len(r._assemble()) + u = r.url + r.url = u + tutils.raises(ValueError, setattr, r, "url", "") + assert r.url == u + assert r.assemble() + assert r.size() == len(r.assemble()) r2 = r.copy() assert r == r2 r.content = None - assert r._assemble() - assert r.size() == len(r._assemble()) + assert r.assemble() + assert r.size() == len(r.assemble()) r.content = CONTENT_MISSING - tutils.raises("Cannot assemble flow with CONTENT_MISSING", r._assemble) + tutils.raises("Cannot assemble flow with CONTENT_MISSING", r.assemble) def test_get_url(self): - f = tutils.tflow() - r = f.request + r = tutils.treq() - assert r.get_url(False, f) == "http://address:22/path" + assert r.url == "http://address:22/path" r.scheme = "https" - assert r.get_url(False, f) == "https://address:22/path" + assert r.url == "https://address:22/path" r.host = "host" r.port = 42 - assert r.get_url(False, f) == "https://host:42/path" + assert r.url == "https://host:42/path" r.host = "address" r.port = 22 - assert r.get_url(False, f) == "https://address:22/path" + assert r.url== "https://address:22/path" - assert r.get_url(True, f) == "https://address:22/path" + assert r.pretty_url(True) == "https://address:22/path" r.headers["Host"] = ["foo.com"] - assert r.get_url(False, f) == "https://address:22/path" - assert r.get_url(True, f) == "https://foo.com:22/path" + assert r.pretty_url(False) == "https://address:22/path" + assert r.pretty_url(True) == "https://foo.com:22/path" def test_path_components(self): - f = tutils.tflow() - r = f.request + r = tutils.treq() r.path = "/" - assert r.get_path_components(f) == [] + assert r.get_path_components() == [] r.path = "/foo/bar" - assert r.get_path_components(f) == ["foo", "bar"] + assert r.get_path_components() == ["foo", "bar"] q = flow.ODict() q["test"] = ["123"] - r.set_query(q, f) - assert r.get_path_components(f) == ["foo", "bar"] - - r.set_path_components([], f) - assert r.get_path_components(f) == [] - r.set_path_components(["foo"], f) - assert r.get_path_components(f) == ["foo"] - r.set_path_components(["/oo"], f) - assert r.get_path_components(f) == ["/oo"] + r.set_query(q) + assert r.get_path_components() == ["foo", "bar"] + + r.set_path_components([]) + assert r.get_path_components() == [] + r.set_path_components(["foo"]) + assert r.get_path_components() == ["foo"] + r.set_path_components(["/oo"]) + assert r.get_path_components() == ["/oo"] assert "%2F" in r.path def test_getset_form_urlencoded(self): @@ -828,26 +826,26 @@ class TestRequest: def test_getset_query(self): h = flow.ODictCaseless() - f = tutils.tflow() - f.request.path = "/foo?x=y&a=b" - q = f.request.get_query(f) + r = tutils.treq() + r.path = "/foo?x=y&a=b" + q = r.get_query() assert q.lst == [("x", "y"), ("a", "b")] - f.request.path = "/" - q = f.request.get_query(f) + r.path = "/" + q = r.get_query() assert not q - f.request.path = "/?adsfa" - q = f.request.get_query(f) + r.path = "/?adsfa" + q = r.get_query() assert q.lst == [("adsfa", "")] - f.request.path = "/foo?x=y&a=b" - assert f.request.get_query(f) - f.request.set_query(flow.ODict([]), f) - assert not f.request.get_query(f) + r.path = "/foo?x=y&a=b" + assert r.get_query() + r.set_query(flow.ODict([])) + assert not r.get_query() qv = flow.ODict([("a", "b"), ("c", "d")]) - f.request.set_query(qv, f) - assert f.request.get_query(f) == qv + r.set_query(qv) + assert r.get_query() == qv def test_anticache(self): h = flow.ODictCaseless() @@ -968,18 +966,18 @@ class TestResponse: def test_simple(self): f = tutils.tflow(resp=True) resp = f.response - assert resp._assemble() - assert resp.size() == len(resp._assemble()) + assert resp.assemble() + assert resp.size() == len(resp.assemble()) resp2 = resp.copy() assert resp2 == resp resp.content = None - assert resp._assemble() - assert resp.size() == len(resp._assemble()) + assert resp.assemble() + assert resp.size() == len(resp.assemble()) resp.content = CONTENT_MISSING - tutils.raises("Cannot assemble flow with CONTENT_MISSING", resp._assemble) + tutils.raises("Cannot assemble flow with CONTENT_MISSING", resp.assemble) def test_refresh(self): r = tutils.tresp() diff --git a/test/test_protocol_http.py b/test/test_protocol_http.py index c2ff7b44..ea6cf3fd 100644 --- a/test/test_protocol_http.py +++ b/test/test_protocol_http.py @@ -1,5 +1,4 @@ from libmproxy.protocol.http import * -from libmproxy.protocol import KILL from cStringIO import StringIO import tutils, tservers @@ -32,11 +31,27 @@ class TestHTTPRequest: f.request.host = f.server_conn.address.host f.request.port = f.server_conn.address.port f.request.scheme = "http" - assert f.request._assemble() == "OPTIONS * HTTP/1.1\r\nHost: address:22\r\n\r\n" + assert f.request.assemble() == "OPTIONS * HTTP/1.1\r\nHost: address:22\r\n\r\n" def test_origin_form(self): s = StringIO("GET /foo\xff HTTP/1.1") tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s) + s = StringIO("GET /foo HTTP/1.1\r\nConnection: Upgrade\r\nUpgrade: h2c") + r = HTTPRequest.from_stream(s) + assert r.headers["Upgrade"] == ["h2c"] + + raw = r._assemble_headers() + assert "Upgrade" not in raw + assert "Host" not in raw + + r.url = "http://example.com/foo" + + raw = r._assemble_headers() + assert "Host" in raw + assert not "Host" in r.headers + r.update_host_header() + assert "Host" in r.headers + def test_authority_form(self): s = StringIO("CONNECT oops-no-port.com HTTP/1.1") @@ -44,26 +59,31 @@ class TestHTTPRequest: s = StringIO("CONNECT address:22 HTTP/1.1") r = HTTPRequest.from_stream(s) r.scheme, r.host, r.port = "http", "address", 22 - assert r._assemble() == "CONNECT address:22 HTTP/1.1\r\nHost: address:22\r\n\r\n" + assert r.assemble() == "CONNECT address:22 HTTP/1.1\r\nHost: address:22\r\n\r\n" + assert r.pretty_url(False) == "address:22" def test_absolute_form(self): s = StringIO("GET oops-no-protocol.com HTTP/1.1") tutils.raises("Bad HTTP request line", HTTPRequest.from_stream, s) s = StringIO("GET http://address:22/ HTTP/1.1") r = HTTPRequest.from_stream(s) - assert r._assemble() == "GET http://address:22/ HTTP/1.1\r\nHost: address:22\r\n\r\n" + assert r.assemble() == "GET http://address:22/ HTTP/1.1\r\nHost: address:22\r\n\r\n" def test_assemble_unknown_form(self): r = tutils.treq() - tutils.raises("Invalid request form", r._assemble, "antiauthority") + tutils.raises("Invalid request form", r.assemble, "antiauthority") def test_set_url(self): - f = tutils.tflow(req=tutils.treq_absolute()) - f.request.set_url("https://otheraddress:42/ORLY", f) - assert f.request.scheme == "https" - assert f.request.host == "otheraddress" - assert f.request.port == 42 - assert f.request.path == "/ORLY" + r = tutils.treq_absolute() + r.url = "https://otheraddress:42/ORLY" + assert r.scheme == "https" + assert r.host == "otheraddress" + assert r.port == 42 + assert r.path == "/ORLY" + + def test_repr(self): + r = tutils.treq() + assert repr(r) class TestHTTPResponse: @@ -86,6 +106,19 @@ class TestHTTPResponse: assert r.content == "" tutils.raises("Invalid server response: 'content", HTTPResponse.from_stream, s, "GET") + def test_repr(self): + r = tutils.tresp() + assert "unknown content type" in repr(r) + r.headers["content-type"] = ["foo"] + assert "foo" in repr(r) + assert repr(tutils.tresp(content=CONTENT_MISSING)) + + +class TestHTTPFlow(object): + def test_repr(self): + f = tutils.tflow(resp=True, err=True) + assert repr(f) + class TestInvalidRequests(tservers.HTTPProxTest): ssl = True @@ -100,120 +133,4 @@ class TestInvalidRequests(tservers.HTTPProxTest): p.connect() r = p.request("get:/p/200") assert r.status_code == 400 - assert "Invalid HTTP request form" in r.content - - -class TestProxyChaining(tservers.HTTPChainProxyTest): - def test_all(self): - self.chain[1].tmaster.replacehooks.add("~q", "foo", "bar") # replace in request - self.chain[0].tmaster.replacehooks.add("~q", "foo", "oh noes!") - self.proxy.tmaster.replacehooks.add("~q", "bar", "baz") - self.chain[0].tmaster.replacehooks.add("~s", "baz", "ORLY") # replace in response - - p = self.pathoc() - req = p.request("get:'%s/p/418:b\"foo\"'" % self.server.urlbase) - assert req.content == "ORLY" - assert req.status_code == 418 - -class TestProxyChainingSSL(tservers.HTTPChainProxyTest): - ssl = True - def test_simple(self): - p = self.pathoc() - req = p.request("get:'/p/418:b\"content\"'") - assert req.content == "content" - assert req.status_code == 418 - - assert self.chain[1].tmaster.state.flow_count() == 2 # CONNECT from pathoc to chain[0], - # request from pathoc to chain[0] - assert self.chain[0].tmaster.state.flow_count() == 2 # CONNECT from chain[1] to proxy, - # request from chain[1] to proxy - assert self.proxy.tmaster.state.flow_count() == 1 # request from chain[0] (regular proxy doesn't store CONNECTs) - - def test_closing_connect_response(self): - """ - https://github.com/mitmproxy/mitmproxy/issues/313 - """ - def handle_request(f): - f.request.httpversion = (1, 0) - del f.request.headers["Content-Length"] - f.reply() - _handle_request = self.chain[0].tmaster.handle_request - self.chain[0].tmaster.handle_request = handle_request - try: - assert self.pathoc().request("get:/p/418").status_code == 418 - finally: - self.chain[0].tmaster.handle_request = _handle_request - - def test_sni(self): - p = self.pathoc(sni="foo.com") - req = p.request("get:'/p/418:b\"content\"'") - assert req.content == "content" - assert req.status_code == 418 - -class TestProxyChainingSSLReconnect(tservers.HTTPChainProxyTest): - ssl = True - - def test_reconnect(self): - """ - Tests proper functionality of ConnectionHandler.server_reconnect mock. - If we have a disconnect on a secure connection that's transparently proxified to - an upstream http proxy, we need to send the CONNECT request again. - """ - def kill_requests(master, attr, exclude): - k = [0] # variable scope workaround: put into array - _func = getattr(master, attr) - def handler(f): - k[0] += 1 - if not (k[0] in exclude): - f.client_conn.finish() - f.error = Error("terminated") - f.reply(KILL) - return _func(f) - setattr(master, attr, handler) - - kill_requests(self.proxy.tmaster, "handle_request", - exclude=[ - # fail first request - 2, # allow second request - ]) - - kill_requests(self.chain[0].tmaster, "handle_request", - exclude=[ - 1, # CONNECT - # fail first request - 3, # reCONNECT - 4, # request - ]) - - p = self.pathoc() - req = p.request("get:'/p/418:b\"content\"'") - assert self.chain[1].tmaster.state.flow_count() == 2 # CONNECT and request - assert self.chain[0].tmaster.state.flow_count() == 4 # CONNECT, failing request, - # reCONNECT, request - assert self.proxy.tmaster.state.flow_count() == 2 # failing request, request - # (doesn't store (repeated) CONNECTs from chain[0] - # as it is a regular proxy) - assert req.content == "content" - assert req.status_code == 418 - - assert not self.proxy.tmaster.state._flow_list[0].response # killed - assert self.proxy.tmaster.state._flow_list[1].response - - assert self.chain[1].tmaster.state._flow_list[0].request.form_in == "authority" - assert self.chain[1].tmaster.state._flow_list[1].request.form_in == "relative" - - assert self.chain[0].tmaster.state._flow_list[0].request.form_in == "authority" - assert self.chain[0].tmaster.state._flow_list[1].request.form_in == "relative" - assert self.chain[0].tmaster.state._flow_list[2].request.form_in == "authority" - assert self.chain[0].tmaster.state._flow_list[3].request.form_in == "relative" - - assert self.proxy.tmaster.state._flow_list[0].request.form_in == "relative" - assert self.proxy.tmaster.state._flow_list[1].request.form_in == "relative" - - req = p.request("get:'/p/418:b\"content2\"'") - - assert req.status_code == 502 - assert self.chain[1].tmaster.state.flow_count() == 3 # + new request - assert self.chain[0].tmaster.state.flow_count() == 6 # + new request, repeated CONNECT from chain[1] - # (both terminated) - assert self.proxy.tmaster.state.flow_count() == 2 # nothing happened here + assert "Invalid HTTP request form" in r.content
\ No newline at end of file diff --git a/test/test_proxy.py b/test/test_proxy.py index 2e206529..2f455992 100644 --- a/test/test_proxy.py +++ b/test/test_proxy.py @@ -23,25 +23,34 @@ class TestServerConnection: self.d.shutdown() def test_simple(self): - sc = ServerConnection((self.d.IFACE, self.d.port), None) + sc = ServerConnection((self.d.IFACE, self.d.port)) sc.connect() f = tutils.tflow() f.server_conn = sc f.request.path = "/p/200:da" - sc.send(f.request._assemble()) + sc.send(f.request.assemble()) assert http.read_response(sc.rfile, f.request.method, 1000) assert self.d.last_log() sc.finish() def test_terminate_error(self): - sc = ServerConnection((self.d.IFACE, self.d.port), None) + sc = ServerConnection((self.d.IFACE, self.d.port)) sc.connect() sc.connection = mock.Mock() sc.connection.recv = mock.Mock(return_value=False) sc.connection.flush = mock.Mock(side_effect=tcp.NetLibDisconnect) sc.finish() + def test_repr(self): + sc = tutils.tserver_conn() + assert "address:22" in repr(sc) + assert "ssl" not in repr(sc) + sc.ssl_established = True + assert "ssl" in repr(sc) + sc.sni = "foo" + assert "foo" in repr(sc) + class TestProcessProxyOptions: def p(self, *args): @@ -112,7 +121,7 @@ class TestProcessProxyOptions: class TestProxyServer: - @tutils.SkipWindows # binding to 0.0.0.0:1 works without special permissions on Windows + @tutils.SkipWindows # binding to 0.0.0.0:1 works without special permissions on Windows def test_err(self): parser = argparse.ArgumentParser() cmdline.common_options(parser) diff --git a/test/test_script.py b/test/test_script.py index 7c421fde..aed7def1 100644 --- a/test/test_script.py +++ b/test/test_script.py @@ -99,7 +99,7 @@ class TestScript: d = Dummy() assert s.run(hook, d)[0] d.reply() - while (time.time() - t_start) < 5 and m.call_count <= 5: + while (time.time() - t_start) < 20 and m.call_count <= 5: if m.call_count == 5: return time.sleep(0.001) diff --git a/test/test_server.py b/test/test_server.py index 6c0829f4..a6d591ed 100644 --- a/test/test_server.py +++ b/test/test_server.py @@ -1,10 +1,10 @@ import socket, time -import mock +from libmproxy.proxy.config import ProxyConfig from netlib import tcp, http_auth, http from libpathod import pathoc, pathod +from netlib.certutils import SSLCert import tutils, tservers -from libmproxy import flow -from libmproxy.protocol import KILL +from libmproxy.protocol import KILL, Error from libmproxy.protocol.http import CONTENT_MISSING """ @@ -21,8 +21,11 @@ class CommonMixin: def test_replay(self): assert self.pathod("304").status_code == 304 - assert len(self.master.state.view) == 1 - l = self.master.state.view[0] + if isinstance(self, tservers.HTTPUpstreamProxTest) and self.ssl: + assert len(self.master.state.view) == 2 + else: + assert len(self.master.state.view) == 1 + l = self.master.state.view[-1] assert l.response.code == 304 l.request.path = "/p/305" rt = self.master.replay_request(l, block=True) @@ -31,18 +34,28 @@ class CommonMixin: # Disconnect error l.request.path = "/p/305:d0" rt = self.master.replay_request(l, block=True) - assert l.error + assert not rt + if isinstance(self, tservers.HTTPUpstreamProxTest): + assert l.response.code == 502 + else: + assert l.error # Port error l.request.port = 1 - self.master.replay_request(l, block=True) - assert l.error + # In upstream mode, we get a 502 response from the upstream proxy server. + # In upstream mode with ssl, the replay will fail as we cannot establish SSL with the upstream proxy. + rt = self.master.replay_request(l, block=True) + assert not rt + if isinstance(self, tservers.HTTPUpstreamProxTest) and not self.ssl: + assert l.response.code == 502 + else: + assert l.error def test_http(self): f = self.pathod("304") assert f.status_code == 304 - l = self.master.state.view[0] + l = self.master.state.view[-1] # In Upstream mode with SSL, we may already have a previous CONNECT request. assert l.client_conn.address assert "host" in l.request.headers assert l.response.code == 304 @@ -55,6 +68,51 @@ class CommonMixin: line = t.rfile.readline() assert ("Bad Request" in line) or ("Bad Gateway" in line) + def test_sni(self): + if not self.ssl: + return + + f = self.pathod("304", sni="testserver.com") + assert f.status_code == 304 + log = self.server.last_log() + assert log["request"]["sni"] == "testserver.com" + +class TcpMixin: + def _ignore_on(self): + conf = ProxyConfig(ignore=[".+:%s" % self.server.port]) + self.config.ignore.append(conf.ignore[0]) + + def _ignore_off(self): + self.config.ignore.pop() + + def test_ignore(self): + spec = '304:h"Alternate-Protocol"="mitmproxy-will-remove-this"' + n = self.pathod(spec) + self._ignore_on() + i = self.pathod(spec) + i2 = self.pathod(spec) + self._ignore_off() + + assert i.status_code == i2.status_code == n.status_code == 304 + assert "Alternate-Protocol" in i.headers + assert "Alternate-Protocol" in i2.headers + assert "Alternate-Protocol" not in n.headers + + # Test that we get the original SSL cert + if self.ssl: + i_cert = SSLCert(i.sslinfo.certchain[0]) + i2_cert = SSLCert(i2.sslinfo.certchain[0]) + n_cert = SSLCert(n.sslinfo.certchain[0]) + + assert i_cert == i2_cert + assert i_cert != n_cert + + # Test Non-HTTP traffic + spec = "200:i0,@100:d0" # this results in just 100 random bytes + assert self.pathod(spec).status_code == 502 # mitmproxy responds with bad gateway + self._ignore_on() + tutils.raises("invalid server response", self.pathod, spec) # pathoc tries to parse answer as HTTP + self._ignore_off() class AppMixin: @@ -64,7 +122,6 @@ class AppMixin: assert "mitmproxy" in ret.content - class TestHTTP(tservers.HTTPProxTest, CommonMixin, AppMixin): def test_app_err(self): p = self.pathoc() @@ -175,7 +232,7 @@ class TestHTTPConnectSSLError(tservers.HTTPProxTest): tutils.raises("502 - Bad Gateway", p.http_connect, dst) -class TestHTTPS(tservers.HTTPProxTest, CommonMixin): +class TestHTTPS(tservers.HTTPProxTest, CommonMixin, TcpMixin): ssl = True ssloptions = pathod.SSLOptions(request_client_cert=True) clientcerts = True @@ -184,12 +241,6 @@ class TestHTTPS(tservers.HTTPProxTest, CommonMixin): assert f.status_code == 304 assert self.server.last_log()["request"]["clientcert"]["keyinfo"] - def test_sni(self): - f = self.pathod("304", sni="testserver.com") - assert f.status_code == 304 - l = self.server.last_log() - assert self.server.last_log()["request"]["sni"] == "testserver.com" - def test_error_post_connect(self): p = self.pathoc() assert p.request("get:/:i0,'invalid\r\n\r\n'").status_code == 400 @@ -217,21 +268,16 @@ class TestHTTPSNoCommonName(tservers.HTTPProxTest): assert f.sslinfo.certchain[0].get_subject().CN == "127.0.0.1" -class TestReverse(tservers.ReverseProxTest, CommonMixin): +class TestReverse(tservers.ReverseProxTest, CommonMixin, TcpMixin): reverse = True -class TestTransparent(tservers.TransparentProxTest, CommonMixin): +class TestTransparent(tservers.TransparentProxTest, CommonMixin, TcpMixin): ssl = False -class TestTransparentSSL(tservers.TransparentProxTest, CommonMixin): +class TestTransparentSSL(tservers.TransparentProxTest, CommonMixin, TcpMixin): ssl = True - def test_sni(self): - f = self.pathod("304", sni="testserver.com") - assert f.status_code == 304 - l = self.server.last_log() - assert l["request"]["sni"] == "testserver.com" def test_sslerr(self): p = pathoc.Pathoc(("localhost", self.proxy.port)) @@ -312,7 +358,7 @@ class TestProxy(tservers.HTTPProxTest): f = self.pathod("200:b@100") assert f.status_code == 200 f = self.master.state.view[0] - assert f.server_conn.peername == ("127.0.0.1", self.server.port) + assert f.server_conn.address == ("127.0.0.1", self.server.port) class TestProxySSL(tservers.HTTPProxTest): ssl=True @@ -330,21 +376,19 @@ class MasterRedirectRequest(tservers.TestMaster): def handle_request(self, f): request = f.request if request.path == "/p/201": - url = request.get_url(False, f) + url = request.url new = "http://127.0.0.1:%s/p/201" % self.redirect_port - request.set_url(new, f) - request.set_url(new, f) + request.url = new f.live.change_server(("127.0.0.1", self.redirect_port), False) - request.set_url(url, f) + request.url = url tutils.raises("SSL handshake error", f.live.change_server, ("127.0.0.1", self.redirect_port), True) - request.set_url(new, f) - request.set_url(url, f) - request.set_url(new, f) + request.url = new tservers.TestMaster.handle_request(self, f) def handle_response(self, f): f.response.content = str(f.client_conn.address.port) + f.response.headers["server-conn-id"] = [str(f.server_conn.source_address.port)] tservers.TestMaster.handle_response(self, f) @@ -377,7 +421,8 @@ class TestRedirectRequest(tservers.HTTPProxTest): assert self.server.last_log() assert not self.server2.last_log() - assert r3.content == r2.content == r1.content + assert r1.content == r2.content == r3.content + assert r1.headers.get_first("server-conn-id") == r3.headers.get_first("server-conn-id") # Make sure that we actually use the same connection in this test case class MasterStreamRequest(tservers.TestMaster): @@ -502,6 +547,132 @@ class TestIncompleteResponse(tservers.HTTPProxTest): class TestCertForward(tservers.HTTPProxTest): certforward = True ssl = True + def test_app_err(self): tutils.raises("handshake error", self.pathod, "200:b@100") + +class TestUpstreamProxy(tservers.HTTPUpstreamProxTest, CommonMixin, AppMixin): + ssl = False + + def test_order(self): + self.proxy.tmaster.replacehooks.add("~q", "foo", "bar") # replace in request + self.chain[0].tmaster.replacehooks.add("~q", "bar", "baz") + self.chain[1].tmaster.replacehooks.add("~q", "foo", "oh noes!") + self.chain[0].tmaster.replacehooks.add("~s", "baz", "ORLY") # replace in response + + p = self.pathoc() + req = p.request("get:'%s/p/418:b\"foo\"'" % self.server.urlbase) + assert req.content == "ORLY" + assert req.status_code == 418 + + +class TestUpstreamProxySSL(tservers.HTTPUpstreamProxTest, CommonMixin, TcpMixin): + ssl = True + + def _ignore_on(self): + super(TestUpstreamProxySSL, self)._ignore_on() + conf = ProxyConfig(ignore=[".+:%s" % self.server.port]) + for proxy in self.chain: + proxy.tmaster.server.config.ignore.append(conf.ignore[0]) + + def _ignore_off(self): + super(TestUpstreamProxySSL, self)._ignore_off() + for proxy in self.chain: + proxy.tmaster.server.config.ignore.pop() + + def test_simple(self): + p = self.pathoc() + req = p.request("get:'/p/418:b\"content\"'") + assert req.content == "content" + assert req.status_code == 418 + + assert self.proxy.tmaster.state.flow_count() == 2 # CONNECT from pathoc to chain[0], + # request from pathoc to chain[0] + assert self.chain[0].tmaster.state.flow_count() == 2 # CONNECT from proxy to chain[1], + # request from proxy to chain[1] + assert self.chain[1].tmaster.state.flow_count() == 1 # request from chain[0] (regular proxy doesn't store CONNECTs) + + def test_closing_connect_response(self): + """ + https://github.com/mitmproxy/mitmproxy/issues/313 + """ + def handle_request(f): + f.request.httpversion = (1, 0) + del f.request.headers["Content-Length"] + f.reply() + _handle_request = self.chain[0].tmaster.handle_request + self.chain[0].tmaster.handle_request = handle_request + try: + assert self.pathoc().request("get:/p/418").status_code == 418 + finally: + self.chain[0].tmaster.handle_request = _handle_request + + +class TestProxyChainingSSLReconnect(tservers.HTTPUpstreamProxTest): + ssl = True + + def test_reconnect(self): + """ + Tests proper functionality of ConnectionHandler.server_reconnect mock. + If we have a disconnect on a secure connection that's transparently proxified to + an upstream http proxy, we need to send the CONNECT request again. + """ + def kill_requests(master, attr, exclude): + k = [0] # variable scope workaround: put into array + _func = getattr(master, attr) + def handler(f): + k[0] += 1 + if not (k[0] in exclude): + f.client_conn.finish() + f.error = Error("terminated") + f.reply(KILL) + return _func(f) + setattr(master, attr, handler) + + kill_requests(self.chain[1].tmaster, "handle_request", + exclude=[ + # fail first request + 2, # allow second request + ]) + + kill_requests(self.chain[0].tmaster, "handle_request", + exclude=[ + 1, # CONNECT + # fail first request + 3, # reCONNECT + 4, # request + ]) + + p = self.pathoc() + req = p.request("get:'/p/418:b\"content\"'") + assert self.proxy.tmaster.state.flow_count() == 2 # CONNECT and request + assert self.chain[0].tmaster.state.flow_count() == 4 # CONNECT, failing request, + # reCONNECT, request + assert self.chain[1].tmaster.state.flow_count() == 2 # failing request, request + # (doesn't store (repeated) CONNECTs from chain[0] + # as it is a regular proxy) + assert req.content == "content" + assert req.status_code == 418 + + assert not self.chain[1].tmaster.state._flow_list[0].response # killed + assert self.chain[1].tmaster.state._flow_list[1].response + + assert self.proxy.tmaster.state._flow_list[0].request.form_in == "authority" + assert self.proxy.tmaster.state._flow_list[1].request.form_in == "relative" + + assert self.chain[0].tmaster.state._flow_list[0].request.form_in == "authority" + assert self.chain[0].tmaster.state._flow_list[1].request.form_in == "relative" + assert self.chain[0].tmaster.state._flow_list[2].request.form_in == "authority" + assert self.chain[0].tmaster.state._flow_list[3].request.form_in == "relative" + + assert self.chain[1].tmaster.state._flow_list[0].request.form_in == "relative" + assert self.chain[1].tmaster.state._flow_list[1].request.form_in == "relative" + + req = p.request("get:'/p/418:b\"content2\"'") + + assert req.status_code == 502 + assert self.proxy.tmaster.state.flow_count() == 3 # + new request + assert self.chain[0].tmaster.state.flow_count() == 6 # + new request, repeated CONNECT from chain[1] + # (both terminated) + assert self.chain[1].tmaster.state.flow_count() == 2 # nothing happened here diff --git a/test/tservers.py b/test/tservers.py index 9f2abbe1..8a2e72a4 100644 --- a/test/tservers.py +++ b/test/tservers.py @@ -84,29 +84,19 @@ class ProxTestBase(object): masterclass = TestMaster externalapp = False certforward = False + @classmethod def setupAll(cls): cls.server = libpathod.test.Daemon(ssl=cls.ssl, ssloptions=cls.ssloptions) cls.server2 = libpathod.test.Daemon(ssl=cls.ssl, ssloptions=cls.ssloptions) - pconf = cls.get_proxy_config() - cls.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy") - cls.config = ProxyConfig( - no_upstream_cert = cls.no_upstream_cert, - confdir = cls.confdir, - authenticator = cls.authenticator, - certforward = cls.certforward, - ssl_ports=([cls.server.port, cls.server2.port] if cls.ssl else []), - **pconf - ) + + cls.config = ProxyConfig(**cls.get_proxy_config()) + tmaster = cls.masterclass(cls.config) tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp) cls.proxy = ProxyThread(tmaster) cls.proxy.start() - @property - def master(cls): - return cls.proxy.tmaster - @classmethod def teardownAll(cls): shutil.rmtree(cls.confdir) @@ -121,24 +111,20 @@ class ProxTestBase(object): self.server2.clear_log() @property - def scheme(self): - return "https" if self.ssl else "http" - - @property - def proxies(self): - """ - The URL base for the server instance. - """ - return ( - (self.scheme, ("127.0.0.1", self.proxy.port)) - ) + def master(self): + return self.proxy.tmaster @classmethod def get_proxy_config(cls): - d = dict() - if cls.clientcerts: - d["clientcerts"] = tutils.test_data.path("data/clientcert") - return d + cls.confdir = os.path.join(tempfile.gettempdir(), "mitmproxy") + return dict( + no_upstream_cert = cls.no_upstream_cert, + confdir = cls.confdir, + authenticator = cls.authenticator, + certforward = cls.certforward, + ssl_ports=([cls.server.port, cls.server2.port] if cls.ssl else []), + clientcerts = tutils.test_data.path("data/clientcert") if cls.clientcerts else None + ) class HTTPProxTest(ProxTestBase): @@ -265,49 +251,50 @@ class ReverseProxTest(ProxTestBase): class ChainProxTest(ProxTestBase): """ - Chain n instances of mitmproxy in a row - because we can. + Chain three instances of mitmproxy in a row to test upstream mode. + Proxy order is cls.proxy -> cls.chain[0] -> cls.chain[1] + cls.proxy and cls.chain[0] are in upstream mode, + cls.chain[1] is in regular mode. """ + chain = None n = 2 - chain_config = [lambda port, sslports: ProxyConfig( - upstream_server= (False, False, "127.0.0.1", port), - http_form_in = "absolute", - http_form_out = "absolute", - ssl_ports=sslports - )] * n + @classmethod def setupAll(cls): - super(ChainProxTest, cls).setupAll() cls.chain = [] - for i in range(cls.n): - sslports = [cls.server.port, cls.server2.port] - config = cls.chain_config[i](cls.proxy.port if i == 0 else cls.chain[-1].port, - sslports) + super(ChainProxTest, cls).setupAll() + for _ in range(cls.n): + config = ProxyConfig(**cls.get_proxy_config()) tmaster = cls.masterclass(config) - tmaster.start_app(APP_HOST, APP_PORT, cls.externalapp) - cls.chain.append(ProxyThread(tmaster)) - cls.chain[-1].start() + proxy = ProxyThread(tmaster) + proxy.start() + cls.chain.insert(0, proxy) + + # Patch the orginal proxy to upstream mode + cls.config = cls.proxy.tmaster.config = cls.proxy.tmaster.server.config = ProxyConfig(**cls.get_proxy_config()) + @classmethod def teardownAll(cls): super(ChainProxTest, cls).teardownAll() - for p in cls.chain: - p.tmaster.shutdown() + for proxy in cls.chain: + proxy.shutdown() def setUp(self): super(ChainProxTest, self).setUp() - for p in self.chain: - p.tmaster.clear_log() - p.tmaster.state.clear() + for proxy in self.chain: + proxy.tmaster.clear_log() + proxy.tmaster.state.clear() + @classmethod + def get_proxy_config(cls): + d = super(ChainProxTest, cls).get_proxy_config() + if cls.chain: # First proxy is in normal mode. + d.update( + mode="upstream", + upstream_server=(False, False, "127.0.0.1", cls.chain[0].port) + ) + return d -class HTTPChainProxyTest(ChainProxTest): - def pathoc(self, sni=None): - """ - Returns a connected Pathoc instance. - """ - p = libpathod.pathoc.Pathoc(("localhost", self.chain[-1].port), ssl=self.ssl, sni=sni) - if self.ssl: - p.connect(("127.0.0.1", self.server.port)) - else: - p.connect() - return p +class HTTPUpstreamProxTest(ChainProxTest, HTTPProxTest): + pass
\ No newline at end of file |