diff options
author | Aldo Cortesi <aldo@corte.si> | 2014-09-07 12:59:35 +1200 |
---|---|---|
committer | Aldo Cortesi <aldo@corte.si> | 2014-09-07 12:59:35 +1200 |
commit | 3d62e90dbf7ea05283e16752531a261e53a4bb47 (patch) | |
tree | c3f5aed62bcf13378522968a1c77375dc8102b53 /libmproxy | |
parent | 0e0cff638c1e055275e77e2af0ae540542f77197 (diff) | |
parent | fdd7b2f108717900e39e3d0ab220ee65b79304ef (diff) | |
download | mitmproxy-3d62e90dbf7ea05283e16752531a261e53a4bb47.tar.gz mitmproxy-3d62e90dbf7ea05283e16752531a261e53a4bb47.tar.bz2 mitmproxy-3d62e90dbf7ea05283e16752531a261e53a4bb47.zip |
Merge pull request #342 from mitmproxy/server_change_api
Server change api
Diffstat (limited to 'libmproxy')
-rw-r--r-- | libmproxy/console/common.py | 2 | ||||
-rw-r--r-- | libmproxy/console/flowview.py | 6 | ||||
-rw-r--r-- | libmproxy/dump.py | 2 | ||||
-rw-r--r-- | libmproxy/filt.py | 4 | ||||
-rw-r--r-- | libmproxy/flow.py | 15 | ||||
-rw-r--r-- | libmproxy/protocol/http.py | 495 | ||||
-rw-r--r-- | libmproxy/protocol/primitives.py | 72 | ||||
-rw-r--r-- | libmproxy/protocol/tcp.py | 4 | ||||
-rw-r--r-- | libmproxy/proxy/__init__.py | 3 | ||||
-rw-r--r-- | libmproxy/proxy/config.py | 6 | ||||
-rw-r--r-- | libmproxy/proxy/connection.py | 18 | ||||
-rw-r--r-- | libmproxy/proxy/primitives.py | 13 | ||||
-rw-r--r-- | libmproxy/proxy/server.py | 25 | ||||
-rw-r--r-- | libmproxy/stateobject.py | 3 |
14 files changed, 340 insertions, 328 deletions
diff --git a/libmproxy/console/common.py b/libmproxy/console/common.py index 5cb3dd2a..104b7216 100644 --- a/libmproxy/console/common.py +++ b/libmproxy/console/common.py @@ -177,7 +177,7 @@ def format_flow(f, focus, extended=False, hostheader=False, padding=2): req_timestamp = f.request.timestamp_start, req_is_replay = f.request.is_replay, req_method = f.request.method, - req_url = f.request.get_url(hostheader=hostheader), + req_url = f.request.pretty_url(hostheader=hostheader), err_msg = f.error.msg if f.error else None, resp_code = f.response.code if f.response else None, diff --git a/libmproxy/console/flowview.py b/libmproxy/console/flowview.py index 3c63ac29..014d44c0 100644 --- a/libmproxy/console/flowview.py +++ b/libmproxy/console/flowview.py @@ -528,7 +528,9 @@ class FlowView(common.WWrap): def set_url(self, url): request = self.flow.request - if not request.set_url(str(url)): + try: + request.url = str(url) + except ValueError: return "Invalid URL." self.master.refresh_flow(self.flow) @@ -608,7 +610,7 @@ class FlowView(common.WWrap): elif part == "q": self.master.view_grideditor(grideditor.QueryEditor(self.master, conn.get_query().lst, self.set_query, conn)) elif part == "u" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST: - self.master.prompt_edit("URL", conn.get_url(), self.set_url) + self.master.prompt_edit("URL", conn.url, self.set_url) elif part == "m" and self.state.view_flow_mode == common.VIEW_FLOW_REQUEST: self.master.prompt_onekey("Method", self.method_options, self.edit_method) elif part == "c" and self.state.view_flow_mode == common.VIEW_FLOW_RESPONSE: diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 8ecd56e7..72ab58a3 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -55,7 +55,7 @@ def str_request(f, showhost): c = f.client_conn.address.host else: c = "[replay]" - r = "%s %s %s"%(c, f.request.method, f.request.get_url(showhost, f)) + r = "%s %s %s"%(c, f.request.method, f.request.pretty_url(showhost)) if f.request.stickycookie: r = "[stickycookie] " + r return r diff --git a/libmproxy/filt.py b/libmproxy/filt.py index 925dbfbb..7d2bd737 100644 --- a/libmproxy/filt.py +++ b/libmproxy/filt.py @@ -208,7 +208,7 @@ class FDomain(_Rex): code = "d" help = "Domain" def __call__(self, f): - return bool(re.search(self.expr, f.request.get_host(False, f), re.IGNORECASE)) + return bool(re.search(self.expr, f.request.host, re.IGNORECASE)) class FUrl(_Rex): @@ -222,7 +222,7 @@ class FUrl(_Rex): return klass(*toks) def __call__(self, f): - return re.search(self.expr, f.request.get_url(False, f)) + return re.search(self.expr, f.request.url) class _Int(_Action): diff --git a/libmproxy/flow.py b/libmproxy/flow.py index df72878f..086710bc 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -259,8 +259,8 @@ class StickyCookieState: Returns a (domain, port, path) tuple. """ return ( - m["domain"] or f.request.get_host(False, f), - f.request.get_port(f), + m["domain"] or f.request.host, + f.request.port, m["path"] or "/" ) @@ -278,7 +278,7 @@ class StickyCookieState: c = Cookie.SimpleCookie(str(i)) m = c.values()[0] k = self.ckey(m, f) - if self.domain_match(f.request.get_host(False, f), k[0]): + if self.domain_match(f.request.host, k[0]): self.jar[self.ckey(m, f)] = m def handle_request(self, f): @@ -286,8 +286,8 @@ class StickyCookieState: if f.match(self.flt): for i in self.jar.keys(): match = [ - self.domain_match(f.request.get_host(False, f), i[0]), - f.request.get_port(f) == i[1], + self.domain_match(f.request.host, i[0]), + f.request.port == i[1], f.request.path.startswith(i[2]) ] if all(match): @@ -306,7 +306,7 @@ class StickyAuthState: self.hosts = {} def handle_request(self, f): - host = f.request.get_host(False, f) + host = f.request.host if "authorization" in f.request.headers: self.hosts[host] = f.request.headers["authorization"] elif f.match(self.flt): @@ -612,6 +612,7 @@ class FlowMaster(controller.Master): if f.request: self.handle_request(f) if f.response: + self.handle_responseheaders(f) self.handle_response(f) if f.error: self.handle_error(f) @@ -668,7 +669,7 @@ class FlowMaster(controller.Master): self.masterq, self.should_exit ) - rt.start() # pragma: no cover + rt.start() # pragma: no cover if block: rt.join() diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 9699c78a..c67cb471 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -26,16 +26,22 @@ def get_line(fp): return line -def send_connect_request(conn, host, port): +def send_connect_request(conn, host, port, update_state=True): upstream_request = HTTPRequest("authority", "CONNECT", None, host, port, None, (1, 1), ODictCaseless(), "") - conn.send(upstream_request._assemble()) + conn.send(upstream_request.assemble()) resp = HTTPResponse.from_stream(conn.rfile, upstream_request.method) if resp.code != 200: raise proxy.ProxyError(resp.code, "Cannot establish SSL " + "connection with upstream proxy: \r\n" + - str(resp._assemble())) + str(resp.assemble())) + if update_state: + conn.state.append(("http", { + "state": "connect", + "host": host, + "port": port} + )) return resp @@ -67,6 +73,9 @@ class decoded(object): class HTTPMessage(stateobject.SimpleStateObject): + """ + Base class for HTTPRequest and HTTPResponse + """ def __init__(self, httpversion, headers, content, timestamp_start=None, timestamp_end=None): self.httpversion = httpversion @@ -151,36 +160,29 @@ class HTTPMessage(stateobject.SimpleStateObject): c += self.headers.replace(pattern, repl, *args, **kwargs) return c - @classmethod - def from_stream(cls, rfile, include_body=True, body_size_limit=None): - """ - Parse an HTTP message from a file stream - """ - raise NotImplementedError # pragma: nocover - def _assemble_first_line(self): """ Returns the assembled request/response line """ - raise NotImplementedError # pragma: nocover + raise NotImplementedError() # pragma: nocover def _assemble_headers(self): """ Returns the assembled headers """ - raise NotImplementedError # pragma: nocover + raise NotImplementedError() # pragma: nocover def _assemble_head(self): """ Returns the assembled request/response line plus headers """ - raise NotImplementedError # pragma: nocover + raise NotImplementedError() # pragma: nocover - def _assemble(self): + def assemble(self): """ Returns the assembled request/response """ - raise NotImplementedError # pragma: nocover + raise NotImplementedError() # pragma: nocover class HTTPRequest(HTTPMessage): @@ -189,7 +191,17 @@ class HTTPRequest(HTTPMessage): Exposes the following attributes: - flow: Flow object the request belongs to + method: HTTP method + + scheme: URL scheme (http/https) (absolute-form only) + + host: Host portion of the URL (absolute-form and authority-form only) + + port: Destination port (absolute-form and authority-form only) + + path: Path portion of the URL (not present in authority-form) + + httpversion: HTTP version tuple, e.g. (1,1) headers: ODictCaseless object @@ -205,18 +217,6 @@ class HTTPRequest(HTTPMessage): form_out: The request form which mitmproxy has send out to the destination - method: HTTP method - - scheme: URL scheme (http/https) (absolute-form only) - - host: Host portion of the URL (absolute-form and authority-form only) - - port: Destination port (absolute-form and authority-form only) - - path: Path portion of the URL (not present in authority-form) - - httpversion: HTTP version tuple - timestamp_start: Timestamp indicating when request transmission started timestamp_end: Timestamp indicating when request transmission ended @@ -358,7 +358,7 @@ class HTTPRequest(HTTPMessage): def _assemble_head(self, form=None): return "%s\r\n%s\r\n" % (self._assemble_first_line(form), self._assemble_headers()) - def _assemble(self, form=None): + def assemble(self, form=None): """ Assembles the request for transmission to the server. We make some modifications to make sure interception works properly. @@ -405,6 +405,12 @@ class HTTPRequest(HTTPMessage): e for e in encoding.ENCODINGS if e in self.headers["accept-encoding"][0] )] + def update_host_header(self): + """ + Update the host header to reflect the current target. + """ + self.headers["Host"] = [self.host] + def get_form_urlencoded(self): """ Retrieves the URL-encoded form data, returning an ODict object. @@ -426,16 +432,16 @@ class HTTPRequest(HTTPMessage): self.headers["Content-Type"] = [HDR_FORM_URLENCODED] self.content = utils.urlencode(odict.lst) - def get_path_components(self, f): + def get_path_components(self): """ Returns the path components of the URL as a list of strings. Components are unquoted. """ - _, _, path, _, _, _ = urlparse.urlparse(self.get_url(False, f)) + _, _, path, _, _, _ = urlparse.urlparse(self.url) return [urllib.unquote(i) for i in path.split("/") if i] - def set_path_components(self, lst, f): + def set_path_components(self, lst): """ Takes a list of strings, and sets the path component of the URL. @@ -443,32 +449,32 @@ class HTTPRequest(HTTPMessage): """ lst = [urllib.quote(i, safe="") for i in lst] path = "/" + "/".join(lst) - scheme, netloc, _, params, query, fragment = urlparse.urlparse(self.get_url(False, f)) - self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment]), f) + scheme, netloc, _, params, query, fragment = urlparse.urlparse(self.url) + self.url = urlparse.urlunparse([scheme, netloc, path, params, query, fragment]) - def get_query(self, f): + def get_query(self): """ Gets the request query string. Returns an ODict object. """ - _, _, _, _, query, _ = urlparse.urlparse(self.get_url(False, f)) + _, _, _, _, query, _ = urlparse.urlparse(self.url) if query: return ODict(utils.urldecode(query)) return ODict([]) - def set_query(self, odict, f): + def set_query(self, odict): """ Takes an ODict object, and sets the request query string. """ - scheme, netloc, path, params, _, fragment = urlparse.urlparse(self.get_url(False, f)) + scheme, netloc, path, params, _, fragment = urlparse.urlparse(self.url) query = utils.urlencode(odict.lst) - self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment]), f) + self.url = urlparse.urlunparse([scheme, netloc, path, params, query, fragment]) - def get_host(self, hostheader, flow): + def pretty_host(self, hostheader): """ Heuristic to get the host of the request. - Note that get_host() does not always return the TCP destination of the request, - e.g. on a transparently intercepted request to an unrelated HTTP proxy. + Note that pretty_host() does not always return the TCP destination of the request, + e.g. if an upstream proxy is in place If hostheader is set to True, the Host: header will be used as additional (and preferred) data source. This is handy in transparent mode, where only the ip of the destination is known, but not the @@ -478,54 +484,27 @@ class HTTPRequest(HTTPMessage): if hostheader: host = self.headers.get_first("host") if not host: - if self.host: - host = self.host - else: - for s in flow.server_conn.state: - if s[0] == "http" and s[1]["state"] == "connect": - host = s[1]["host"] - break - if not host: - host = flow.server_conn.address.host + host = self.host host = host.encode("idna") return host - def get_scheme(self, flow): - """ - Returns the request port, either from the request itself or from the flow's server connection - """ - if self.scheme: - return self.scheme - if self.form_out == "authority": # On SSLed connections, the original CONNECT request is still unencrypted. - return "http" - return "https" if flow.server_conn.ssl_established else "http" - - def get_port(self, flow): - """ - Returns the request port, either from the request itself or from the flow's server connection - """ - if self.port: - return self.port - for s in flow.server_conn.state: - if s[0] == "http" and s[1].get("state") == "connect": - return s[1]["port"] - return flow.server_conn.address.port + def pretty_url(self, hostheader): + if self.form_out == "authority": # upstream proxy mode + return "%s:%s" % (self.pretty_host(hostheader), self.port) + return utils.unparse_url(self.scheme, + self.pretty_host(hostheader), + self.port, + self.path).encode('ascii') - def get_url(self, hostheader, flow): + @property + def url(self): """ Returns a URL string, constructed from the Request's URL components. - - If hostheader is True, we use the value specified in the request - Host header to construct the URL. """ - if self.form_out == "authority": # upstream proxy mode - return "%s:%s" % (self.get_host(hostheader, flow), self.get_port(flow)) - return utils.unparse_url(self.get_scheme(flow), - self.get_host(hostheader, flow), - self.get_port(flow), - self.path).encode('ascii') + return self.pretty_url(False) - def set_url(self, url, flow): + @url.setter + def url(self, url): """ Parses a URL specification, and updates the Request's information accordingly. @@ -534,30 +513,8 @@ class HTTPRequest(HTTPMessage): """ parts = http.parse_url(url) if not parts: - return False - scheme, host, port, path = parts - is_ssl = (True if scheme == "https" else False) - - self.path = path - - if host != self.get_host(False, flow) or port != self.get_port(flow): - if flow.live: - flow.live.change_server((host, port), ssl=is_ssl) - else: - # There's not live server connection, we're just changing the attributes here. - flow.server_conn = ServerConnection((host, port), - proxy.AddressPriority.MANUALLY_CHANGED) - flow.server_conn.ssl_established = is_ssl - - # If this is an absolute request, replace the attributes on the request object as well. - if self.host: - self.host = host - if self.port: - self.port = port - if self.scheme: - self.scheme = scheme - - return True + raise ValueError("Invalid URL: %s" % url) + self.scheme, self.host, self.port, self.path = parts def get_cookies(self): cookie_headers = self.headers.get("cookie") @@ -590,7 +547,7 @@ class HTTPResponse(HTTPMessage): Exposes the following attributes: - flow: Flow object the request belongs to + httpversion: HTTP version tuple, e.g. (1,1) code: HTTP response code @@ -602,8 +559,6 @@ class HTTPResponse(HTTPMessage): is content associated, but not present. CONTENT_MISSING evaluates to False to make checking for the presence of content natural. - httpversion: HTTP version tuple - timestamp_start: Timestamp indicating when request transmission started timestamp_end: Timestamp indicating when request transmission ended @@ -682,7 +637,8 @@ class HTTPResponse(HTTPMessage): if self.content: headers["Content-Length"] = [str(len(self.content))] - elif not preserve_transfer_encoding and 'Transfer-Encoding' in self.headers: # add content-length for chuncked transfer-encoding with no content + # add content-length for chuncked transfer-encoding with no content + elif not preserve_transfer_encoding and 'Transfer-Encoding' in self.headers: headers["Content-Length"] = ["0"] return str(headers) @@ -691,7 +647,7 @@ class HTTPResponse(HTTPMessage): return '%s\r\n%s\r\n' % ( self._assemble_first_line(), self._assemble_headers(preserve_transfer_encoding=preserve_transfer_encoding)) - def _assemble(self): + def assemble(self): """ Assembles the response for transmission to the client. We make some modifications to make sure interception works properly. @@ -774,12 +730,14 @@ class HTTPResponse(HTTPMessage): class HTTPFlow(Flow): """ - A Flow is a collection of objects representing a single HTTP + A HTTPFlow is a collection of objects representing a single HTTP transaction. The main attributes are: request: HTTPRequest object response: HTTPResponse object error: Error object + server_conn: ServerConnection object + client_conn: ClientConnection object Note that it's possible for a Flow to have both a response and an error object. This might happen, for instance, when a response was received @@ -816,7 +774,7 @@ class HTTPFlow(Flow): s = "<HTTPFlow" for a in ("request", "response", "error", "client_conn", "server_conn"): if getattr(self, a, False): - s += "\r\n %s = {flow.%s}" % (a,a) + s += "\r\n %s = {flow.%s}" % (a, a) s += ">" return s.format(flow=self) @@ -895,6 +853,10 @@ class HttpAuthenticationError(Exception): class HTTPHandler(ProtocolHandler): + """ + HTTPHandler implements mitmproxys understanding of the HTTP protocol. + + """ def __init__(self, c): super(HTTPHandler, self).__init__(c) self.expected_form_in = c.config.http_form_in @@ -905,19 +867,21 @@ class HTTPHandler(ProtocolHandler): while self.handle_flow(): pass - def get_response_from_server(self, request, include_body=True): + def get_response_from_server(self, flow): self.c.establish_server_connection() - request_raw = request._assemble() + request_raw = flow.request.assemble() - for i in range(2): + for attempt in (0, 1): try: self.c.server_conn.send(request_raw) - res = HTTPResponse.from_stream(self.c.server_conn.rfile, request.method, - body_size_limit=self.c.config.body_size_limit, include_body=include_body) - return res + # Only get the headers at first... + flow.response = HTTPResponse.from_stream(self.c.server_conn.rfile, flow.request.method, + body_size_limit=self.c.config.body_size_limit, + include_body=False) + break except (tcp.NetLibDisconnect, http.HttpErrorConnClosed), v: self.c.log("error in server communication: %s" % repr(v), level="debug") - if i < 1: + if attempt == 0: # In any case, we try to reconnect at least once. # This is necessary because it might be possible that we already initiated an upstream connection # after clientconnect that has already been expired, e.g consider the following event log: @@ -931,13 +895,24 @@ class HTTPHandler(ProtocolHandler): else: raise + # call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True + self.c.channel.ask("responseheaders", flow) + + # now get the rest of the request body, if body still needs to be read but not streaming this response + if flow.response.stream: + flow.response.content = CONTENT_MISSING + else: + flow.response.content = http.read_http_body(self.c.server_conn.rfile, flow.response.headers, + self.c.config.body_size_limit, + flow.request.method, flow.response.code, False) + def handle_flow(self): flow = HTTPFlow(self.c.client_conn, self.c.server_conn, self.live) try: try: req = HTTPRequest.from_stream(self.c.client_conn.rfile, body_size_limit=self.c.config.body_size_limit) - except tcp.NetLibDisconnect: # specifically ignore disconnects that happen before/between requests. + except tcp.NetLibDisconnect: # don't throw an error for disconnects that happen before/between requests. return False self.c.log("request", "debug", [req._assemble_first_line(req.form_in)]) ret = self.process_request(flow, req) @@ -951,8 +926,7 @@ class HTTPHandler(ProtocolHandler): # sent through to the Master. flow.request = req request_reply = self.c.channel.ask("request", flow) - self.determine_server_address(flow, flow.request) # The inline script may have changed request.host - flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow + self.process_server_address(flow) # The inline script may have changed request.host if request_reply is None or request_reply == KILL: return False @@ -960,20 +934,7 @@ class HTTPHandler(ProtocolHandler): if isinstance(request_reply, HTTPResponse): flow.response = request_reply else: - - # read initially in "stream" mode, so we can get the headers separately - flow.response = self.get_response_from_server(flow.request, include_body=False) - - # call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True - self.c.channel.ask("responseheaders", flow) - - # now get the rest of the request body, if body still needs to be read but not streaming this response - if flow.response.stream: - flow.response.content = CONTENT_MISSING - else: - flow.response.content = http.read_http_body(self.c.server_conn.rfile, flow.response.headers, - self.c.config.body_size_limit, - flow.request.method, flow.response.code, False) + self.get_response_from_server(flow) # no further manipulation of self.c.server_conn beyond this point # we can safely set it as the final attribute value here. @@ -984,72 +945,39 @@ class HTTPHandler(ProtocolHandler): if response_reply is None or response_reply == KILL: return False - if not flow.response.stream: - # no streaming: - # we already received the full response from the server and can send it to the client straight away. - self.c.client_conn.send(flow.response._assemble()) - else: - # streaming: - # First send the body and then transfer the response incrementally: - h = flow.response._assemble_head(preserve_transfer_encoding=True) - self.c.client_conn.send(h) - for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, - flow.response.headers, - self.c.config.body_size_limit, flow.request.method, - flow.response.code, False, 4096): - for part in chunk: - self.c.client_conn.wfile.write(part) - self.c.client_conn.wfile.flush() - flow.response.timestamp_end = utils.timestamp() - - flow.timestamp_end = utils.timestamp() + self.send_response_to_client(flow) - close_connection = ( - http.connection_close(flow.request.httpversion, flow.request.headers) or - http.connection_close(flow.response.httpversion, flow.response.headers) or - http.expected_http_body_size(flow.response.headers, False, flow.request.method, - flow.response.code) == -1) - if close_connection: - if flow.request.form_in == "authority" and flow.response.code == 200: - # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313: - # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header - pass - else: - return False + if self.check_close_connection(flow): + return False # We sent a CONNECT request to an upstream proxy. if flow.request.form_in == "authority" and flow.response.code == 200: - # TODO: Eventually add headers (space/usefulness tradeoff) - # Make sure to add state info before the actual upgrade happens. - # During the upgrade, we may receive an SNI indication from the client, + # TODO: Possibly add headers (memory consumption/usefulness tradeoff) + # Make sure to add state info before the actual processing of the CONNECT request happens. + # During an SSL upgrade, we may receive an SNI indication from the client, # which resets the upstream connection. If this is the case, we must # already re-issue the CONNECT request at this point. self.c.server_conn.state.append(("http", {"state": "connect", "host": flow.request.host, "port": flow.request.port})) - - if self.c.check_ignore_address((flow.request.host, flow.request.port)): - self.c.log("Ignore host: %s:%s" % self.c.server_conn.address(), "info") - TCPHandler(self.c).handle_messages() + if not self.process_connect_request((flow.request.host, flow.request.port)): return False - else: - if flow.request.port in self.c.config.ssl_ports: - self.ssl_upgrade() - self.skip_authentication = True # If the user has changed the target server on this connection, # restore the original target server flow.live.restore_server() - flow.live = None - return True + return True # Next flow please. except (HttpAuthenticationError, http.HttpError, proxy.ProxyError, tcp.NetLibError), e: self.handle_error(e, flow) + finally: + flow.timestamp_end = utils.timestamp() + flow.live = None # Connection is not live anymore. return False def handle_server_reconnect(self, state): if state["state"] == "connect": - send_connect_request(self.c.server_conn, state["host"], state["port"]) + send_connect_request(self.c.server_conn, state["host"], state["port"], update_state=False) else: # pragma: nocover raise RuntimeError("Unknown State: %s" % state["state"]) @@ -1093,16 +1021,6 @@ class HTTPHandler(ProtocolHandler): self.c.client_conn.wfile.write(html_content) self.c.client_conn.wfile.flush() - def ssl_upgrade(self): - """ - Upgrade the connection to SSL after an authority (CONNECT) request has been made. - """ - self.c.log("Received CONNECT request. Upgrading to SSL...", "debug") - self.expected_form_in = "relative" - self.expected_form_out = "relative" - self.c.establish_ssl(server=True, client=True) - self.c.log("Upgrade to SSL completed.", "debug") - def process_request(self, flow, request): """ @returns: @@ -1115,14 +1033,30 @@ class HTTPHandler(ProtocolHandler): if not self.skip_authentication: self.authenticate(request) + # Determine .scheme, .host and .port attributes + # For absolute-form requests, they are directly given in the request. + # For authority-form requests, we only need to determine the request scheme. + # For relative-form requests, we need to determine host and port as well. + if not request.scheme: + request.scheme = "https" if flow.server_conn and flow.server_conn.ssl_established else "http" + if not request.host: + # Host/Port Complication: In upstream mode, use the server we CONNECTed to, + # not the upstream proxy. + if flow.server_conn: + for s in flow.server_conn.state: + if s[0] == "http" and s[1]["state"] == "connect": + request.host, request.port = s[1]["host"], s[1]["port"] + if not request.host and flow.server_conn: + request.host, request.port = flow.server_conn.address.host, flow.server_conn.address.port + + # Now we can process the request. if request.form_in == "authority": if self.c.client_conn.ssl_established: raise http.HttpError(400, "Must not CONNECT on already encrypted connection") if self.expected_form_in == "absolute": - if not self.c.config.get_upstream_server: - self.c.set_server_address((request.host, request.port), - proxy.AddressPriority.FROM_PROTOCOL) + if not self.c.config.get_upstream_server: # Regular mode + self.c.set_server_address((request.host, request.port)) flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow self.c.establish_server_connection() self.c.client_conn.send( @@ -1131,34 +1065,123 @@ class HTTPHandler(ProtocolHandler): ('Proxy-agent: %s\r\n' % self.c.server_version) + '\r\n' ) - - if self.c.check_ignore_address(self.c.server_conn.address): - self.c.log("Ignore host: %s:%s" % self.c.server_conn.address(), "info") - TCPHandler(self.c).handle_messages() - return False - else: - if self.c.server_conn.address.port in self.c.config.ssl_ports: - self.ssl_upgrade() - self.skip_authentication = True - return True - else: + return self.process_connect_request(self.c.server_conn.address) + else: # upstream proxy mode return None + else: + pass # CONNECT should never occur if we don't expect absolute-form requests + elif request.form_in == self.expected_form_in: + + request.form_out = self.expected_form_out + if request.form_in == "absolute": if request.scheme != "http": raise http.HttpError(400, "Invalid request scheme: %s" % request.scheme) - self.determine_server_address(flow, request) - request.form_out = self.expected_form_out + if request.form_out == "relative": + self.c.set_server_address((request.host, request.port)) + flow.server_conn = self.c.server_conn + return None raise http.HttpError(400, "Invalid HTTP request form (expected: %s, got: %s)" % (self.expected_form_in, request.form_in)) - def determine_server_address(self, flow, request): - if request.form_in == "absolute": - self.c.set_server_address((request.host, request.port), - proxy.AddressPriority.FROM_PROTOCOL) - flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow + def process_server_address(self, flow): + # Depending on the proxy mode, server handling is entirely different + # We provide a mostly unified API to the user, which needs to be unfiddled here + # ( See also: https://github.com/mitmproxy/mitmproxy/issues/337 ) + address = netlib.tcp.Address((flow.request.host, flow.request.port)) + + ssl = (flow.request.scheme == "https") + + if self.c.config.http_form_in == self.c.config.http_form_out == "absolute": # Upstream Proxy mode + + # The connection to the upstream proxy may have a state we may need to take into account. + connected_to = None + for s in flow.server_conn.state: + if s[0] == "http" and s[1]["state"] == "connect": + connected_to = tcp.Address((s[1]["host"], s[1]["port"])) + + # We need to reconnect if the current flow either requires a (possibly impossible) + # change to the connection state, e.g. the host has changed but we already CONNECTed somewhere else. + needs_server_change = ( + ssl != self.c.server_conn.ssl_established + or + (connected_to and address != connected_to) # HTTP proxying is "stateless", CONNECT isn't. + ) + + if needs_server_change: + # force create new connection to the proxy server to reset state + self.live.change_server(self.c.server_conn.address, force=True) + if ssl: + send_connect_request(self.c.server_conn, address.host, address.port) + self.c.establish_ssl(server=True) + else: + # If we're not in upstream mode, we just want to update the host and possibly establish TLS. + self.live.change_server(address, ssl=ssl) # this is a no op if the addresses match. + + flow.server_conn = self.c.server_conn + + def send_response_to_client(self, flow): + if not flow.response.stream: + # no streaming: + # we already received the full response from the server and can send it to the client straight away. + self.c.client_conn.send(flow.response.assemble()) + else: + # streaming: + # First send the body and then transfer the response incrementally: + h = flow.response._assemble_head(preserve_transfer_encoding=True) + self.c.client_conn.send(h) + for chunk in http.read_http_body_chunked(self.c.server_conn.rfile, + flow.response.headers, + self.c.config.body_size_limit, flow.request.method, + flow.response.code, False, 4096): + for part in chunk: + self.c.client_conn.wfile.write(part) + self.c.client_conn.wfile.flush() + flow.response.timestamp_end = utils.timestamp() + + def check_close_connection(self, flow): + """ + Checks if the connection should be closed depending on the HTTP semantics. Returns True, if so. + """ + close_connection = ( + http.connection_close(flow.request.httpversion, flow.request.headers) or + http.connection_close(flow.response.httpversion, flow.response.headers) or + http.expected_http_body_size(flow.response.headers, False, flow.request.method, + flow.response.code) == -1) + if close_connection: + if flow.request.form_in == "authority" and flow.response.code == 200: + # Workaround for https://github.com/mitmproxy/mitmproxy/issues/313: + # Some proxies (e.g. Charles) send a CONNECT response with HTTP/1.0 and no Content-Length header + pass + else: + return True + return False + + def process_connect_request(self, address): + """ + Process a CONNECT request. + Returns True if the CONNECT request has been processed successfully. + Returns False, if the connection should be closed immediately. + """ + address = tcp.Address.wrap(address) + if self.c.check_ignore_address(address): + self.c.log("Ignore host: %s:%s" % address(), "info") + TCPHandler(self.c).handle_messages() + return False + else: + self.expected_form_in = "relative" + self.expected_form_out = "relative" + self.skip_authentication = True + + if address.port in self.c.config.ssl_ports: + self.c.log("Received CONNECT request to SSL port. Upgrading to SSL...", "debug") + self.c.establish_ssl(server=True, client=True) + self.c.log("Upgrade to SSL completed.", "debug") + + return True def authenticate(self, request): if self.c.config.authenticator: @@ -1178,33 +1201,29 @@ class RequestReplayThread(threading.Thread): threading.Thread.__init__(self) def run(self): + r = self.flow.request + form_out_backup = r.form_out try: - r = self.flow.request - form_out_backup = r.form_out - - r.form_out = self.config.http_form_out - server_address, server_ssl = False, False - if self.config.get_upstream_server: - try: - # this will fail in transparent mode - upstream_info = self.config.get_upstream_server(self.flow.client_conn) - server_ssl = upstream_info[1] - server_address = upstream_info[2:] - except proxy.ProxyError: - pass - if not server_address: - server_address = (r.get_host(False, self.flow), r.get_port(self.flow)) - - server = ServerConnection(server_address, None) - server.connect() - - if server_ssl or r.get_scheme(self.flow) == "https": - if self.config.http_form_out == "absolute": # form_out == absolute -> forward mode -> send CONNECT - send_connect_request(server, r.get_host(), r.get_port()) + # In all modes, we directly connect to the server displayed + if self.config.http_form_out == "absolute": # form_out == absolute -> forward mode + server_address = self.config.get_upstream_server(self.flow.client_conn)[2:] + server = ServerConnection(server_address) + server.connect() + if r.scheme == "https": + send_connect_request(server, r.host, r.port) + server.establish_ssl(self.config.clientcerts, sni=r.host) r.form_out = "relative" - server.establish_ssl(self.config.clientcerts, - self.flow.server_conn.sni) - server.send(r._assemble()) + else: + r.form_out = "absolute" + else: + server_address = (r.host, r.port) + server = ServerConnection(server_address) + server.connect() + if r.scheme == "https": + server.establish_ssl(self.config.clientcerts, sni=r.host) + r.form_out = "relative" + + server.send(r.assemble()) self.flow.response = HTTPResponse.from_stream(server.rfile, r.method, body_size_limit=self.config.body_size_limit) self.channel.ask("response", self.flow) diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py index a84b4061..ecad9d9e 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/protocol/primitives.py @@ -2,7 +2,6 @@ from __future__ import absolute_import import copy import netlib.tcp from .. import stateobject, utils, version -from ..proxy.primitives import AddressPriority from ..proxy.connection import ClientConnection, ServerConnection @@ -13,9 +12,9 @@ class Error(stateobject.SimpleStateObject): """ An Error. - This is distinct from an HTTP error response (say, a code 500), which - is represented by a normal Response object. This class is responsible - for indicating errors that fall outside of normal HTTP communications, + This is distinct from an protocol error response (say, a HTTP code 500), which + is represented by a normal HTTPResponse object. This class is responsible + for indicating errors that fall outside of normal protocol communications, like interrupted connections, timeouts, protocol errors. Exposes the following attributes: @@ -53,13 +52,17 @@ class Error(stateobject.SimpleStateObject): class Flow(stateobject.SimpleStateObject): + """ + A Flow is a collection of objects representing a single transaction. + This class is usually subclassed for each protocol, e.g. HTTPFlow. + """ def __init__(self, conntype, client_conn, server_conn, live=None): self.conntype = conntype self.client_conn = client_conn """@type: ClientConnection""" self.server_conn = server_conn """@type: ServerConnection""" - self.live = live # Used by flow.request.set_url to change the server address + self.live = live """@type: LiveConnection""" self.error = None @@ -118,6 +121,10 @@ class Flow(stateobject.SimpleStateObject): class ProtocolHandler(object): + """ + A ProtocolHandler implements an application-layer protocol, e.g. HTTP. + See: libmproxy.protocol.http.HTTPHandler + """ def __init__(self, c): self.c = c """@type: libmproxy.proxy.server.ConnectionHandler""" @@ -149,48 +156,53 @@ class ProtocolHandler(object): class LiveConnection(object): """ - This facade allows protocol handlers to interface with a live connection, - without requiring the expose the ConnectionHandler. + This facade allows interested parties (FlowMaster, inline scripts) to interface with a live connection, + without requiring to expose the internals of the ConnectionHandler. """ def __init__(self, c): - self._c = c + self.c = c """@type: libmproxy.proxy.server.ConnectionHandler""" + self._backup_server_conn = None + """@type: libmproxy.proxy.connection.ServerConnection""" - def change_server(self, address, ssl, persistent_change=False): + def change_server(self, address, ssl=False, force=False, persistent_change=False): address = netlib.tcp.Address.wrap(address) - if address != self._c.server_conn.address: + if force or address != self.c.server_conn.address or ssl != self.c.server_conn.ssl_established: - self._c.log("Change server connection: %s:%s -> %s:%s" % ( - self._c.server_conn.address.host, - self._c.server_conn.address.port, + self.c.log("Change server connection: %s:%s -> %s:%s [persistent: %s]" % ( + self.c.server_conn.address.host, + self.c.server_conn.address.port, address.host, - address.port + address.port, + persistent_change ), "debug") - if not hasattr(self, "_backup_server_conn"): - self._backup_server_conn = self._c.server_conn - self._c.server_conn = None + if not self._backup_server_conn: + self._backup_server_conn = self.c.server_conn + self.c.server_conn = None else: # This is at least the second temporary change. We can kill the current connection. - self._c.del_server_connection() + self.c.del_server_connection() - self._c.set_server_address(address, AddressPriority.MANUALLY_CHANGED) - self._c.establish_server_connection(ask=False) + self.c.set_server_address(address) + self.c.establish_server_connection(ask=False) if ssl: - self._c.establish_ssl(server=True) - if hasattr(self, "_backup_server_conn") and persistent_change: - del self._backup_server_conn + self.c.establish_ssl(server=True) + if persistent_change: + self._backup_server_conn = None def restore_server(self): - if not hasattr(self, "_backup_server_conn"): + # TODO: Similar to _backup_server_conn, introduce _cache_server_conn, which keeps the changed connection open + # This may be beneficial if a user is rewriting all requests from http to https or similar. + if not self._backup_server_conn: return - self._c.log("Restore original server connection: %s:%s -> %s:%s" % ( - self._c.server_conn.address.host, - self._c.server_conn.address.port, + self.c.log("Restore original server connection: %s:%s -> %s:%s" % ( + self.c.server_conn.address.host, + self.c.server_conn.address.port, self._backup_server_conn.address.host, self._backup_server_conn.address.port ), "debug") - self._c.del_server_connection() - self._c.server_conn = self._backup_server_conn - del self._backup_server_conn
\ No newline at end of file + self.c.del_server_connection() + self.c.server_conn = self._backup_server_conn + self._backup_server_conn = None
\ No newline at end of file diff --git a/libmproxy/protocol/tcp.py b/libmproxy/protocol/tcp.py index 57a48ab9..990c502a 100644 --- a/libmproxy/protocol/tcp.py +++ b/libmproxy/protocol/tcp.py @@ -59,11 +59,11 @@ class TCPHandler(ProtocolHandler): # if one of the peers is over SSL, we need to send bytes/strings if not src.ssl_established: # only ssl to dst, i.e. we revc'd into buf but need bytes/string now. contents = buf[:size].tobytes() - # self.c.log("%s %s\r\n%s" % (direction, dst_str, cleanBin(contents)), "debug") + self.c.log("%s %s\r\n%s" % (direction, dst_str, cleanBin(contents)), "debug") dst.connection.send(contents) else: # socket.socket.send supports raw bytearrays/memoryviews - # self.c.log("%s %s\r\n%s" % (direction, dst_str, cleanBin(buf.tobytes())), "debug") + self.c.log("%s %s\r\n%s" % (direction, dst_str, cleanBin(buf.tobytes())), "debug") dst.connection.send(buf[:size]) except socket.error as e: self.c.log("TCP connection closed unexpectedly.", "debug") diff --git a/libmproxy/proxy/__init__.py b/libmproxy/proxy/__init__.py index f5d6a2d0..e4c20030 100644 --- a/libmproxy/proxy/__init__.py +++ b/libmproxy/proxy/__init__.py @@ -1 +1,2 @@ -from .primitives import *
\ No newline at end of file +from .primitives import * +from .config import ProxyConfig diff --git a/libmproxy/proxy/config.py b/libmproxy/proxy/config.py index 6d4c078b..ea815c69 100644 --- a/libmproxy/proxy/config.py +++ b/libmproxy/proxy/config.py @@ -1,8 +1,8 @@ from __future__ import absolute_import import os -from .. import utils, platform import re from netlib import http_auth, certutils +from .. import utils, platform from .primitives import ConstUpstreamServerResolver, TransparentUpstreamServerResolver TRANSPARENT_SSL_PORTS = [443, 8443] @@ -11,7 +11,7 @@ CONF_DIR = "~/.mitmproxy" class ProxyConfig: - def __init__(self, confdir=CONF_DIR, clientcerts=None, + def __init__(self, confdir=CONF_DIR, ca_file=None, clientcerts=None, no_upstream_cert=False, body_size_limit=None, mode=None, upstream_server=None, http_form_in=None, http_form_out=None, authenticator=None, ignore=[], @@ -44,7 +44,7 @@ class ProxyConfig: self.ignore = [re.compile(i, re.IGNORECASE) for i in ignore] self.authenticator = authenticator self.confdir = os.path.expanduser(confdir) - self.ca_file = os.path.join(self.confdir, CONF_BASENAME + "-ca.pem") + self.ca_file = ca_file or os.path.join(self.confdir, CONF_BASENAME + "-ca.pem") self.certstore = certutils.CertStore.from_store(self.confdir, CONF_BASENAME) for spec, cert in certs: self.certstore.add_cert_file(spec, cert) diff --git a/libmproxy/proxy/connection.py b/libmproxy/proxy/connection.py index d99ffa9b..de8e20d8 100644 --- a/libmproxy/proxy/connection.py +++ b/libmproxy/proxy/connection.py @@ -72,13 +72,10 @@ class ClientConnection(tcp.BaseHandler, stateobject.SimpleStateObject): class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject): - def __init__(self, address, priority): + def __init__(self, address): tcp.TCPClient.__init__(self, address) - self.priority = priority self.state = [] # a list containing (conntype, state) tuples - self.peername = None - self.sockname = None self.timestamp_start = None self.timestamp_end = None self.timestamp_tcp_setup = None @@ -99,8 +96,6 @@ class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject): _stateobject_attributes = dict( state=list, - peername=tuple, - sockname=tuple, timestamp_start=float, timestamp_end=float, timestamp_tcp_setup=float, @@ -115,9 +110,10 @@ class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject): def _get_state(self): d = super(ServerConnection, self)._get_state() d.update( - address={"address": self.address(), "use_ipv6": self.address.use_ipv6}, - source_address= {"address": self.source_address(), - "use_ipv6": self.source_address.use_ipv6} if self.source_address else None, + address={"address": self.address(), + "use_ipv6": self.address.use_ipv6}, + source_address= ({"address": self.source_address(), + "use_ipv6": self.source_address.use_ipv6} if self.source_address else None), cert=self.cert.to_pem() if self.cert else None ) return d @@ -131,7 +127,7 @@ class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject): @classmethod def _from_state(cls, state): - f = cls(tuple(), None) + f = cls(tuple()) f._load_state(state) return f @@ -141,8 +137,6 @@ class ServerConnection(tcp.TCPClient, stateobject.SimpleStateObject): def connect(self): self.timestamp_start = utils.timestamp() tcp.TCPClient.connect(self) - self.peername = self.connection.getpeername() - self.sockname = self.connection.getsockname() self.timestamp_tcp_setup = utils.timestamp() def send(self, message): diff --git a/libmproxy/proxy/primitives.py b/libmproxy/proxy/primitives.py index e09f23e4..8c674381 100644 --- a/libmproxy/proxy/primitives.py +++ b/libmproxy/proxy/primitives.py @@ -45,19 +45,6 @@ class TransparentUpstreamServerResolver(UpstreamServerResolver): return [ssl, ssl] + list(dst) -class AddressPriority(object): - """ - Enum that signifies the priority of the given address when choosing the destination host. - Higher is better (None < i) - """ - MANUALLY_CHANGED = 3 - """user changed the target address in the ui""" - FROM_SETTINGS = 2 - """upstream server from arguments (reverse proxy, upstream proxy or from transparent resolver)""" - FROM_PROTOCOL = 1 - """derived from protocol (e.g. absolute-form http requests)""" - - class Log: def __init__(self, msg, level="info"): self.msg = msg diff --git a/libmproxy/proxy/server.py b/libmproxy/proxy/server.py index 092eae54..aacc908e 100644 --- a/libmproxy/proxy/server.py +++ b/libmproxy/proxy/server.py @@ -1,11 +1,10 @@ from __future__ import absolute_import -import re import socket from OpenSSL import SSL from netlib import tcp -from .primitives import ProxyServerError, Log, ProxyError, AddressPriority +from .primitives import ProxyServerError, Log, ProxyError from .connection import ClientConnection, ServerConnection from ..protocol.handle import protocol_handler from .. import version @@ -76,7 +75,7 @@ class ConnectionHandler: client_ssl, server_ssl = False, False if self.config.get_upstream_server: upstream_info = self.config.get_upstream_server(self.client_conn.connection) - self.set_server_address(upstream_info[2:], AddressPriority.FROM_SETTINGS) + self.set_server_address(upstream_info[2:]) client_ssl, server_ssl = upstream_info[:2] if self.check_ignore_address(self.server_conn.address): self.log("Ignore host: %s:%s" % self.server_conn.address(), "info") @@ -129,27 +128,22 @@ class ConnectionHandler: else: return False - def set_server_address(self, address, priority): + def set_server_address(self, address): """ Sets a new server address with the given priority. Does not re-establish either connection or SSL handshake. """ address = tcp.Address.wrap(address) - if self.server_conn: - if self.server_conn.priority > priority: - self.log("Attempt to change server address, " - "but priority is too low (is: %s, got: %s)" % ( - self.server_conn.priority, priority), "debug") - return - if self.server_conn.address == address: - self.server_conn.priority = priority # Possibly increase priority - return + # Don't reconnect to the same destination. + if self.server_conn and self.server_conn.address == address: + return + if self.server_conn: self.del_server_connection() self.log("Set new server address: %s:%s" % (address.host, address.port), "debug") - self.server_conn = ServerConnection(address, priority) + self.server_conn = ServerConnection(address) def establish_server_connection(self, ask=True): """ @@ -212,12 +206,11 @@ class ConnectionHandler: def server_reconnect(self): address = self.server_conn.address had_ssl = self.server_conn.ssl_established - priority = self.server_conn.priority state = self.server_conn.state sni = self.sni self.log("(server reconnect follows)", "debug") self.del_server_connection() - self.set_server_address(address, priority) + self.set_server_address(address) self.establish_server_connection() for s in state: diff --git a/libmproxy/stateobject.py b/libmproxy/stateobject.py index 3437b90e..6fb73c24 100644 --- a/libmproxy/stateobject.py +++ b/libmproxy/stateobject.py @@ -21,6 +21,9 @@ class StateObject(object): except AttributeError: # we may compare with something that's not a StateObject return False + def __ne__(self, other): + return not self.__eq__(other) + class SimpleStateObject(StateObject): """ |