From b0cfeff06d9dd99a16dfae19c5df3c73c5864fb9 Mon Sep 17 00:00:00 2001 From: Maximilian Hils Date: Wed, 3 Sep 2014 16:57:56 +0200 Subject: fix #341 - work on flows instead of request/response internally. --- libmproxy/app.py | 8 +-- libmproxy/console/__init__.py | 26 +++++----- libmproxy/dump.py | 32 ++++++------ libmproxy/filt.py | 4 +- libmproxy/flow.py | 106 ++++++++++++++++++--------------------- libmproxy/protocol/http.py | 102 +++++++++++++++++-------------------- libmproxy/protocol/primitives.py | 23 +-------- libmproxy/script.py | 11 ++-- 8 files changed, 132 insertions(+), 180 deletions(-) (limited to 'libmproxy') diff --git a/libmproxy/app.py b/libmproxy/app.py index 9941d6ea..ed7ec72a 100644 --- a/libmproxy/app.py +++ b/libmproxy/app.py @@ -1,7 +1,7 @@ from __future__ import absolute_import import flask -import os.path, os -from . import proxy +import os +from .proxy import config mapp = flask.Flask(__name__) mapp.debug = True @@ -18,12 +18,12 @@ def index(): @mapp.route("/cert/pem") def certs_pem(): - p = os.path.join(master().server.config.confdir, proxy.config.CONF_BASENAME + "-ca-cert.pem") + p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.pem") return flask.Response(open(p, "rb").read(), mimetype='application/x-x509-ca-cert') @mapp.route("/cert/p12") def certs_p12(): - p = os.path.join(master().server.config.confdir, proxy.config.CONF_BASENAME + "-ca-cert.p12") + p = os.path.join(master().server.config.confdir, config.CONF_BASENAME + "-ca-cert.p12") return flask.Response(open(p, "rb").read(), mimetype='application/x-pkcs12') diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 1325aae5..a5920915 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -268,8 +268,8 @@ class ConsoleState(flow.State): d = self.flowsettings.get(flow, {}) return d.get(key, default) - def add_request(self, req): - f = flow.State.add_request(self, req) + def add_request(self, f): + flow.State.add_request(self, f) if self.focus is None: self.set_focus(0) elif self.follow_focus: @@ -996,11 +996,11 @@ class ConsoleMaster(flow.FlowMaster): if hasattr(self.statusbar, "refresh_flow"): self.statusbar.refresh_flow(c) - def process_flow(self, f, r): + def process_flow(self, f): if self.state.intercept and f.match(self.state.intercept) and not f.request.is_replay: f.intercept() else: - r.reply() + f.reply() self.sync_list_view() self.refresh_flow(f) @@ -1022,20 +1022,20 @@ class ConsoleMaster(flow.FlowMaster): self.eventlist.set_focus(len(self.eventlist)-1) # Handlers - def handle_error(self, r): - f = flow.FlowMaster.handle_error(self, r) + def handle_error(self, f): + f = flow.FlowMaster.handle_error(self, f) if f: - self.process_flow(f, r) + self.process_flow(f) return f - def handle_request(self, r): - f = flow.FlowMaster.handle_request(self, r) + def handle_request(self, f): + f = flow.FlowMaster.handle_request(self, f) if f: - self.process_flow(f, r) + self.process_flow(f) return f - def handle_response(self, r): - f = flow.FlowMaster.handle_response(self, r) + def handle_response(self, f): + f = flow.FlowMaster.handle_response(self, f) if f: - self.process_flow(f, r) + self.process_flow(f) return f diff --git a/libmproxy/dump.py b/libmproxy/dump.py index aeb34cc3..8ecd56e7 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -50,13 +50,13 @@ def str_response(resp): return r -def str_request(req, showhost): - if req.flow.client_conn: - c = req.flow.client_conn.address.host +def str_request(f, showhost): + if f.client_conn: + c = f.client_conn.address.host else: c = "[replay]" - r = "%s %s %s"%(c, req.method, req.get_url(showhost)) - if req.stickycookie: + r = "%s %s %s"%(c, f.request.method, f.request.get_url(showhost, f)) + if f.request.stickycookie: r = "[stickycookie] " + r return r @@ -185,16 +185,16 @@ class DumpMaster(flow.FlowMaster): result = " << %s"%f.error.msg if self.o.flow_detail == 1: - print >> self.outfile, str_request(f.request, self.showhost) + print >> self.outfile, str_request(f, self.showhost) print >> self.outfile, result elif self.o.flow_detail == 2: - print >> self.outfile, str_request(f.request, self.showhost) + print >> self.outfile, str_request(f, self
/*  Math routines for Win32
    Copyright (C) 2005 - 2015 Tristan Gingold.

    GHDL is free software; you can redistribute it and/or modify it under
    the terms of the GNU General Public License as published by the Free
    Software Foundation; either version 2, or (at your option) any later
    version.

    GHDL is distributed in the hope that it will be useful, but WITHOUT ANY
    WARRANTY; without even the implied warranty of MERCHANTABILITY or
    FITNESS FOR A PARTICULAR PURPOSE.  See the GNU General Public License
    for more details.

    You should have received a copy of the GNU General Public License
    along with GCC; see the file COPYING.  If not, write to the Free
    Software Foundation, 59 Temple Place - Suite 330, Boston, MA
    02111-1307, USA.

    As a special exception, if other files instantiate generics from this
    unit, or you link this unit with other files to produce an executable,
    this unit does not by itself cause the resulting executable to be
    covered by the GNU General Public License. This exception does not
    however invalidate any other reasons why the executable file might be
    covered by the GNU Public License.
*/

#include <math.h>

double acosh (double x)
{
  return log (x + sqrt (x*x - 1));
}

double asinh (double x)
{
  return log (x + sqrt (x*x + 1));
}

double atanh (double x)
{
  return log ((1 + x) / (1 - x)) / 2;
}

#ifndef WITH_GNAT_RUN_TIME
void __gnat_raise_storage_error(void)
{
   abort ();
}

void __gnat_raise_program_error(void)
{
   abort ();
}
#endif
generated by cgit v1.2.3 (git 2.25.1) at 2025-09-10 07:58:29 +0000
if self.client_playback: - self.client_playback.clear(f) - self.process_new_response(f) - if self.stream: - self.stream.add(f) - else: - r.reply() + def handle_response(self, f): + self.state.add_response(f) + self.replacehooks.run(f) + self.setheaders.run(f) + self.run_script_hook("response", f) + if self.client_playback: + self.client_playback.clear(f) + self.process_new_response(f) + if self.stream: + self.stream.add(f) return f def shutdown(self): diff --git a/libmproxy/protocol/http.py b/libmproxy/protocol/http.py index 658c08ed..3f9eecb3 100644 --- a/libmproxy/protocol/http.py +++ b/libmproxy/protocol/http.py @@ -77,9 +77,6 @@ class HTTPMessage(stateobject.SimpleStateObject): self.timestamp_start = timestamp_start if timestamp_start is not None else utils.timestamp() self.timestamp_end = timestamp_end if timestamp_end is not None else utils.timestamp() - self.flow = None # will usually be set by the flow backref mixin - """@type: HTTPFlow""" - _stateobject_attributes = dict( httpversion=tuple, headers=ODictCaseless, @@ -346,10 +343,10 @@ class HTTPRequest(HTTPMessage): del headers[k] if headers["Upgrade"] == ["h2c"]: # Suppress HTTP2 https://http2.github.io/http2-spec/index.html#discover-http del headers["Upgrade"] - if not 'host' in headers: + if not 'host' in headers and self.scheme and self.host and self.port: headers["Host"] = [utils.hostport(self.scheme, - self.host or self.flow.server_conn.address.host, - self.port or self.flow.server_conn.address.port)] + self.host, + self.port)] if self.content: headers["Content-Length"] = [str(len(self.content))] @@ -429,16 +426,16 @@ class HTTPRequest(HTTPMessage): self.headers["Content-Type"] = [HDR_FORM_URLENCODED] self.content = utils.urlencode(odict.lst) - def get_path_components(self): + def get_path_components(self, f): """ Returns the path components of the URL as a list of strings. Components are unquoted. """ - _, _, path, _, _, _ = urlparse.urlparse(self.get_url()) + _, _, path, _, _, _ = urlparse.urlparse(self.get_url(False, f)) return [urllib.unquote(i) for i in path.split("/") if i] - def set_path_components(self, lst): + def set_path_components(self, lst, f): """ Takes a list of strings, and sets the path component of the URL. @@ -446,27 +443,27 @@ class HTTPRequest(HTTPMessage): """ lst = [urllib.quote(i, safe="") for i in lst] path = "/" + "/".join(lst) - scheme, netloc, _, params, query, fragment = urlparse.urlparse(self.get_url()) - self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment])) + scheme, netloc, _, params, query, fragment = urlparse.urlparse(self.get_url(False, f)) + self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment]), f) - def get_query(self): + def get_query(self, f): """ Gets the request query string. Returns an ODict object. """ - _, _, _, _, query, _ = urlparse.urlparse(self.get_url()) + _, _, _, _, query, _ = urlparse.urlparse(self.get_url(False, f)) if query: return ODict(utils.urldecode(query)) return ODict([]) - def set_query(self, odict): + def set_query(self, odict, f): """ Takes an ODict object, and sets the request query string. """ - scheme, netloc, path, params, _, fragment = urlparse.urlparse(self.get_url()) + scheme, netloc, path, params, _, fragment = urlparse.urlparse(self.get_url(False, f)) query = utils.urlencode(odict.lst) - self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment])) + self.set_url(urlparse.urlunparse([scheme, netloc, path, params, query, fragment]), f) - def get_host(self, hostheader=False): + def get_host(self, hostheader, flow): """ Heuristic to get the host of the request. @@ -484,16 +481,16 @@ class HTTPRequest(HTTPMessage): if self.host: host = self.host else: - for s in self.flow.server_conn.state: + for s in flow.server_conn.state: if s[0] == "http" and s[1]["state"] == "connect": host = s[1]["host"] break if not host: - host = self.flow.server_conn.address.host + host = flow.server_conn.address.host host = host.encode("idna") return host - def get_scheme(self): + def get_scheme(self, flow): """ Returns the request port, either from the request itself or from the flow's server connection """ @@ -501,20 +498,20 @@ class HTTPRequest(HTTPMessage): return self.scheme if self.form_out == "authority": # On SSLed connections, the original CONNECT request is still unencrypted. return "http" - return "https" if self.flow.server_conn.ssl_established else "http" + return "https" if flow.server_conn.ssl_established else "http" - def get_port(self): + def get_port(self, flow): """ Returns the request port, either from the request itself or from the flow's server connection """ if self.port: return self.port - for s in self.flow.server_conn.state: + for s in flow.server_conn.state: if s[0] == "http" and s[1].get("state") == "connect": return s[1]["port"] - return self.flow.server_conn.address.port + return flow.server_conn.address.port - def get_url(self, hostheader=False): + def get_url(self, hostheader, flow): """ Returns a URL string, constructed from the Request's URL components. @@ -522,13 +519,13 @@ class HTTPRequest(HTTPMessage): Host header to construct the URL. """ if self.form_out == "authority": # upstream proxy mode - return "%s:%s" % (self.get_host(hostheader), self.get_port()) - return utils.unparse_url(self.get_scheme(), - self.get_host(hostheader), - self.get_port(), + return "%s:%s" % (self.get_host(hostheader, flow), self.get_port(flow)) + return utils.unparse_url(self.get_scheme(flow), + self.get_host(hostheader, flow), + self.get_port(flow), self.path).encode('ascii') - def set_url(self, url): + def set_url(self, url, flow): """ Parses a URL specification, and updates the Request's information accordingly. @@ -543,14 +540,14 @@ class HTTPRequest(HTTPMessage): self.path = path - if host != self.get_host() or port != self.get_port(): - if self.flow.live: - self.flow.live.change_server((host, port), ssl=is_ssl) + if host != self.get_host(False, flow) or port != self.get_port(flow): + if flow.live: + flow.live.change_server((host, port), ssl=is_ssl) else: # There's not live server connection, we're just changing the attributes here. - self.flow.server_conn = ServerConnection((host, port), + flow.server_conn = ServerConnection((host, port), proxy.AddressPriority.MANUALLY_CHANGED) - self.flow.server_conn.ssl_established = is_ssl + flow.server_conn.ssl_established = is_ssl # If this is an absolute request, replace the attributes on the request object as well. if self.host: @@ -802,8 +799,6 @@ class HTTPFlow(Flow): self.intercepting = False # FIXME: Should that rather be an attribute of Flow? - _backrefattr = Flow._backrefattr + ("request", "response") - _stateobject_attributes = Flow._stateobject_attributes.copy() _stateobject_attributes.update( request=HTTPRequest, @@ -855,13 +850,10 @@ class HTTPFlow(Flow): Kill this request. """ self.error = Error("Connection killed") - self.error.reply = controller.DummyReply() - if self.request and not self.request.reply.acked: - self.request.reply(KILL) - elif self.response and not self.response.reply.acked: - self.response.reply(KILL) - master.handle_error(self.error) self.intercepting = False + self.reply(KILL) + self.reply = controller.DummyReply() + master.handle_error(self) def intercept(self): """ @@ -874,12 +866,8 @@ class HTTPFlow(Flow): """ Continue with the flow - called after an intercept(). """ - if self.request: - if not self.request.reply.acked: - self.request.reply() - elif self.response and not self.response.reply.acked: - self.response.reply() - self.intercepting = False + self.intercepting = False + self.reply() def replace(self, pattern, repl, *args, **kwargs): """ @@ -961,7 +949,7 @@ class HTTPHandler(ProtocolHandler): # in an Error object that has an attached request that has not been # sent through to the Master. flow.request = req - request_reply = self.c.channel.ask("request", flow.request) + request_reply = self.c.channel.ask("request", flow) self.determine_server_address(flow, flow.request) # The inline script may have changed request.host flow.server_conn = self.c.server_conn # Update server_conn attribute on the flow @@ -976,7 +964,7 @@ class HTTPHandler(ProtocolHandler): flow.response = self.get_response_from_server(flow.request, include_body=False) # call the appropriate script hook - this is an opportunity for an inline script to set flow.stream = True - self.c.channel.ask("responseheaders", flow.response) + self.c.channel.ask("responseheaders", flow) # now get the rest of the request body, if body still needs to be read but not streaming this response if flow.response.stream: @@ -991,7 +979,7 @@ class HTTPHandler(ProtocolHandler): flow.server_conn = self.c.server_conn self.c.log("response", "debug", [flow.response._assemble_first_line()]) - response_reply = self.c.channel.ask("response", flow.response) + response_reply = self.c.channel.ask("response", flow) if response_reply is None or response_reply == KILL: return False @@ -1079,7 +1067,7 @@ class HTTPHandler(ProtocolHandler): # TODO: no flows without request or with both request and response at the moment. if flow.request and not flow.response: flow.error = Error(message) - self.c.channel.ask("error", flow.error) + self.c.channel.ask("error", flow) try: code = getattr(error, "code", 502) @@ -1204,12 +1192,12 @@ class RequestReplayThread(threading.Thread): except proxy.ProxyError: pass if not server_address: - server_address = (r.get_host(), r.get_port()) + server_address = (r.get_host(False, self.flow), r.get_port(self.flow)) server = ServerConnection(server_address, None) server.connect() - if server_ssl or r.get_scheme() == "https": + if server_ssl or r.get_scheme(self.flow) == "https": if self.config.http_form_out == "absolute": # form_out == absolute -> forward mode -> send CONNECT send_connect_request(server, r.get_host(), r.get_port()) r.form_out = "relative" @@ -1218,9 +1206,9 @@ class RequestReplayThread(threading.Thread): server.send(r._assemble()) self.flow.response = HTTPResponse.from_stream(server.rfile, r.method, body_size_limit=self.config.body_size_limit) - self.channel.ask("response", self.flow.response) + self.channel.ask("response", self.flow) except (proxy.ProxyError, http.HttpError, tcp.NetLibError), v: self.flow.error = Error(repr(v)) - self.channel.ask("error", self.flow.error) + self.channel.ask("error", self.flow) finally: r.form_out = form_out_backup \ No newline at end of file diff --git a/libmproxy/protocol/primitives.py b/libmproxy/protocol/primitives.py index a227d904..a84b4061 100644 --- a/libmproxy/protocol/primitives.py +++ b/libmproxy/protocol/primitives.py @@ -9,24 +9,6 @@ from ..proxy.connection import ClientConnection, ServerConnection KILL = 0 # const for killed requests -class BackreferenceMixin(object): - """ - If an attribute from the _backrefattr tuple is set, - this mixin sets a reference back on the attribute object. - Example: - e = Error() - f = Flow() - f.error = e - assert f is e.flow - """ - _backrefattr = tuple() - - def __setattr__(self, key, value): - super(BackreferenceMixin, self).__setattr__(key, value) - if key in self._backrefattr and value is not None: - setattr(value, self._backrefname, self) - - class Error(stateobject.SimpleStateObject): """ An Error. @@ -70,7 +52,7 @@ class Error(stateobject.SimpleStateObject): return c -class Flow(stateobject.SimpleStateObject, BackreferenceMixin): +class Flow(stateobject.SimpleStateObject): def __init__(self, conntype, client_conn, server_conn, live=None): self.conntype = conntype self.client_conn = client_conn @@ -84,9 +66,6 @@ class Flow(stateobject.SimpleStateObject, BackreferenceMixin): """@type: Error""" self._backup = None - _backrefattr = ("error",) - _backrefname = "flow" - _stateobject_attributes = dict( error=Error, client_conn=ClientConnection, diff --git a/libmproxy/script.py b/libmproxy/script.py index e582c4e8..706d84d5 100644 --- a/libmproxy/script.py +++ b/libmproxy/script.py @@ -125,13 +125,8 @@ def _handle_concurrent_reply(fn, o, *args, **kwargs): def concurrent(fn): - if fn.func_name in ["request", "response", "error"]: - def _concurrent(ctx, flow): - r = getattr(flow, fn.func_name) - _handle_concurrent_reply(fn, r, ctx, flow) - return _concurrent - elif fn.func_name in ["clientconnect", "serverconnect", "clientdisconnect"]: - def _concurrent(ctx, conn): - _handle_concurrent_reply(fn, conn, ctx, conn) + if fn.func_name in ("request", "response", "error", "clientconnect", "serverconnect", "clientdisconnect"): + def _concurrent(ctx, obj): + _handle_concurrent_reply(fn, obj, ctx, obj) return _concurrent raise NotImplementedError("Concurrent decorator not supported for this method.") -- cgit v1.2.3