diff options
Diffstat (limited to 'libmproxy/flow.py')
| -rw-r--r-- | libmproxy/flow.py | 135 |
1 files changed, 70 insertions, 65 deletions
diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 6aa26f4a..e5fdf424 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -2,16 +2,17 @@ This module provides more sophisticated flow tracking and provides filtering and interception facilities. """ from __future__ import absolute_import -import base64 -import hashlib, Cookie, cookielib, re, threading -import os -import flask -import requests +import hashlib +import Cookie +import cookielib +import re from netlib import odict, wsgi import netlib.http -from . import controller, protocol, tnetstring, filt, script, version, app +from . import controller, protocol, tnetstring, filt, script, version +from .onboarding import app from .protocol import http, handle -from .proxy.config import parse_host_pattern +from .proxy.config import HostMatcher +import urlparse ODict = odict.ODict ODictCaseless = odict.ODictCaseless @@ -150,10 +151,13 @@ class StreamLargeBodies(object): def run(self, flow, is_request): r = flow.request if is_request else flow.response code = flow.response.code if flow.response else None - expected_size = netlib.http.expected_http_body_size(r.headers, is_request, flow.request.method, code) + expected_size = netlib.http.expected_http_body_size( + r.headers, is_request, flow.request.method, code + ) if not (0 <= expected_size <= self.max_size): r.stream = True + class ClientPlaybackState: def __init__(self, flows, exit): self.flows, self.exit = flows, exit @@ -190,12 +194,12 @@ class ClientPlaybackState: class ServerPlaybackState: - def __init__(self, headers, flows, exit, nopop): + def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content): """ headers: Case-insensitive list of request headers that should be included in request-response matching. """ - self.headers, self.exit, self.nopop = headers, exit, nopop + self.headers, self.exit, self.nopop, self.ignore_params, self.ignore_content = headers, exit, nopop, ignore_params, ignore_content self.fmap = {} for i in flows: if i.response: @@ -210,14 +214,29 @@ class ServerPlaybackState: Calculates a loose hash of the flow request. """ r = flow.request + + _, _, path, _, query, _ = urlparse.urlparse(r.url) + queriesArray = urlparse.parse_qsl(query) + + filtered = [] + for p in queriesArray: + if p[0] not in self.ignore_params: + filtered.append(p) + key = [ str(r.host), str(r.port), str(r.scheme), str(r.method), - str(r.path), - str(r.content), - ] + str(path), + ] + if not self.ignore_content: + key.append(str(r.content)) + + for p in filtered: + key.append(p[0]) + key.append(p[1]) + if self.headers: hdrs = [] for i in self.headers: @@ -446,43 +465,19 @@ class FlowMaster(controller.Master): self.refresh_server_playback = False self.replacehooks = ReplaceHooks() self.setheaders = SetHeaders() + self.replay_ignore_params = False + self.replay_ignore_content = None + self.stream = None self.apps = AppRegistry() - def start_app(self, host, port, external): - if not external: - self.apps.add( - app.mapp, - host, - port - ) - else: - @app.mapp.before_request - def patch_environ(*args, **kwargs): - flask.request.environ["mitmproxy.master"] = self - - # the only absurd way to shut down a flask/werkzeug server. - # http://flask.pocoo.org/snippets/67/ - shutdown_secret = base64.b32encode(os.urandom(30)) - - @app.mapp.route('/shutdown/<secret>') - def shutdown(secret): - if secret == shutdown_secret: - flask.request.environ.get('werkzeug.server.shutdown')() - - # Workaround: Monkey-patch shutdown function to stop the app. - # Improve this when we switch werkzeugs http server for something useful. - _shutdown = self.shutdown - def _shutdownwrap(): - _shutdown() - requests.get("http://%s:%s/shutdown/%s" % (host, port, shutdown_secret)) - self.shutdown = _shutdownwrap - - threading.Thread(target=app.mapp.run, kwargs={ - "use_reloader": False, - "host": host, - "port": port}).start() + def start_app(self, host, port): + self.apps.add( + app.mapp, + host, + port + ) def add_event(self, e, level="info"): """ @@ -520,11 +515,17 @@ class FlowMaster(controller.Master): for script in self.scripts: self.run_single_script_hook(script, name, *args, **kwargs) - def get_ignore(self): - return [i.pattern for i in self.server.config.ignore] + def get_ignore_filter(self): + return self.server.config.check_ignore.patterns + + def set_ignore_filter(self, host_patterns): + self.server.config.check_ignore = HostMatcher(host_patterns) - def set_ignore(self, ignore): - self.server.config.ignore = parse_host_pattern(ignore) + def get_tcp_filter(self): + return self.server.config.check_tcp.patterns + + def set_tcp_filter(self, host_patterns): + self.server.config.check_tcp = HostMatcher(host_patterns) def set_stickycookie(self, txt): if txt: @@ -563,12 +564,14 @@ class FlowMaster(controller.Master): def stop_client_playback(self): self.client_playback = None - def start_server_playback(self, flows, kill, headers, exit, nopop): + def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params, ignore_content): """ flows: List of flows. kill: Boolean, should we kill requests not part of the replay? + ignore_params: list of parameters to ignore in server replay + ignore_content: true if request content should be ignored in server replay """ - self.server_playback = ServerPlaybackState(headers, flows, exit, nopop) + self.server_playback = ServerPlaybackState(headers, flows, exit, nopop, ignore_params, ignore_content) self.kill_nonreplay = kill def stop_server_playback(self): @@ -585,7 +588,7 @@ class FlowMaster(controller.Master): rflow = self.server_playback.next_flow(flow) if not rflow: return None - response = http.HTTPResponse._from_state(rflow.response._get_state()) + response = http.HTTPResponse.from_state(rflow.response.get_state()) response.is_replay = True if self.refresh_server_playback: response.refresh() @@ -595,7 +598,7 @@ class FlowMaster(controller.Master): return True return None - def tick(self, q): + def tick(self, q, timeout): if self.client_playback: e = [ self.client_playback.done(), @@ -604,9 +607,9 @@ class FlowMaster(controller.Master): ] if all(e): self.shutdown() - self.client_playback.tick(self) + self.client_playback.tick(self, timeout) - return controller.Master.tick(self, q) + return controller.Master.tick(self, q, timeout) def duplicate_flow(self, f): return self.load_flow(f.copy()) @@ -664,6 +667,8 @@ class FlowMaster(controller.Master): """ Returns None if successful, or error message if not. """ + if f.live: + return "Can't replay request which is still live..." if f.intercepting: return "Can't replay while intercepting..." if f.request.content == http.CONTENT_MISSING: @@ -676,11 +681,11 @@ class FlowMaster(controller.Master): f.error = None self.process_new_request(f) rt = http.RequestReplayThread( - self.server.config, - f, - self.masterq, - self.should_exit - ) + self.server.config, + f, + self.masterq, + self.should_exit + ) rt.start() # pragma: no cover if block: rt.join() @@ -732,7 +737,7 @@ class FlowMaster(controller.Master): self.stream_large_bodies.run(f, False) f.reply() - return f + return f def handle_response(self, f): self.state.add_response(f) @@ -769,7 +774,7 @@ class FlowWriter: self.fo = fo def add(self, flow): - d = flow._get_state() + d = flow.get_state() tnetstring.dump(d, self.fo) @@ -795,7 +800,7 @@ class FlowReader: v = ".".join(str(i) for i in data["version"]) raise FlowReadError("Incompatible serialized data version: %s"%v) off = self.fo.tell() - yield handle.protocols[data["conntype"]]["flow"]._from_state(data) + yield handle.protocols[data["type"]]["flow"].from_state(data) except ValueError, v: # Error is due to EOF if self.fo.tell() == off and self.fo.read() == '': @@ -811,5 +816,5 @@ class FilteredFlowWriter: def add(self, f): if self.filt and not f.match(self.filt): return - d = f._get_state() + d = f.get_state() tnetstring.dump(d, self.fo) |