diff options
| -rw-r--r-- | libmproxy/console.py | 6 | ||||
| -rw-r--r-- | libmproxy/playback.py | 129 | ||||
| -rw-r--r-- | libmproxy/record.py | 68 | ||||
| -rw-r--r-- | libmproxy/recorder.py | 276 |
4 files changed, 0 insertions, 479 deletions
diff --git a/libmproxy/console.py b/libmproxy/console.py index fbd3617a..187502a6 100644 --- a/libmproxy/console.py +++ b/libmproxy/console.py @@ -19,7 +19,6 @@ import cStringIO import urwid.curses_display import urwid import controller, utils, filt, proxy, flow -import recorder class Stop(Exception): pass @@ -731,9 +730,6 @@ class ConsoleState(flow.State): self.set_focus(self.focus) return ret - def start_recording(self, recorder): - self.store = recorder - def get_focus(self): if not self.view or self.focus is None: return None, None @@ -812,8 +808,6 @@ class ConsoleMaster(flow.FlowMaster): self.stickycookie = None self.stickyhosts = {} - if getattr(options, "cache", None) is not None: - self.state.start_recording(recorder.Recorder(options)) def spawn_external_viewer(self, data, contenttype): if contenttype: diff --git a/libmproxy/playback.py b/libmproxy/playback.py deleted file mode 100644 index 920b2e0c..00000000 --- a/libmproxy/playback.py +++ /dev/null @@ -1,129 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2010 Henrik Nordstrom <henrik@henriknordstrom.net> -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# HENRIK NORDSTROM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT -# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Alternatively you may use this file under a GPLv3 license as follows: -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import sys -import controller -import utils -import proxy -import recorder - -class PlaybackMaster(controller.Master): - """ - A simple master that plays back recorded responses. - """ - def __init__(self, server, options): - self.verbosity = options.verbose - self.store = recorder.Recorder(options) - controller.Master.__init__(self, server) - - def run(self): - try: - return controller.Master.run(self) - except KeyboardInterrupt: - self.shutdown() - - def process_missing_response(self, request): - response = None - print >> sys.stderr, self.store.normalize_request(request).assemble_proxy() - print >> sys.stderr, "Actions:" - print >> sys.stderr, " q Quit" - print >> sys.stderr, " a(dd) Add pattern rule" - print >> sys.stderr, " A(dd) Add pattern rule (forced)" - print >> sys.stderr, " e(rror) respond with a 404 error" - print >> sys.stderr, " k(ill) kill the request, empty response" - print >> sys.stderr, " f(orward) forward the request to the requested server and cache response" - command = raw_input("Action: ") - command = command[:1] - if command == 'q': - self.shutdown() - return None - elif command == 'a' or command == 'A': - filt = raw_input("Filter: ") - search = raw_input("Search pattern: ") - replace = raw_input("Replacement string: ") - self.store.add_rule(filt, search, replace) - if command == 'A': - self.store.save_rule(filt, search, replace) - elif command == 'e': - return proxy.Response(request, "404", "Not found", utils.Headers(), "Not found") - elif command == 'k': - return None - elif command == 'f': - return request - else: - print >> sys.stderr, "ERROR: Unknown command" - return self.process_missing_response(request) - try: - response = self.store.get_response(request) - if command == 'a': - self.store.save_rule(filt, search, replace) - except proxy.ProxyError: - print >> sys.stderr, "ERROR: Malformed substitution rule" - self.store.forget_last_rule() - response = self.process_missing_response(request) - except IOError: - print >> sys.stderr, "NOTICE: Response still not found" - if command == 'a': - self.store.forget_last_rule() - response = self.process_missing_response(request) - return response - - def handle_request(self, msg): - request = msg - try: - response = self.store.get_response(request) - except IOError: - if self.verbosity > 0: - print >> sys.stderr, ">>", - print >> sys.stderr, request.short() - print >> sys.stderr, "<<", - print >> sys.stderr, "ERROR: No matching response.", - print >> sys.stderr, ",".join(self.store.cookies) - response = self.process_missing_response(msg) - msg.ack(response) - - def handle_response(self, msg): - request = msg.request - response = msg - if self.verbosity > 0: - print >> sys.stderr, ">>", - print >> sys.stderr, request.short() - print >> sys.stderr, "<<", - print >> sys.stderr, response.short() - if not response.is_cached(): - self.store.save_response(response) - msg.ack(self.store.filter_response(msg)) diff --git a/libmproxy/record.py b/libmproxy/record.py deleted file mode 100644 index d32c8711..00000000 --- a/libmproxy/record.py +++ /dev/null @@ -1,68 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2010 Henrik Nordstrom <henrik@henriknordstrom.net> -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# HENRIK NORDSTROM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT -# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Alternatively you may use this file under a GPLv3 license as follows: -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import sys -import controller -import utils -import recorder - -class RecordMaster(controller.Master): - """ - A simple master that just records to files. - """ - def __init__(self, server, options): - self.verbosity = options.verbose - self.store = recorder.Recorder(options) - controller.Master.__init__(self, server) - - def run(self): - try: - return controller.Master.run(self) - except KeyboardInterrupt: - self.shutdown() - - def handle_request(self, msg): - msg.ack(self.store.filter_request(msg)) - - def handle_response(self, msg): - if self.verbosity > 0: - print >> sys.stderr, ">>", - print >> sys.stderr, msg.request.short() - print >> sys.stderr, "<<", - print >> sys.stderr, msg.short() - self.store.save_response(msg) - msg.ack(self.store.filter_response(msg)) diff --git a/libmproxy/recorder.py b/libmproxy/recorder.py deleted file mode 100644 index 9e4a1f89..00000000 --- a/libmproxy/recorder.py +++ /dev/null @@ -1,276 +0,0 @@ -#!/usr/bin/env python - -# Copyright (C) 2010 Henrik Nordstrom <henrik@henriknordstrom.net> -# -# Permission is hereby granted, free of charge, to any person obtaining a -# copy of this software and associated documentation files (the "Software"), -# to deal in the Software without restriction, including without limitation -# the rights to use, copy, modify, merge, publish, distribute, sublicense, -# and/or sell copies of the Software, and to permit persons to whom the -# Software is furnished to do so, subject to the following conditions: -# -# The above copyright notice and this permission notice shall be included -# in all copies or substantial portions of the Software. -# -# THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS -# OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY, -# FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL -# HENRIK NORDSTROM BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER LIABILITY, -# WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM, OUT -# OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE -# SOFTWARE. -# -# Alternatively you may use this file under a GPLv3 license as follows: -# -# This program is free software: you can redistribute it and/or modify -# it under the terms of the GNU General Public License as published by -# the Free Software Foundation, either version 3 of the License, or -# (at your option) any later version. -# -# This program is distributed in the hope that it will be useful, -# but WITHOUT ANY WARRANTY; without even the implied warranty of -# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the -# GNU General Public License for more details. -# -# You should have received a copy of the GNU General Public License -# along with this program. If not, see <http://www.gnu.org/licenses/>. - -import sys -import time -import hashlib -import utils -import proxy -import collections -import itertools -import string -import Cookie -import filt -import re -import cStringIO - - - -class PatternRule: - """ - Request pattern rule - :_ivar _match filt pattern rule - :_ivar _search Regex pattern to search for - :_ivar _replace Replacement string - """ - def __init__(self, pattern, search, replace): - self.match = filt.parse(pattern) - self.search = re.compile(search) - self.replace = replace - def execute(self, request, text): - if self.match and not self.match(request): - return text - return re.sub(self.search, self.replace, text) - -class RecorderConnection(proxy.ServerConnection): - """ - Simulated ServerConnection connecting to the cache - """ - # Note: This may chane in future. Division between Recorder - # and RecorderConnection is not yet finalized - def __init__(self, request, fp): - self.host = request.host - self.port = request.port - self.scheme = request.scheme - self.close = False - self.server = fp - self.rfile = fp - self.wfile = fp - - def send_request(self, request): - self.request = request - - def read_response(self): - response = proxy.ServerConnection.read_response(self) - response.cached = True - return response - -class Recorder: - """ - A simple record/playback cache - """ - def __init__(self, options): - self.sequence = collections.defaultdict(int) - self.cookies = {} - try: - for cookie in options.cookies: - self.cookies[cookie] = True - except AttributeError: pass - try: - self.verbosity = options.verbose - except AttributeError: - self.verbosity = False - self.storedir = options.cache - self.patterns = [] - self.indexfp = None - self.reset_config() - - def reset_config(self): - self.patterns = [] - self.load_config("default") - - def add_rule(self, match, search, replace): - self.patterns.append(PatternRule(match, search, replace)) - - def forget_last_rule(self): - self.patterns.pop() - - def save_rule(self, match, search, replace, configfile = "default"): - fp = self.open(configfile + ".cfg", "a") - print >> fp, "Condition: " + match - print >> fp, "Search: " + search - print >> fp, "Replace: " + replace - fp.close() - - def load_config(self, name): - """ - Load configuration settings from name - """ - try: - file = name + ".cfg" - if self.verbosity > 2: - print >> sys.stderr, "config: " + file - fp = self.open(file, "r") - except IOError: - return False - for line in fp: - directive, value = line.split(" ", 1) - value = value.strip("\r\n") - if directive == "Cookie:": - self.cookies[value] = True - if directive == "Condition:": - match = value - if directive == "Search:": - search = value - if directive == "Replace:": - self.add_rule(match, search, value) - fp.close() - return True - - def filter_request(self, request): - """ - Filter forwarded requests to enable better recording - """ - request = request.copy() - headers = request.headers - utils.try_del(headers, 'if-modified-since') - utils.try_del(headers, 'if-none-match') - return request - - def normalize_request(self, request): - """ - Filter request to simplify storage matching - """ - request.close = False - req_text = request.assemble_proxy() - orig_req_text = req_text - for pattern in self.patterns: - req_text = pattern.execute(request, req_text) - if req_text == orig_req_text: - return request - fp = cStringIO.StringIO(req_text) - request_line = fp.readline() - method, scheme, host, port, path, httpminor = proxy.parse_request_line(request_line) - headers = utils.Headers() - headers.read(fp) - if request.content is None: - content = None - else: - content = fp.read() - return proxy.Request(request.client_conn, host, port, scheme, method, path, headers, content) - - def open(self, path, mode): - return open(self.storedir + "/" + path, mode) - - def pathn(self, request): - """ - Create cache file name and sequence number - """ - request = self.normalize_request(request) - request = self.filter_request(request) - headers = request.headers - urlkey = (request.host + request.path)[:80].translate(string.maketrans(":/?","__.")) - id = "" - if headers.has_key("cookie"): - cookies = Cookie.SimpleCookie("; ".join(headers["cookie"])) - del headers["cookie"] - for key, morsel in cookies.iteritems(): - if self.cookies.has_key(key): - id = id + key + "=" + morsel.value + "\n" - if self.verbosity > 1: - print >> sys.stderr, "ID: " + id - m = hashlib.sha224(id) - req_text = request.assemble_proxy() - if self.verbosity > 2: - print >> sys.stderr, req_text - m.update(req_text) - path = urlkey+"."+m.hexdigest() - n = str(self.sequence[path]) - if self.verbosity > 1: - print >> sys.stderr, "PATH: " + path + "." + n - return path, n - - def filter_response(self, response): - if response.headers.has_key('set-cookie'): - for header in response.headers['set-cookie']: - key = header.split('=',1)[0] - self.cookies[key] = True - return response - - def save_response(self, response): - """ - Save response for later playback - """ - - if self.indexfp is None: - self.indexfp = self.open("index.txt", "a") - try: - cfg = self.open("default.cfg", "r") - except: - cfg = self.open("default.cfg", "w") - for cookie in iter(self.cookies): - print >> cfg, "Cookie: " + cookie - cfg.close() - request = response.request - req_text = request.assemble_proxy() - resp_text = response.assemble() - path, n = self.pathn(request) - self.sequence[path] += 1 - - f = self.open(path+"."+n+".req", 'w') - f.write(req_text) - f.close() - f = self.open(path+"."+n+".resp", 'w') - f.write(resp_text) - f.close() - - print >> self.indexfp , time.time(), request.method, request.path - if request.headers.has_key('referer'): - print >> self.indexfp, 'referer:', ','.join(request.headers['referer']) - if len(self.cookies) > 0: - print >> self.indexfp, 'cookies:', ','.join(self.cookies) - print >> self.indexfp , path - print >> self.indexfp , "" - self.indexfp.flush() - - - def get_response(self, request): - """ - Retrieve previously saved response saved by save_response - """ - path, n = self.pathn(request) - try: - fp = self.open(path+"."+n+".resp", 'r') - self.sequence[path]+=1 - except IOError: - fp = self.open(path+".resp", 'r') - server = RecorderConnection(request, fp) - fp = None # Handed over to RecorderConnection - server.send_request(request) - response = server.read_response() - server.terminate() - return response |