From 803d631f04f6af5bb54bbb46b3efec39622ca216 Mon Sep 17 00:00:00 2001 From: Wade Catron Date: Sat, 7 Mar 2015 08:38:18 -0800 Subject: Adding a server replay option to ignore host when searching for matching requests --- libmproxy/cmdline.py | 8 +++++++- libmproxy/console/__init__.py | 5 ++++- libmproxy/console/flowlist.py | 6 ++++-- libmproxy/dump.py | 3 +++ libmproxy/flow.py | 27 +++++++++++++++++++------- test/test_flow.py | 44 ++++++++++++++++++++++++++----------------- 6 files changed, 65 insertions(+), 28 deletions(-) diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index e45cc54c..b7270fb9 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -192,7 +192,8 @@ def get_common_options(options): nopop=options.nopop, replay_ignore_content = options.replay_ignore_content, replay_ignore_params = options.replay_ignore_params, - replay_ignore_payload_params = options.replay_ignore_payload_params + replay_ignore_payload_params = options.replay_ignore_payload_params, + replay_ignore_host = options.replay_ignore_host ) @@ -479,6 +480,11 @@ def common_options(parser): to replay. Can be passed multiple times. """ ) + group.add_argument( + "--replay-ignore-host", + action="store_true", dest="replay_ignore_host", default=False, + help="Ignore request's destination host while searching for a saved flow to replay" + ) group = parser.add_argument_group( "Replacements", diff --git a/libmproxy/console/__init__.py b/libmproxy/console/__init__.py index 0db06832..cc8a0c1f 100644 --- a/libmproxy/console/__init__.py +++ b/libmproxy/console/__init__.py @@ -528,7 +528,10 @@ class ConsoleMaster(flow.FlowMaster): flows, self.killextra, self.rheaders, False, self.nopop, - self.options.replay_ignore_params, self.options.replay_ignore_content, self.options.replay_ignore_payload_params + self.options.replay_ignore_params, + self.options.replay_ignore_content, + self.options.replay_ignore_payload_params, + self.options.replay_ignore_host ) def spawn_editor(self, data): diff --git a/libmproxy/console/flowlist.py b/libmproxy/console/flowlist.py index 9e7c6d69..2a6a98c8 100644 --- a/libmproxy/console/flowlist.py +++ b/libmproxy/console/flowlist.py @@ -126,7 +126,8 @@ class ConnectionItem(common.WWrap): self.master.killextra, self.master.rheaders, False, self.master.nopop, self.master.options.replay_ignore_params, self.master.options.replay_ignore_content, - self.master.options.replay_ignore_payload_params + self.master.options.replay_ignore_payload_params, + self.master.options.replay_ignore_host ) elif k == "t": self.master.start_server_playback( @@ -134,7 +135,8 @@ class ConnectionItem(common.WWrap): self.master.killextra, self.master.rheaders, False, self.master.nopop, self.master.options.replay_ignore_params, self.master.options.replay_ignore_content, - self.master.options.replay_ignore_payload_params + self.master.options.replay_ignore_payload_params, + self.master.options.replay_ignore_host ) else: self.master.path_prompt( diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 59ddcf5f..ab58966f 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -40,6 +40,7 @@ class Options(object): "replay_ignore_content", "replay_ignore_params", "replay_ignore_payload_params", + "replay_ignore_host" ] def __init__(self, **kwargs): @@ -78,6 +79,7 @@ class DumpMaster(flow.FlowMaster): self.showhost = options.showhost self.replay_ignore_params = options.replay_ignore_params self.replay_ignore_content = options.replay_ignore_content + self.replay_ignore_host = options.replay_ignore_host self.refresh_server_playback = options.refresh_server_playback self.replay_ignore_payload_params = options.replay_ignore_payload_params @@ -119,6 +121,7 @@ class DumpMaster(flow.FlowMaster): options.replay_ignore_params, options.replay_ignore_content, options.replay_ignore_payload_params, + options.replay_ignore_host ) if options.client_replay: diff --git a/libmproxy/flow.py b/libmproxy/flow.py index 43580109..8343c183 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -203,12 +203,19 @@ class ClientPlaybackState: class ServerPlaybackState: - def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content, ignore_payload_params): + def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content, + ignore_payload_params, ignore_host): """ headers: Case-insensitive list of request headers that should be included in request-response matching. """ - self.headers, self.exit, self.nopop, self.ignore_params, self.ignore_content, self.ignore_payload_params = headers, exit, nopop, ignore_params, ignore_content, ignore_payload_params + self.headers = headers + self.exit = exit + self.nopop = nopop + self.ignore_params = ignore_params + self.ignore_content = ignore_content + self.ignore_payload_params = ignore_payload_params + self.ignore_host = ignore_host self.fmap = {} for i in flows: if i.response: @@ -228,7 +235,6 @@ class ServerPlaybackState: queriesArray = urlparse.parse_qsl(query) key = [ - str(r.host), str(r.port), str(r.scheme), str(r.method), @@ -245,6 +251,9 @@ class ServerPlaybackState: else: key.append(str(r.content)) + if not self.ignore_host: + key.append(r.host) + filtered = [] ignore_params = self.ignore_params or [] for p in queriesArray: @@ -616,6 +625,7 @@ class FlowMaster(controller.Master): self.setheaders = SetHeaders() self.replay_ignore_params = False self.replay_ignore_content = None + self.replay_ignore_host = False self.stream = None self.apps = AppRegistry() @@ -712,16 +722,19 @@ class FlowMaster(controller.Master): def stop_client_playback(self): self.client_playback = None - def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params, ignore_content, - ignore_payload_params): + def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params, + ignore_content, ignore_payload_params, ignore_host): """ flows: List of flows. kill: Boolean, should we kill requests not part of the replay? ignore_params: list of parameters to ignore in server replay ignore_content: true if request content should be ignored in server replay + ignore_payload_params: list of content params to ignore in server replay + ignore_host: true if request host should be ignored in server replay """ - self.server_playback = ServerPlaybackState(headers, flows, exit, nopop, ignore_params, ignore_content, - ignore_payload_params) + self.server_playback = ServerPlaybackState(headers, flows, exit, nopop, + ignore_params, ignore_content, + ignore_payload_params, ignore_host) self.kill_nonreplay = kill def stop_server_playback(self): diff --git a/test/test_flow.py b/test/test_flow.py index 6d77f075..b41eb630 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -114,7 +114,7 @@ class TestClientPlaybackState: class TestServerPlaybackState: def test_hash(self): - s = flow.ServerPlaybackState(None, [], False, False, None, False, None) + s = flow.ServerPlaybackState(None, [], False, False, None, False, None, False) r = tutils.tflow() r2 = tutils.tflow() @@ -126,7 +126,7 @@ class TestServerPlaybackState: assert s._hash(r) != s._hash(r2) def test_headers(self): - s = flow.ServerPlaybackState(["foo"], [], False, False, None, False, None) + s = flow.ServerPlaybackState(["foo"], [], False, False, None, False, None, False) r = tutils.tflow(resp=True) r.request.headers["foo"] = ["bar"] r2 = tutils.tflow(resp=True) @@ -147,7 +147,7 @@ class TestServerPlaybackState: r2 = tutils.tflow(resp=True) r2.request.headers["key"] = ["two"] - s = flow.ServerPlaybackState(None, [r, r2], False, False, None, False, None) + s = flow.ServerPlaybackState(None, [r, r2], False, False, None, False, None, False) assert s.count() == 2 assert len(s.fmap.keys()) == 1 @@ -168,7 +168,7 @@ class TestServerPlaybackState: r2 = tutils.tflow(resp=True) r2.request.headers["key"] = ["two"] - s = flow.ServerPlaybackState(None, [r, r2], False, True, None, False, None) + s = flow.ServerPlaybackState(None, [r, r2], False, True, None, False, None, False) assert s.count() == 2 s.next_flow(r) @@ -176,7 +176,7 @@ class TestServerPlaybackState: def test_ignore_params(self): - s = flow.ServerPlaybackState(None, [], False, False, ["param1", "param2"], False, None) + s = flow.ServerPlaybackState(None, [], False, False, ["param1", "param2"], False, None, False) r = tutils.tflow(resp=True) r.request.path="/test?param1=1" r2 = tutils.tflow(resp=True) @@ -190,7 +190,7 @@ class TestServerPlaybackState: assert not s._hash(r) == s._hash(r2) def test_ignore_payload_params(self): - s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"]) + s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"], False) r = tutils.tflow(resp=True) r.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"] r.request.content = "paramx=x¶m1=1" @@ -216,7 +216,7 @@ class TestServerPlaybackState: assert not s._hash(r) == s._hash(r2) def test_ignore_payload_params_other_content_type(self): - s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"]) + s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"], False) r = tutils.tflow(resp=True) r.request.headers["Content-Type"] = ["application/json"] r.request.content = '{"param1":"1"}' @@ -231,7 +231,7 @@ class TestServerPlaybackState: def test_ignore_payload_wins_over_params(self): #NOTE: parameters are mutually exclusive in options - s = flow.ServerPlaybackState(None, [], False, False, None, True, ["param1", "param2"]) + s = flow.ServerPlaybackState(None, [], False, False, None, True, ["param1", "param2"], False) r = tutils.tflow(resp=True) r.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"] r.request.content = "paramx=y" @@ -242,10 +242,10 @@ class TestServerPlaybackState: assert s._hash(r) == s._hash(r2) def test_ignore_content(self): - s = flow.ServerPlaybackState(None, [], False, False, None, False, None) + s = flow.ServerPlaybackState(None, [], False, False, None, False, None, False) r = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True) - + r.request.content = "foo" r2.request.content = "foo" assert s._hash(r) == s._hash(r2) @@ -253,7 +253,7 @@ class TestServerPlaybackState: assert not s._hash(r) == s._hash(r2) #now ignoring content - s = flow.ServerPlaybackState(None, [], False, False, None, True, None) + s = flow.ServerPlaybackState(None, [], False, False, None, True, None, False) r = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True) r.request.content = "foo" @@ -266,6 +266,17 @@ class TestServerPlaybackState: r2.request.content = None assert s._hash(r) == s._hash(r2) + def test_ignore_host(self): + s = flow.ServerPlaybackState(None, [], False, False, None, False, None, True) + r = tutils.tflow(resp=True) + r2 = tutils.tflow(resp=True) + + r.request.host="address" + r2.request.host="address" + assert s._hash(r) == s._hash(r2) + r2.request.host="wrong_address" + assert s._hash(r) == s._hash(r2) + class TestFlow: def test_copy(self): @@ -748,9 +759,8 @@ class TestFlowMaster: f = tutils.tflow(resp=True) pb = [tutils.tflow(resp=True), f] - fm = flow.FlowMaster(DummyServer(ProxyConfig()), s) - assert not fm.start_server_playback(pb, False, [], False, False, None, False, None) + assert not fm.start_server_playback(pb, False, [], False, False, None, False, None, False) assert not fm.start_client_playback(pb, False) fm.client_playback.testing = True @@ -773,16 +783,16 @@ class TestFlowMaster: fm.refresh_server_playback = True assert not fm.do_server_playback(tutils.tflow()) - fm.start_server_playback(pb, False, [], False, False, None, False, None) + fm.start_server_playback(pb, False, [], False, False, None, False, None, False) assert fm.do_server_playback(tutils.tflow()) - fm.start_server_playback(pb, False, [], True, False, None, False, None) + fm.start_server_playback(pb, False, [], True, False, None, False, None, False) r = tutils.tflow() r.request.content = "gibble" assert not fm.do_server_playback(r) assert fm.do_server_playback(tutils.tflow()) - fm.start_server_playback(pb, False, [], True, False, None, False, None) + fm.start_server_playback(pb, False, [], True, False, None, False, None, False) q = Queue.Queue() fm.tick(q, 0) assert fm.should_exit.is_set() @@ -797,7 +807,7 @@ class TestFlowMaster: pb = [f] fm = flow.FlowMaster(None, s) fm.refresh_server_playback = True - fm.start_server_playback(pb, True, [], False, False, None, False, None) + fm.start_server_playback(pb, True, [], False, False, None, False, None, False) f = tutils.tflow() f.request.host = "nonexistent" -- cgit v1.2.3