diff options
| -rw-r--r-- | libmproxy/cmdline.py | 16 | ||||
| -rw-r--r-- | libmproxy/dump.py | 5 | ||||
| -rw-r--r-- | libmproxy/flow.py | 39 | ||||
| -rw-r--r-- | test/test_flow.py | 76 |
4 files changed, 109 insertions, 27 deletions
diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py index bf5add33..ec03d63e 100644 --- a/libmproxy/cmdline.py +++ b/libmproxy/cmdline.py @@ -183,7 +183,8 @@ def get_common_options(options): verbosity=options.verbose, nopop=options.nopop, replay_ignore_content = options.replay_ignore_content, - replay_ignore_params = options.replay_ignore_params + replay_ignore_params = options.replay_ignore_params, + replay_ignore_payload_params = options.replay_ignore_payload_params ) @@ -438,13 +439,24 @@ def common_options(parser): help="Disable response pop from response flow. " "This makes it possible to replay same response multiple times." ) - group.add_argument( + payload = group.add_mutually_exclusive_group() + payload.add_argument( "--replay-ignore-content", action="store_true", dest="replay_ignore_content", default=False, help=""" Ignore request's content while searching for a saved flow to replay """ ) + payload.add_argument( + "--replay-ignore-payload-param", + action="append", dest="replay_ignore_payload_params", type=str, + help=""" + Request's payload parameters (application/x-www-form-urlencoded) to + be ignored while searching for a saved flow to replay. + Can be passed multiple times. + """ + ) + group.add_argument( "--replay-ignore-param", action="append", dest="replay_ignore_params", type=str, diff --git a/libmproxy/dump.py b/libmproxy/dump.py index 8f260745..731592dc 100644 --- a/libmproxy/dump.py +++ b/libmproxy/dump.py @@ -39,6 +39,7 @@ class Options(object): "outfile", "replay_ignore_content", "replay_ignore_params", + "replay_ignore_payload_params", ] def __init__(self, **kwargs): @@ -78,6 +79,7 @@ class DumpMaster(flow.FlowMaster): self.replay_ignore_params = options.replay_ignore_params self.replay_ignore_content = options.replay_ignore_content self.refresh_server_playback = options.refresh_server_playback + self.replay_ignore_payload_params = options.replay_ignore_payload_params self.set_stream_large_bodies(options.stream_large_bodies) @@ -115,7 +117,8 @@ class DumpMaster(flow.FlowMaster): not options.keepserving, options.nopop, options.replay_ignore_params, - options.replay_ignore_content + options.replay_ignore_content, + options.replay_ignore_payload_params, ) if options.client_replay: diff --git a/libmproxy/flow.py b/libmproxy/flow.py index d3ae383e..904a64b1 100644 --- a/libmproxy/flow.py +++ b/libmproxy/flow.py @@ -201,12 +201,12 @@ class ClientPlaybackState: class ServerPlaybackState: - def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content): + def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content, ignore_payload_params): """ headers: Case-insensitive list of request headers that should be included in request-response matching. """ - self.headers, self.exit, self.nopop, self.ignore_params, self.ignore_content = headers, exit, nopop, ignore_params, ignore_content + self.headers, self.exit, self.nopop, self.ignore_params, self.ignore_content, self.ignore_payload_params = headers, exit, nopop, ignore_params, ignore_content, ignore_payload_params self.fmap = {} for i in flows: if i.response: @@ -225,22 +225,37 @@ class ServerPlaybackState: _, _, path, _, query, _ = urlparse.urlparse(r.url) queriesArray = urlparse.parse_qsl(query) - filtered = [] - ignore_params = self.ignore_params or [] - for p in queriesArray: - if p[0] not in ignore_params: - filtered.append(p) - key = [ str(r.host), str(r.port), str(r.scheme), str(r.method), str(path), - ] + ] + if not self.ignore_content: - key.append(str(r.content)) + ignore_payload_params = self.ignore_payload_params or [] + ct = r.headers["Content-Type"] + if len(ct) > 0: + ct = ct[0] + if len(ignore_payload_params) > 0 and ct == "application/x-www-form-urlencoded": + parsedContent = urlparse.parse_qsl(r.content) + filtered = [] + for p in parsedContent: + if p[0] not in ignore_payload_params: + filtered.append(p) + + for p in filtered: + key.append(p[0]) + key.append(p[1]) + else: + key.append(str(r.content)) + filtered = [] + ignore_params = self.ignore_params or [] + for p in queriesArray: + if p[0] not in ignore_params: + filtered.append(p) for p in filtered: key.append(p[0]) key.append(p[1]) @@ -697,14 +712,14 @@ class FlowMaster(controller.Master): def stop_client_playback(self): self.client_playback = None - def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params, ignore_content): + def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params, ignore_content, ignore_payload_params): """ flows: List of flows. kill: Boolean, should we kill requests not part of the replay? ignore_params: list of parameters to ignore in server replay ignore_content: true if request content should be ignored in server replay """ - self.server_playback = ServerPlaybackState(headers, flows, exit, nopop, ignore_params, ignore_content) + self.server_playback = ServerPlaybackState(headers, flows, exit, nopop, ignore_params, ignore_content, ignore_payload_params) self.kill_nonreplay = kill def stop_server_playback(self): diff --git a/test/test_flow.py b/test/test_flow.py index fdfac62f..48f5ba55 100644 --- a/test/test_flow.py +++ b/test/test_flow.py @@ -114,7 +114,7 @@ class TestClientPlaybackState: class TestServerPlaybackState: def test_hash(self): - s = flow.ServerPlaybackState(None, [], False, False, None, False) + s = flow.ServerPlaybackState(None, [], False, False, None, False, None) r = tutils.tflow() r2 = tutils.tflow() @@ -126,7 +126,7 @@ class TestServerPlaybackState: assert s._hash(r) != s._hash(r2) def test_headers(self): - s = flow.ServerPlaybackState(["foo"], [], False, False, None, False) + s = flow.ServerPlaybackState(["foo"], [], False, False, None, False, None) r = tutils.tflow(resp=True) r.request.headers["foo"] = ["bar"] r2 = tutils.tflow(resp=True) @@ -147,7 +147,7 @@ class TestServerPlaybackState: r2 = tutils.tflow(resp=True) r2.request.headers["key"] = ["two"] - s = flow.ServerPlaybackState(None, [r, r2], False, False, None, False) + s = flow.ServerPlaybackState(None, [r, r2], False, False, None, False, None) assert s.count() == 2 assert len(s.fmap.keys()) == 1 @@ -168,7 +168,7 @@ class TestServerPlaybackState: r2 = tutils.tflow(resp=True) r2.request.headers["key"] = ["two"] - s = flow.ServerPlaybackState(None, [r, r2], False, True, None, False) + s = flow.ServerPlaybackState(None, [r, r2], False, True, None, False, None) assert s.count() == 2 s.next_flow(r) @@ -176,7 +176,7 @@ class TestServerPlaybackState: def test_ignore_params(self): - s = flow.ServerPlaybackState(None, [], False, False, ["param1", "param2"], False) + s = flow.ServerPlaybackState(None, [], False, False, ["param1", "param2"], False, None) r = tutils.tflow(resp=True) r.request.path="/test?param1=1" r2 = tutils.tflow(resp=True) @@ -189,8 +189,60 @@ class TestServerPlaybackState: r2.request.path="/test?param3=2" assert not s._hash(r) == s._hash(r2) + def test_ignore_payload_params(self): + s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"]) + r = tutils.tflow(resp=True) + r.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"] + r.request.content = "paramx=x¶m1=1" + r2 = tutils.tflow(resp=True) + r2.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"] + r2.request.content = "paramx=x¶m1=1" + # same parameters + assert s._hash(r) == s._hash(r2) + # ignored parameters != + r2.request.content = "paramx=x¶m1=2" + assert s._hash(r) == s._hash(r2) + # missing parameter + r2.request.content="paramx=x" + assert s._hash(r) == s._hash(r2) + # ignorable parameter added + r2.request.content="paramx=x¶m1=2" + assert s._hash(r) == s._hash(r2) + # not ignorable parameter changed + r2.request.content="paramx=y¶m1=1" + assert not s._hash(r) == s._hash(r2) + # not ignorable parameter missing + r2.request.content="param1=1" + assert not s._hash(r) == s._hash(r2) + + def test_ignore_payload_params_other_content_type(self): + s = flow.ServerPlaybackState(None, [], False, False, None, False, ["param1", "param2"]) + r = tutils.tflow(resp=True) + r.request.headers["Content-Type"] = ["application/json"] + r.request.content = '{"param1":"1"}' + r2 = tutils.tflow(resp=True) + r2.request.headers["Content-Type"] = ["application/json"] + r2.request.content = '{"param1":"1"}' + # same content + assert s._hash(r) == s._hash(r2) + # distint content (note only x-www-form-urlencoded payload is analysed) + r2.request.content = '{"param1":"2"}' + assert not s._hash(r) == s._hash(r2) + + def test_ignore_payload_wins_over_params(self): + #NOTE: parameters are mutually exclusive in options + s = flow.ServerPlaybackState(None, [], False, False, None, True, ["param1", "param2"]) + r = tutils.tflow(resp=True) + r.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"] + r.request.content = "paramx=y" + r2 = tutils.tflow(resp=True) + r2.request.headers["Content-Type"] = ["application/x-www-form-urlencoded"] + r2.request.content = "paramx=x" + # same parameters + assert s._hash(r) == s._hash(r2) + def test_ignore_content(self): - s = flow.ServerPlaybackState(None, [], False, False, None, False) + s = flow.ServerPlaybackState(None, [], False, False, None, False, None) r = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True) @@ -201,7 +253,7 @@ class TestServerPlaybackState: assert not s._hash(r) == s._hash(r2) #now ignoring content - s = flow.ServerPlaybackState(None, [], False, False, None, True) + s = flow.ServerPlaybackState(None, [], False, False, None, True, None) r = tutils.tflow(resp=True) r2 = tutils.tflow(resp=True) r.request.content = "foo" @@ -695,7 +747,7 @@ class TestFlowMaster: pb = [tutils.tflow(resp=True), f] fm = flow.FlowMaster(DummyServer(ProxyConfig()), s) - assert not fm.start_server_playback(pb, False, [], False, False, None, False) + assert not fm.start_server_playback(pb, False, [], False, False, None, False, None) assert not fm.start_client_playback(pb, False) fm.client_playback.testing = True @@ -718,16 +770,16 @@ class TestFlowMaster: fm.refresh_server_playback = True assert not fm.do_server_playback(tutils.tflow()) - fm.start_server_playback(pb, False, [], False, False, None, False) + fm.start_server_playback(pb, False, [], False, False, None, False, None) assert fm.do_server_playback(tutils.tflow()) - fm.start_server_playback(pb, False, [], True, False, None, False) + fm.start_server_playback(pb, False, [], True, False, None, False, None) r = tutils.tflow() r.request.content = "gibble" assert not fm.do_server_playback(r) assert fm.do_server_playback(tutils.tflow()) - fm.start_server_playback(pb, False, [], True, False, None, False) + fm.start_server_playback(pb, False, [], True, False, None, False, None) q = Queue.Queue() fm.tick(q, 0) assert fm.should_exit.is_set() @@ -742,7 +794,7 @@ class TestFlowMaster: pb = [f] fm = flow.FlowMaster(None, s) fm.refresh_server_playback = True - fm.start_server_playback(pb, True, [], False, False, None, False) + fm.start_server_playback(pb, True, [], False, False, None, False, None) f = tutils.tflow() f.request.host = "nonexistent" |