--replay-ignore-content & --replay-ignore-param ported from branch 0.10

4 files changed, 97 insertions, 17 deletions

diff --git a/libmproxy/cmdline.py b/libmproxy/cmdline.py
index a3cf07ef..f6cd1ab8 100644
--- a/libmproxy/cmdline.py
+++ b/libmproxy/cmdline.py
@@ -176,6 +176,8 @@ def get_common_options(options):
         wfile=options.wfile,
         verbosity=options.verbose,
         nopop=options.nopop,
+        replay_ignore_content = options.replay_ignore_content,
+        replay_ignore_params = options.replay_ignore_params
     )
 
 
@@ -369,6 +371,17 @@ def common_options(parser):
         help="Disable response pop from response flow. "
              "This makes it possible to replay same response multiple times."
     )
+    group.add_argument(
+        "--replay-ignore-content",
+        action="store_true", dest="replay_ignore_content", default=False,
+        help="Ignore request's content while searching for a saved flow to replay"
+    )
+    group.add_argument(
+        "--replay-ignore-param",
+        action="append", dest="replay_ignore_params", type=str,
+        help="Request's parameters to be ignored while searching for a saved flow to replay"
+           "Can be passed multiple times."
+    )    
 
     group = parser.add_argument_group(
         "Replacements",
diff --git a/libmproxy/dump.py b/libmproxy/dump.py
index 0349a3b5..ccb2b5b5 100644
--- a/libmproxy/dump.py
+++ b/libmproxy/dump.py
@@ -34,6 +34,8 @@ class Options(object):
         "stream_large_bodies",
         "verbosity",
         "wfile",
+        "replay_ignore_content",
+        "replay_ignore_params",
     ]
     def __init__(self, **kwargs):
         for k, v in kwargs.items():
@@ -69,6 +71,8 @@ class DumpMaster(flow.FlowMaster):
         self.anticache = options.anticache
         self.anticomp = options.anticomp
         self.showhost = options.showhost
+        self.replay_ignore_params = options.replay_ignore_params    
+        self.replay_ignore_content = options.replay_ignore_content
         self.refresh_server_playback = options.refresh_server_playback
 
         self.set_stream_large_bodies(options.stream_large_bodies)
@@ -106,7 +110,9 @@ class DumpMaster(flow.FlowMaster):
                 self._readflow(options.server_replay),
                 options.kill, options.rheaders,
                 not options.keepserving,
-                options.nopop
+                options.nopop,
+                options.replay_ignore_params,
+                options.replay_ignore_content
             )
 
         if options.client_replay:
diff --git a/libmproxy/flow.py b/libmproxy/flow.py
index 4dc3a272..440798bc 100644
--- a/libmproxy/flow.py
+++ b/libmproxy/flow.py
@@ -12,6 +12,7 @@ from . import controller, protocol, tnetstring, filt, script, version
 from .onboarding import app
 from .protocol import http, handle
 from .proxy.config import parse_host_pattern
+import urlparse
 
 ODict = odict.ODict
 ODictCaseless = odict.ODictCaseless
@@ -193,12 +194,12 @@ class ClientPlaybackState:
 
 
 class ServerPlaybackState:
-    def __init__(self, headers, flows, exit, nopop):
+    def __init__(self, headers, flows, exit, nopop, ignore_params, ignore_content):
         """
             headers: Case-insensitive list of request headers that should be
             included in request-response matching.
         """
-        self.headers, self.exit, self.nopop = headers, exit, nopop
+        self.headers, self.exit, self.nopop, self.ignore_params, self.ignore_content = headers, exit, nopop, ignore_params, ignore_content
         self.fmap = {}
         for i in flows:
             if i.response:
@@ -213,14 +214,29 @@ class ServerPlaybackState:
             Calculates a loose hash of the flow request.
         """
         r = flow.request
+
+        _, _, path, _, query, _ = urlparse.urlparse(r.url)
+        queriesArray = urlparse.parse_qsl(query)
+
+        filtered = []
+        for p in queriesArray:
+          if p[0] not in self.ignore_params:
+            filtered.append(p)
+
         key = [
             str(r.host),
             str(r.port),
             str(r.scheme),
             str(r.method),
-            str(r.path),
-            str(r.content),
-        ]
+            str(path),
+         ]
+        if not self.ignore_content:
+            key.append(str(r.content))
+
+        for p in filtered:
+            key.append(p[0])
+            key.append(p[1])
+
         if self.headers:
             hdrs = []
             for i in self.headers:
@@ -449,6 +465,9 @@ class FlowMaster(controller.Master):
         self.refresh_server_playback = False
         self.replacehooks = ReplaceHooks()
         self.setheaders = SetHeaders()
+        self.replay_ignore_params = False    
+        self.replay_ignore_content = None
+
 
         self.stream = None
         self.apps = AppRegistry()
@@ -539,12 +558,14 @@ class FlowMaster(controller.Master):
     def stop_client_playback(self):
         self.client_playback = None
 
-    def start_server_playback(self, flows, kill, headers, exit, nopop):
+    def start_server_playback(self, flows, kill, headers, exit, nopop, ignore_params, ignore_content):
         """
             flows: List of flows.
             kill: Boolean, should we kill requests not part of the replay?
+            ignore_params: list of parameters to ignore in server replay
+            ignore_content: true if request content should be ignored in server replay
         """
-        self.server_playback = ServerPlaybackState(headers, flows, exit, nopop)
+        self.server_playback = ServerPlaybackState(headers, flows, exit, nopop, ignore_params, ignore_content)
         self.kill_nonreplay = kill
 
     def stop_server_playback(self):
diff --git a/test/test_flow.py b/test/test_flow.py
index c5254d11..b74119dd 100644
--- a/test/test_flow.py
+++ b/test/test_flow.py
@@ -111,7 +111,7 @@ class TestClientPlaybackState:
 
 class TestServerPlaybackState:
     def test_hash(self):
-        s = flow.ServerPlaybackState(None, [], False, False)
+        s = flow.ServerPlaybackState(None, [], False, False, None, False)
         r = tutils.tflow()
         r2 = tutils.tflow()
 
@@ -123,7 +123,7 @@ class TestServerPlaybackState:
         assert s._hash(r) != s._hash(r2)
 
     def test_headers(self):
-        s = flow.ServerPlaybackState(["foo"], [], False, False)
+        s = flow.ServerPlaybackState(["foo"], [], False, False, None, False)
         r = tutils.tflow(resp=True)
         r.request.headers["foo"] = ["bar"]
         r2 = tutils.tflow(resp=True)
@@ -144,7 +144,7 @@ class TestServerPlaybackState:
         r2 = tutils.tflow(resp=True)
         r2.request.headers["key"] = ["two"]
 
-        s = flow.ServerPlaybackState(None, [r, r2], False, False)
+        s = flow.ServerPlaybackState(None, [r, r2], False, False, None, False)
         assert s.count() == 2
         assert len(s.fmap.keys()) == 1
 
@@ -165,13 +165,53 @@ class TestServerPlaybackState:
         r2 = tutils.tflow(resp=True)
         r2.request.headers["key"] = ["two"]
 
-        s = flow.ServerPlaybackState(None, [r, r2], False, True)
+        s = flow.ServerPlaybackState(None, [r, r2], False, True, None, False)
 
         assert s.count() == 2
         s.next_flow(r)
         assert s.count() == 2
 
 
+    def test_ignore_params(self):
+        s = flow.ServerPlaybackState(None, [], False, False, ["param1", "param2"], False)
+        r = tutils.tflow(resp=True)
+        r.request.path="/test?param1=1"
+        r2 = tutils.tflow(resp=True)
+        r2.request.path="/test"
+        assert s._hash(r) == s._hash(r2)
+        r2.request.path="/test?param1=2"
+        assert s._hash(r) == s._hash(r2)
+        r2.request.path="/test?param2=1"
+        assert s._hash(r) == s._hash(r2)
+        r2.request.path="/test?param3=2"
+        assert not s._hash(r) == s._hash(r2)
+
+    def test_ignore_content(self):
+        s = flow.ServerPlaybackState(None, [], False, False, None, False)
+        r = tutils.tflow(resp=True)
+        r2 = tutils.tflow(resp=True)
+        
+        r.request.content = "foo"
+        r2.request.content = "foo"
+        assert s._hash(r) == s._hash(r2)
+        r2.request.content = "bar"
+        assert not s._hash(r) == s._hash(r2)
+
+        #now ignoring content
+        s = flow.ServerPlaybackState(None, [], False, False, None, True)
+        r = tutils.tflow(resp=True)
+        r2 = tutils.tflow(resp=True)
+        r.request.content = "foo"
+        r2.request.content = "foo"
+        assert s._hash(r) == s._hash(r2)
+        r2.request.content = "bar"
+        assert s._hash(r) == s._hash(r2)
+        r2.request.content = ""
+        assert s._hash(r) == s._hash(r2)
+        r2.request.content = None
+        assert s._hash(r) == s._hash(r2)
+
+
 class TestFlow:
     def test_copy(self):
         f = tutils.tflow(resp=True)
@@ -640,7 +680,7 @@ class TestFlowMaster:
         f = tutils.tflow(resp=True)
         pb = [tutils.tflow(resp=True), f]
         fm = flow.FlowMaster(None, s)
-        assert not fm.start_server_playback(pb, False, [], False, False)
+        assert not fm.start_server_playback(pb, False, [], False, False, None, False)
         assert not fm.start_client_playback(pb, False)
 
         q = Queue.Queue()
@@ -662,16 +702,16 @@ class TestFlowMaster:
         fm.refresh_server_playback = True
         assert not fm.do_server_playback(tutils.tflow())
 
-        fm.start_server_playback(pb, False, [], False, False)
+        fm.start_server_playback(pb, False, [], False, False, None, False)
         assert fm.do_server_playback(tutils.tflow())
 
-        fm.start_server_playback(pb, False, [], True, False)
+        fm.start_server_playback(pb, False, [], True, False, None, False)
         r = tutils.tflow()
         r.request.content = "gibble"
         assert not fm.do_server_playback(r)
         assert fm.do_server_playback(tutils.tflow())
 
-        fm.start_server_playback(pb, False, [], True, False)
+        fm.start_server_playback(pb, False, [], True, False, None, False)
         q = Queue.Queue()
         fm.tick(q, 0)
         assert fm.should_exit.is_set()
@@ -686,7 +726,7 @@ class TestFlowMaster:
         pb = [f]
         fm = flow.FlowMaster(None, s)
         fm.refresh_server_playback = True
-        fm.start_server_playback(pb, True, [], False, False)
+        fm.start_server_playback(pb, True, [], False, False, None, False)
 
         f = tutils.tflow()
         f.request.host = "nonexistent"