Fix non-deterministic test failures in export

We had various places in the code where we relied on incidental order of dict
keys. Add a helper to multidict, and fix.

3 files changed, 33 insertions, 18 deletions

diff --git a/mitmproxy/flow/export.py b/mitmproxy/flow/export.py
index 97cee984..f0ac02ab 100644
--- a/mitmproxy/flow/export.py
+++ b/mitmproxy/flow/export.py
@@ -9,6 +9,13 @@ from six.moves.urllib.parse import quote, quote_plus
 import netlib.http
 
 
+def dictstr(items, indent):
+    lines = []
+    for k, v in items:
+        lines.append(indent + "%s: %s,\n" % (repr(k), repr(v)))
+    return "{\n%s}\n" % "".join(lines)
+
+
 def curl_command(flow):
     data = "curl "
 
@@ -47,24 +54,19 @@ def python_code(flow):
     args = ""
     headers = ""
     if flow.request.headers:
-        lines = ["    '%s': '%s',\n" % (k, v) for k, v in flow.request.headers.fields]
-        headers += "\nheaders = {\n%s}\n" % "".join(lines)
+        headers += "\nheaders = %s\n" % dictstr(flow.request.headers.fields, "    ")
         args += "\n    headers=headers,"
 
     params = ""
     if flow.request.query:
-        lines = ["    %s: %s,\n" % (repr(k), repr(v)) for k, v in flow.request.query.to_dict().items()]
-        params = "\nparams = {\n%s}\n" % "".join(lines)
+        params = "\nparams = %s\n" % dictstr(flow.request.query.collect(), "    ")
         args += "\n    params=params,"
 
     data = ""
     if flow.request.body:
         json_obj = is_json(flow.request.headers, flow.request.body)
         if json_obj:
-            # Without the separators field json.dumps() produces
-            # trailing white spaces: https://bugs.python.org/issue16333
-            data = json.dumps(json_obj, indent=4, separators=(',', ': '))
-            data = "\njson = %s\n" % data
+            data = "\njson = %s\n" % dictstr(sorted(json_obj.items()), "    ")
             args += "\n    json=json,"
         else:
             data = "\ndata = '''%s'''\n" % flow.request.body
@@ -78,7 +80,6 @@ def python_code(flow):
         method=flow.request.method,
         args=args,
     )
-
     return code
 
 
@@ -142,7 +143,7 @@ def locust_code(flow):
 
     params = ""
     if flow.request.query:
-        lines = ["            %s: %s,\n" % (repr(k), repr(v)) for k, v in flow.request.query.to_dict().items()]
+        lines = ["            %s: %s,\n" % (repr(k), repr(v)) for k, v in flow.request.query.collect()]
         params = "\n        params = {\n%s        }\n" % "".join(lines)
         args += "\n            params=params,"
 
diff --git a/netlib/multidict.py b/netlib/multidict.py
index 0350ae4f..dc0f3466 100644
--- a/netlib/multidict.py
+++ b/netlib/multidict.py
@@ -178,6 +178,22 @@ class _MultiDict(MutableMapping, basetypes.Serializable):
         if key in self:
             del self[key]
 
+    def collect(self):
+        """
+            Returns a list of (key, value) tuples, where values are either
+            singular if threre is only one matching item for a key, or a list
+            if there are more than one. The order of the keys matches the order
+            in the underlying fields list.
+        """
+        coll = []
+        for key in self:
+            values = self.get_all(key)
+            if len(values) == 1:
+                coll.append([key, values[0]])
+            else:
+                coll.append([key, values])
+        return coll
+
     def to_dict(self):
         """
         Get the MultiDict as a plain Python dict.
@@ -197,12 +213,8 @@ class _MultiDict(MutableMapping, basetypes.Serializable):
             }
         """
         d = {}
-        for key in self:
-            values = self.get_all(key)
-            if len(values) == 1:
-                d[key] = values[0]
-            else:
-                d[key] = values
+        for k, v in self.collect():
+            d[k] = v
         return d
 
     def get_state(self):
diff --git a/test/mitmproxy/data/test_flow_export/python_post_json.py b/test/mitmproxy/data/test_flow_export/python_post_json.py
index 7e105bf6..6c1b9740 100644
--- a/test/mitmproxy/data/test_flow_export/python_post_json.py
+++ b/test/mitmproxy/data/test_flow_export/python_post_json.py
@@ -6,11 +6,13 @@ headers = {
     'content-type': 'application/json',
 }
 
+
 json = {
-    "name": "example",
-    "email": "example@example.com"
+    u'email': u'example@example.com',
+    u'name': u'example',
 }
 
+
 response = requests.request(
     method='POST',
     url=url,