Merge pull request #2141 from mhils/decouple-file-format-version

Decouple mitmproxy and file format version

3 files changed, 34 insertions, 9 deletions

diff --git a/mitmproxy/flow.py b/mitmproxy/flow.py
index cc5f0aed..73d6090e 100644
--- a/mitmproxy/flow.py
+++ b/mitmproxy/flow.py
@@ -93,7 +93,7 @@ class Flow(stateobject.StateObject):
 
     def get_state(self):
         d = super().get_state()
-        d.update(version=version.IVERSION)
+        d.update(version=version.FLOW_FORMAT_VERSION)
         if self._backup and self._backup != d:
             d.update(backup=self._backup)
         return d
diff --git a/mitmproxy/io_compat.py b/mitmproxy/io_compat.py
index 16cbc9fe..8fa7b77c 100644
--- a/mitmproxy/io_compat.py
+++ b/mitmproxy/io_compat.py
@@ -2,7 +2,7 @@
 This module handles the import of mitmproxy flows generated by old versions.
 """
 
-from typing import Any
+from typing import Any, Dict
 
 from mitmproxy import version
 from mitmproxy.utils import strutils
@@ -82,6 +82,9 @@ def convert_018_019(data):
 
 
 def convert_019_100(data):
+    # convert_unicode needs to be called for every dual release and the first py3-only release
+    data = convert_unicode(data)
+
     data["version"] = (1, 0, 0)
     return data
 
@@ -105,6 +108,11 @@ def convert_200_300(data):
     return data
 
 
+def convert_300_4(data):
+    data["version"] = 4
+    return data
+
+
 def _convert_dict_keys(o: Any) -> Any:
     if isinstance(o, dict):
         return {strutils.always_str(k): _convert_dict_keys(v) for k, v in o.items()}
@@ -155,19 +163,32 @@ converters = {
     (0, 19): convert_019_100,
     (1, 0): convert_100_200,
     (2, 0): convert_200_300,
+    (3, 0): convert_300_4,
 }
 
 
-def migrate_flow(flow_data):
+def migrate_flow(flow_data: Dict[str, Any]) -> Dict[str, Any]:
     while True:
-        flow_version = tuple(flow_data.get(b"version", flow_data.get("version")))
-        if flow_version[:2] == version.IVERSION[:2]:
+        flow_version = flow_data.get(b"version", flow_data.get("version"))
+
+        # Historically, we used the mitmproxy minor version tuple as the flow format version.
+        if not isinstance(flow_version, int):
+            flow_version = tuple(flow_version)[:2]
+
+        if flow_version == version.FLOW_FORMAT_VERSION:
             break
-        elif flow_version[:2] in converters:
-            flow_data = converters[flow_version[:2]](flow_data)
+        elif flow_version in converters:
+            flow_data = converters[flow_version](flow_data)
         else:
-            v = ".".join(str(i) for i in flow_version)
+            should_upgrade = (
+                isinstance(flow_version, int)
+                and flow_version > version.FLOW_FORMAT_VERSION
+            )
             raise ValueError(
-                "{} cannot read files serialized with version {}.".format(version.MITMPROXY, v)
+                "{} cannot read files with flow format version {}{}.".format(
+                    version.MITMPROXY,
+                    flow_version,
+                    ", please update mitmproxy" if should_upgrade else ""
+                )
             )
     return flow_data
diff --git a/mitmproxy/version.py b/mitmproxy/version.py
index d23b2d19..2882d1fb 100644
--- a/mitmproxy/version.py
+++ b/mitmproxy/version.py
@@ -3,5 +3,9 @@ VERSION = ".".join(str(i) for i in IVERSION)
 PATHOD = "pathod " + VERSION
 MITMPROXY = "mitmproxy " + VERSION
 
+# Serialization format version. This is displayed nowhere, it just needs to be incremented by one
+# for each change the the file format.
+FLOW_FORMAT_VERSION = 4
+
 if __name__ == "__main__":
     print(VERSION)