update kaitai and recompile formats

10 files changed, 74 insertions, 31 deletions

diff --git a/.gitignore b/.gitignore
index c289ed50..915a81b4 100644
--- a/.gitignore
+++ b/.gitignore
@@ -11,6 +11,7 @@ MANIFEST
 .cache/
 .tox*/
 build/
+mitmproxy/contrib/kaitaistruct/*.ksy
 
 # UI
 
diff --git a/mitmproxy/contentviews/image/image_parser.py b/mitmproxy/contentviews/image/image_parser.py
index 062fb38e..7c74669a 100644
--- a/mitmproxy/contentviews/image/image_parser.py
+++ b/mitmproxy/contentviews/image/image_parser.py
@@ -37,7 +37,7 @@ def parse_gif(data: bytes) -> Metadata:
     descriptor = img.logical_screen_descriptor
     parts = [
         ('Format', 'Compuserve GIF'),
-        ('Version', "GIF{}".format(img.header.version.decode('ASCII'))),
+        ('Version', "GIF{}".format(img.hdr.version)),
         ('Size', "{} x {} px".format(descriptor.screen_width, descriptor.screen_height)),
         ('background', str(descriptor.bg_color_index))
     ]
diff --git a/mitmproxy/contrib/kaitaistruct/exif.py b/mitmproxy/contrib/kaitaistruct/exif.py
index 6236a708..2e8f1f9f 100644
--- a/mitmproxy/contrib/kaitaistruct/exif.py
+++ b/mitmproxy/contrib/kaitaistruct/exif.py
@@ -1,16 +1,18 @@
 # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
-# The source was exif.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/24e2d00048b8084ceec30a187a79cb87a79a48ba/image/exif.ksy
 
 import array
 import struct
 import zlib
 from enum import Enum
+from pkg_resources import parse_version
 
-from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO
+from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO
 
 
-from .exif_le import ExifLe
-from .exif_be import ExifBe
+if parse_version(ks_version) < parse_version('0.7'):
+    raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version))
+
+from exif_le import ExifLe
 class Exif(KaitaiStruct):
     def __init__(self, _io, _parent=None, _root=None):
         self._io = _io
@@ -22,3 +24,5 @@ class Exif(KaitaiStruct):
             self.body = ExifLe(self._io)
         elif _on == 19789:
             self.body = ExifBe(self._io)
+
+
diff --git a/mitmproxy/contrib/kaitaistruct/exif_be.py b/mitmproxy/contrib/kaitaistruct/exif_be.py
index 7980a9e8..8a6e7a2b 100644
--- a/mitmproxy/contrib/kaitaistruct/exif_be.py
+++ b/mitmproxy/contrib/kaitaistruct/exif_be.py
@@ -1,14 +1,17 @@
 # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
-# The source was exif_be.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/24e2d00048b8084ceec30a187a79cb87a79a48ba/image/exif_be.ksy
 
 import array
 import struct
 import zlib
 from enum import Enum
+from pkg_resources import parse_version
 
-from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO
+from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO
 
 
+if parse_version(ks_version) < parse_version('0.7'):
+    raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version))
+
 class ExifBe(KaitaiStruct):
     def __init__(self, _io, _parent=None, _root=None):
         self._io = _io
@@ -569,3 +572,5 @@ class ExifBe(KaitaiStruct):
         self._m_ifd0 = self._root.Ifd(self._io, self, self._root)
         self._io.seek(_pos)
         return self._m_ifd0 if hasattr(self, '_m_ifd0') else None
+
+
diff --git a/mitmproxy/contrib/kaitaistruct/exif_le.py b/mitmproxy/contrib/kaitaistruct/exif_le.py
index 207b3beb..84e53a38 100644
--- a/mitmproxy/contrib/kaitaistruct/exif_le.py
+++ b/mitmproxy/contrib/kaitaistruct/exif_le.py
@@ -1,14 +1,17 @@
 # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
-# The source was exif_le.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/24e2d00048b8084ceec30a187a79cb87a79a48ba/image/exif_le.ksy
 
 import array
 import struct
 import zlib
 from enum import Enum
+from pkg_resources import parse_version
 
-from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO
+from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO
 
 
+if parse_version(ks_version) < parse_version('0.7'):
+    raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version))
+
 class ExifLe(KaitaiStruct):
     def __init__(self, _io, _parent=None, _root=None):
         self._io = _io
@@ -569,3 +572,5 @@ class ExifLe(KaitaiStruct):
         self._m_ifd0 = self._root.Ifd(self._io, self, self._root)
         self._io.seek(_pos)
         return self._m_ifd0 if hasattr(self, '_m_ifd0') else None
+
+
diff --git a/mitmproxy/contrib/kaitaistruct/gif.py b/mitmproxy/contrib/kaitaistruct/gif.py
index 61499cc7..820df568 100644
--- a/mitmproxy/contrib/kaitaistruct/gif.py
+++ b/mitmproxy/contrib/kaitaistruct/gif.py
@@ -1,14 +1,17 @@
 # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
-# The source was png.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/562154250bea0081fed4e232751b934bc270a0c7/image/gif.ksy
 
 import array
 import struct
 import zlib
 from enum import Enum
+from pkg_resources import parse_version
 
-from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO
+from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO
 
 
+if parse_version(ks_version) < parse_version('0.7'):
+    raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version))
+
 class Gif(KaitaiStruct):
 
     class BlockType(Enum):
@@ -24,8 +27,8 @@ class Gif(KaitaiStruct):
         self._io = _io
         self._parent = _parent
         self._root = _root if _root else self
-        self.header = self._root.Header(self._io, self, self._root)
-        self.logical_screen_descriptor = self._root.LogicalScreenDescriptor(self._io, self, self._root)
+        self.hdr = self._root.Header(self._io, self, self._root)
+        self.logical_screen_descriptor = self._root.LogicalScreenDescriptorStruct(self._io, self, self._root)
         if self.logical_screen_descriptor.has_color_table:
             self._raw_global_color_table = self._io.read_bytes((self.logical_screen_descriptor.color_table_size * 3))
             io = KaitaiStream(BytesIO(self._raw_global_color_table))
@@ -55,7 +58,7 @@ class Gif(KaitaiStruct):
             self.blue = self._io.read_u1()
 
 
-    class LogicalScreenDescriptor(KaitaiStruct):
+    class LogicalScreenDescriptorStruct(KaitaiStruct):
         def __init__(self, _io, _parent=None, _root=None):
             self._io = _io
             self._parent = _parent
@@ -163,7 +166,7 @@ class Gif(KaitaiStruct):
             self._parent = _parent
             self._root = _root if _root else self
             self.magic = self._io.ensure_fixed_contents(struct.pack('3b', 71, 73, 70))
-            self.version = self._io.read_bytes(3)
+            self.version = (self._io.read_bytes(3)).decode(u"ASCII")
 
 
     class ExtGraphicControl(KaitaiStruct):
@@ -245,3 +248,6 @@ class Gif(KaitaiStruct):
                 self.body = self._root.ExtGraphicControl(self._io, self, self._root)
             else:
                 self.body = self._root.Subblocks(self._io, self, self._root)
+
+
+
diff --git a/mitmproxy/contrib/kaitaistruct/jpeg.py b/mitmproxy/contrib/kaitaistruct/jpeg.py
index 08e382a9..a49c83b9 100644
--- a/mitmproxy/contrib/kaitaistruct/jpeg.py
+++ b/mitmproxy/contrib/kaitaistruct/jpeg.py
@@ -1,15 +1,18 @@
 # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
-# The source was jpeg.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/24e2d00048b8084ceec30a187a79cb87a79a48ba/image/jpeg.ksy
 
 import array
 import struct
 import zlib
 from enum import Enum
+from pkg_resources import parse_version
 
-from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO
+from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO
 
 
-from .exif import Exif
+if parse_version(ks_version) < parse_version('0.7'):
+    raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version))
+
+from exif import Exif
 class Jpeg(KaitaiStruct):
 
     class ComponentId(Enum):
@@ -127,7 +130,7 @@ class Jpeg(KaitaiStruct):
             self._io = _io
             self._parent = _parent
             self._root = _root if _root else self
-            self.magic = self._io.read_strz("ASCII", 0, False, True, True)
+            self.magic = (self._io.read_bytes_term(0, False, True, True)).decode(u"ASCII")
             _on = self.magic
             if _on == u"Exif":
                 self.body = self._root.ExifInJpeg(self._io, self, self._root)
@@ -195,7 +198,7 @@ class Jpeg(KaitaiStruct):
             self._io = _io
             self._parent = _parent
             self._root = _root if _root else self
-            self.magic = self._io.read_str_byte_limit(5, "ASCII")
+            self.magic = (self._io.read_bytes(5)).decode(u"ASCII")
             self.version_major = self._io.read_u1()
             self.version_minor = self._io.read_u1()
             self.density_units = self._root.SegmentApp0.DensityUnit(self._io.read_u1())
@@ -204,3 +207,6 @@ class Jpeg(KaitaiStruct):
             self.thumbnail_x = self._io.read_u1()
             self.thumbnail_y = self._io.read_u1()
             self.thumbnail = self._io.read_bytes(((self.thumbnail_x * self.thumbnail_y) * 3))
+
+
+
diff --git a/mitmproxy/contrib/kaitaistruct/make.sh b/mitmproxy/contrib/kaitaistruct/make.sh
new file mode 100755
index 00000000..218d5198
--- /dev/null
+++ b/mitmproxy/contrib/kaitaistruct/make.sh
@@ -0,0 +1,10 @@
+#!/usr/bin/env bash
+
+wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/exif_be.ksy
+wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/exif_le.ksy
+wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/exif.ksy
+wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/gif.ksy
+wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/jpeg.ksy
+wget -N https://raw.githubusercontent.com/kaitai-io/kaitai_struct_formats/master/image/png.ksy
+
+kaitai-struct-compiler --target python --opaque-types=true *.ksy
diff --git a/mitmproxy/contrib/kaitaistruct/png.py b/mitmproxy/contrib/kaitaistruct/png.py
index 2f3c1a5c..98a70693 100644
--- a/mitmproxy/contrib/kaitaistruct/png.py
+++ b/mitmproxy/contrib/kaitaistruct/png.py
@@ -1,14 +1,17 @@
 # This is a generated file! Please edit source .ksy file and use kaitai-struct-compiler to rebuild
-# The source was png.ksy from here - https://github.com/kaitai-io/kaitai_struct_formats/blob/9370c720b7d2ad329102d89bdc880ba6a706ef26/image/png.ksy
 
 import array
 import struct
 import zlib
 from enum import Enum
+from pkg_resources import parse_version
 
-from kaitaistruct import KaitaiStruct, KaitaiStream, BytesIO
+from kaitaistruct import __version__ as ks_version, KaitaiStruct, KaitaiStream, BytesIO
 
 
+if parse_version(ks_version) < parse_version('0.7'):
+    raise Exception("Incompatible Kaitai Struct Python API: 0.7 or later is required, but you have %s" % (ks_version))
+
 class Png(KaitaiStruct):
 
     class ColorType(Enum):
@@ -51,7 +54,7 @@ class Png(KaitaiStruct):
             self._parent = _parent
             self._root = _root if _root else self
             self.len = self._io.read_u4be()
-            self.type = self._io.read_str_byte_limit(4, "UTF-8")
+            self.type = (self._io.read_bytes(4)).decode(u"UTF-8")
             _on = self.type
             if _on == u"iTXt":
                 self._raw_body = self._io.read_bytes(self.len)
@@ -194,7 +197,7 @@ class Png(KaitaiStruct):
             self._io = _io
             self._parent = _parent
             self._root = _root if _root else self
-            self.keyword = self._io.read_strz("UTF-8", 0, False, True, True)
+            self.keyword = (self._io.read_bytes_term(0, False, True, True)).decode(u"UTF-8")
             self.compression_method = self._io.read_u1()
             self._raw_text_datastream = self._io.read_bytes_full()
             self.text_datastream = zlib.decompress(self._raw_text_datastream)
@@ -259,12 +262,12 @@ class Png(KaitaiStruct):
             self._io = _io
             self._parent = _parent
             self._root = _root if _root else self
-            self.keyword = self._io.read_strz("UTF-8", 0, False, True, True)
+            self.keyword = (self._io.read_bytes_term(0, False, True, True)).decode(u"UTF-8")
             self.compression_flag = self._io.read_u1()
             self.compression_method = self._io.read_u1()
-            self.language_tag = self._io.read_strz("ASCII", 0, False, True, True)
-            self.translated_keyword = self._io.read_strz("UTF-8", 0, False, True, True)
-            self.text = self._io.read_str_eos("UTF-8")
+            self.language_tag = (self._io.read_bytes_term(0, False, True, True)).decode(u"ASCII")
+            self.translated_keyword = (self._io.read_bytes_term(0, False, True, True)).decode(u"UTF-8")
+            self.text = (self._io.read_bytes_full()).decode(u"UTF-8")
 
 
     class TextChunk(KaitaiStruct):
@@ -272,8 +275,8 @@ class Png(KaitaiStruct):
             self._io = _io
             self._parent = _parent
             self._root = _root if _root else self
-            self.keyword = self._io.read_strz("iso8859-1", 0, False, True, True)
-            self.text = self._io.read_str_eos("iso8859-1")
+            self.keyword = (self._io.read_bytes_term(0, False, True, True)).decode(u"iso8859-1")
+            self.text = (self._io.read_bytes_full()).decode(u"iso8859-1")
 
 
     class TimeChunk(KaitaiStruct):
@@ -287,3 +290,6 @@ class Png(KaitaiStruct):
             self.hour = self._io.read_u1()
             self.minute = self._io.read_u1()
             self.second = self._io.read_u1()
+
+
+
diff --git a/setup.py b/setup.py
index a758a31b..cb25454e 100644
--- a/setup.py
+++ b/setup.py
@@ -70,7 +70,7 @@ setup(
         "html2text>=2016.1.8, <=2016.9.19",
         "hyperframe>=5.0, <6",
         "jsbeautifier>=1.6.3, <1.7",
-        "kaitaistruct>=0.6, <0.7",
+        "kaitaistruct>=0.7, <0.8",
         "passlib>=1.6.5, <1.8",
         "pyasn1>=0.1.9, <0.3",
         "pyOpenSSL>=16.0, <17.0",