package/utils/bsdiff/Makefile


1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
48
49
50
51
52
53
54
55
56
57
58
59
60
61
62
63
64
65
66
67
68
69
70
71
72
73
74
75
76
77
78
79

#
# Copyright (C) 2016 OpenWrt.org
#
# This is free software, licensed under the GNU General Public License v2.
# See /LICENSE for more information.
#

include $(TOPDIR)/rules.mk

PKG_NAME:=bsdiff
PKG_VERSION:=4.3
PKG_RELEASE:=1

PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz
PKG_SOURCE_URL:=http://www.daemonology.net/bsdiff/
PKG_HASH:=18821588b2dc5bf159aa37d3bcb7b885d85ffd1e19f23a0c57a58723fea85f48
PKG_MAINTAINER:=Hauke Mehrtens <hauke@hauke-m.de>
HOST_BUILD_DEPENDS:=bzip2/host

PKG_LICENSE:=BSD-2-Clause

include $(INCLUDE_DIR)/host-build.mk
include $(INCLUDE_DIR)/package.mk

define Package/bsdiff
  SECTION:=utils
  CATEGORY:=Utilities
  DEPENDS:=+libbz2
  TITLE:=Binary diff tool
  URL:=http://www.daemonology.net/bsdiff/
endef

define Package/bspatch
  SECTION:=utils
  CATEGORY:=Utilities
  DEPENDS:=+libbz2
  TITLE:=Binary patch tool
  URL:=http://www.daemonology.net/bsdiff/
endef


define Build/Compile
	$(TARGET_CC) $(TARGET_CFLAGS) $(TARGET_CPPFLAGS) $(TARGET_LDFLAGS) \
		-o $(PKG_BUILD_DIR)/bsdiff \
		$(PKG_BUILD_DIR)/bsdiff.c -lbz2
	$(TARGET_CC) $(TARGET_CFLAGS) $(TARGET_CPPFLAGS) $(TARGET_LDFLAGS) \
		-o $(PKG_BUILD_DIR)/bspatch \
		$(PKG_BUILD_DIR)/bspatch.c -lbz2
endef

define Package/bsdiff/install
	$(INSTALL_DIR) $(1)/usr/bin/
	$(INSTALL_BIN) $(PKG_BUILD_DIR)/bsdiff $(1)/usr/bin/bsdiff
endef

define Package/bspatch/install
	$(INSTALL_DIR) $(1)/usr/bin/
	$(INSTALL_BIN) $(PKG_BUILD_DIR)/bspatch $(1)/usr/bin/bspatch
endef

define Host/Install
	$(INSTALL_DIR) $(STAGING_DIR_HOSTPKG)/bin/
	$(MAKE) -C $(HOST_BUILD_DIR) PREFIX=$(STAGING_DIR_HOSTPKG)/ install
endef

define Host/Compile
	$(HOSTCC) $(HOST_CFLAGS) $(HOST_LDFLAGS) \
		-o $(HOST_BUILD_DIR)/bsdiff \
		$(HOST_BUILD_DIR)/bsdiff.c -lbz2
endef

define Host/Install
	$(INSTALL_BIN) $(HOST_BUILD_DIR)/bsdiff $(STAGING_DIR_HOSTPKG)/bin/
endef

$(eval $(call HostBuild))

$(eval $(call BuildPackage,bsdiff))
$(eval $(call BuildPackage,bspatch))
import operator
import os
import abc
import functools
import pyparsing as pp

from mitmproxy.utils import strutils
from mitmproxy.utils import human

from . import generators, exceptions


class Settings:

    def __init__(
        self,
        is_client=False,
        staticdir=None,
        unconstrained_file_access=False,
        request_host=None,
        websocket_key=None,
        protocol=None,
    ):
        self.is_client = is_client
        self.staticdir = staticdir
        self.unconstrained_file_access = unconstrained_file_access
        self.request_host = request_host
        self.websocket_key = websocket_key  # TODO: refactor this into the protocol
        self.protocol = protocol


Sep = pp.Optional(pp.Literal(":")).suppress()


v_integer = pp.Word(pp.nums)\
    .setName("integer")\
    .setParseAction(lambda toks: int(toks[0]))


v_literal = pp.MatchFirst(
    [
        pp.QuotedString(
            "\"",
            unquoteResults=True,
            multiline=True
        ),
        pp.QuotedString(
            "'",
            unquoteResults=True,
            multiline=True
        ),
    ]
)

v_naked_literal = pp.MatchFirst(
    [
        v_literal,
        pp.Word("".join(i for i in pp.printables if i not in ",:\n@\'\""))
    ]
)


class Token:

    """
        A token in the specification language. Tokens are immutable. The token
        classes have no meaning in and of themselves, and are combined into
        Components and Actions to build the language.
    """
    __metaclass__ = abc.ABCMeta

    @classmethod
    def expr(cls):  # pragma: no cover
        """
            A parse expression.
        """
        return None

    @abc.abstractmethod
    def spec(self):  # pragma: no cover
        """
            A parseable specification for this token.
        """
        return None

    @property
    def unique_name(self):
        """
            Controls uniqueness constraints for tokens. No two tokens with the
            same name will be allowed. If no uniquness should be applied, this
            should be None.
        """
        return self.__class__.__name__.lower()

    def resolve(self, settings_, msg_):
        """
            Resolves this token to ready it for transmission. This means that
            the calculated offsets of actions are fixed.

            settings: a language.Settings instance
            msg: The containing message
        """
        return self

    def __repr__(self):
        return self.spec()


class _TokValueLiteral(Token):

    def __init__(self, val):
        self.val = strutils.escaped_str_to_bytes(val)

    def get_generator(self, settings_):
        return self.val

    def freeze(self, settings_):
        return self


class TokValueLiteral(_TokValueLiteral):

    """
        A literal with Python-style string escaping
    """
    @classmethod
    def expr(cls):
        e = v_literal.copy()
        return e.setParseAction(cls.parseAction)

    @classmethod
    def parseAction(cls, x):
        v = cls(*x)
        return v

    def spec(self):
        inner = strutils.bytes_to_escaped_str(self.val)
        inner = inner.replace(r"'", r"\x27")
        return "'" + inner + "'"


class TokValueNakedLiteral(_TokValueLiteral):

    @classmethod
    def expr(cls):
        e = v_naked_literal.copy()
        return e.setParseAction(lambda x: cls(*x))

    def spec(self):
        return strutils.bytes_to_escaped_str(self.val, escape_single_quotes=True)


class TokValueGenerate(Token):

    def __init__(self, usize, unit, datatype):
        if not unit:
            unit = "b"
        self.usize, self.unit, self.datatype = usize, unit, datatype

    def bytes(self):
        return self.usize * human.SIZE_UNITS[self.unit]

    def get_generator(self, settings_):
        return generators.RandomGenerator(self.datatype, self.bytes())

    def freeze(self, settings):
        g = self.get_generator(settings)
        return TokValueLiteral(strutils.bytes_to_escaped_str(g[:], escape_single_quotes=True))

    @classmethod
    def expr(cls):
        e = pp.Literal("@").suppress() + v_integer

        u = functools.reduce(
            operator.or_,
            [pp.Literal(i) for i in human.SIZE_UNITS.keys()]
        ).leaveWhitespace()
        e = e + pp.Optional(u, default=None)

        s = pp.Literal(",").suppress()
        s += functools.reduce(
            operator.or_,
            [pp.Literal(i) for i in generators.DATATYPES.keys()]
        )
        e += pp.Optional(s, default="bytes")
        return e.setParseAction(lambda x: cls(*x))

    def spec(self):
        s = "@%s" % self.usize
        if self.unit != "b":
            s += self.unit
        if self.datatype != "bytes":
            s += ",%s" % self.datatype
        return s


class TokValueFile(Token):

    def __init__(self, path):
        self.path = str(path)

    @classmethod
    def expr(cls):
        e = pp.Literal("<").suppress()
        e = e + v_naked_literal
        return e.setParseAction(lambda x: cls(*x))

    def freeze(self, settings_):
        return self

    def get_generator(self, settings):
        if not settings.staticdir:
            raise exceptions.FileAccessDenied("File access disabled.")
        s = os.path.expanduser(self.path)
        s = os.path.normpath(
            os.path.abspath(os.path.join(settings.staticdir, s))
        )
        uf = settings.unconstrained_file_access
        if not uf and not s.startswith(os.path.normpath(settings.staticdir)):
            raise exceptions.FileAccessDenied(
                "File access outside of configured directory"
            )
        if not os.path.isfile(s):
            raise exceptions.FileAccessDenied("File not readable")
        return generators.FileGenerator(s)

    def spec(self):
        return "<'%s'" % self.path


TokValue = pp.MatchFirst(
    [
        TokValueGenerate.expr(),
        TokValueFile.expr(),
        TokValueLiteral.expr()
    ]
)


TokNakedValue = pp.MatchFirst(
    [
        TokValueGenerate.expr(),
        TokValueFile.expr(),
        TokValueLiteral.expr(),
        TokValueNakedLiteral.expr(),
    ]
)


TokOffset = pp.MatchFirst(
    [
        v_integer,
        pp.Literal("r"),
        pp.Literal("a")
    ]
)


class _Component(Token):

    """
        A value component of the primary specification of an message.
        Components produce byte values describing the bytes of the message.
    """

    def values(self, settings):  # pragma: no cover
        """
           A sequence of values, which can either be strings or generators.
        """
        pass

    def string(self, settings=None):
        """
            A bytestring representation of the object.
        """
        return b"".join(i[:] for i in self.values(settings or {}))


class KeyValue(_Component):

    """
        A key/value pair.
        cls.preamble: leader
    """

    def __init__(self, key, value):
        self.key, self.value = key, value

    @classmethod
    def expr(cls):
        e = pp.Literal(cls.preamble).suppress()
        e += TokValue
        e += pp.Literal("=").suppress()
        e += TokValue
        return e.setParseAction(lambda x: cls(*x))

    def spec(self):
        return "%s%s=%s" % (self.preamble, self.key.spec(), self.value.spec())

    def freeze(self, settings):
        return self.__class__(
            self.key.freeze(settings), self.value.freeze(settings)
        )


class CaselessLiteral(_Component):

    """
        A caseless token that can take only one value.
    """

    def __init__(self, value):
        self.value = value

    @classmethod
    def expr(cls):
        spec = pp.CaselessLiteral(cls.TOK)
        spec = spec.setParseAction(lambda x: cls(*x))
        return spec

    def values(self, settings):
        return self.TOK

    def spec(self):
        return self.TOK

    def freeze(self, settings_):
        return self


class OptionsOrValue(_Component):

    """
        Can be any of a specified set of options, or a value specifier.
    """
    preamble = ""
    options = []

    def __init__(self, value):
        # If it's a string, we were passed one of the options, so we lower-case
        # it to be canonical. The user can specify a different case by using a
        # string value literal.
        self.option_used = False
        if isinstance(value, str):
            for i in self.options:
                # Find the exact option value in a case-insensitive way
                if i.lower() == value.lower():
                    self.option_used = True
                    value = TokValueLiteral(i)
                    break
        self.value = value

    @classmethod
    def expr(cls):
        parts = [pp.CaselessLiteral(i) for i in cls.options]
        m = pp.MatchFirst(parts)
        spec = m | TokValue.copy()
        spec = spec.setParseAction(lambda x: cls(*x))
        if cls.preamble:
            spec = pp.Literal(cls.preamble).suppress() + spec
        return spec

    def values(self, settings):
        return [
            self.value.get_generator(settings)
        ]

    def spec(self):
        s = self.value.spec()
        if s[1:-1].lower() in self.options:
            s = s[1:-1].lower()
        return "%s%s" % (self.preamble, s)

    def freeze(self, settings):
        return self.__class__(self.value.freeze(settings))


class Integer(_Component):
    bounds = (None, None)
    preamble = ""

    def __init__(self, value):
        v = int(value)
        outofbounds = any([
            self.bounds[0] is not None and v < self.bounds[0],
            self.bounds[1] is not None and v > self.bounds[1]
        ])
        if outofbounds:
            raise exceptions.ParseException(
                "Integer value must be between %s and %s." % self.bounds,
                0, 0
            )
        self.value = str(value).encode()

    @classmethod
    def expr(cls):
        e = v_integer.copy()
        if cls.preamble:
            e = pp.Literal(cls.preamble).suppress() + e
        return e.setParseAction(lambda x: cls(*x))

    def values(self, settings):
        return [self.value]

    def spec(self):
        return "%s%s" % (self.preamble, self.value.decode())

    def freeze(self, settings_):
        return self


class Value(_Component):

    """
        A value component lead by an optional preamble.
    """
    preamble = ""

    def __init__(self, value):
        self.value = value

    @classmethod
    def expr(cls):
        e = (TokValue | TokNakedValue)
        if cls.preamble:
            e = pp.Literal(cls.preamble).suppress() + e
        return e.setParseAction(lambda x: cls(*x))

    def values(self, settings):
        return [self.value.get_generator(settings)]

    def spec(self):
        return "%s%s" % (self.preamble, self.value.spec())

    def freeze(self, settings):
        return self.__class__(self.value.freeze(settings))


class FixedLengthValue(Value):

    """
        A value component lead by an optional preamble.
    """
    preamble = ""
    length = None

    def __init__(self, value):
        Value.__init__(self, value)
        lenguess = None
        try:
            lenguess = len(value.get_generator(Settings()))
        except exceptions.RenderError:
            pass
        # This check will fail if we know the length upfront
        if lenguess is not None and lenguess != self.length:
            raise exceptions.RenderError(
                "Invalid value length: '%s' is %s bytes, should be %s." % (
                    self.spec(),
                    lenguess,
                    self.length
                )
            )

    def values(self, settings):
        ret = Value.values(self, settings)
        l = sum(len(i) for i in ret)
        # This check will fail if we don't know the length upfront - i.e. for
        # file inputs
        if l != self.length:
            raise exceptions.RenderError(
                "Invalid value length: '%s' is %s bytes, should be %s." % (
                    self.spec(),
                    l,
                    self.length
                )
            )
        return ret


class Boolean(_Component):

    """
        A boolean flag.
            name  = true
            -name = false
    """
    name = ""

    def __init__(self, value):
        self.value = value

    @classmethod
    def expr(cls):
        e = pp.Optional(pp.Literal("-"), default=True)
        e += pp.Literal(cls.name).suppress()

        def parse(s_, loc_, toks):
            val = True
            if toks[0] == "-":
                val = False
            return cls(val)

        return e.setParseAction(parse)

    def spec(self):
        return "%s%s" % ("-" if not self.value else "", self.name)


class IntField(_Component):

    """
        An integer field, where values can optionally specified by name.
    """
    names = {}
    max = 16
    preamble = ""

    def __init__(self, value):
        self.origvalue = value
        self.value = self.names.get(value, value)
        if self.value > self.max:
            raise exceptions.ParseException(
                "Value can't exceed %s" % self.max, 0, 0
            )

    @classmethod
    def expr(cls):
        parts = [pp.CaselessLiteral(i) for i in cls.names.keys()]
        m = pp.MatchFirst(parts)
        spec = m | v_integer.copy()
        spec = spec.setParseAction(lambda x: cls(*x))
        if cls.preamble:
            spec = pp.Literal(cls.preamble).suppress() + spec
        return spec

    def values(self, settings):
        return [str(self.value)]

    def spec(self):
        return "%s%s" % (self.preamble, self.origvalue)


class NestedMessage(Token):

    """
        A nested message, as an escaped string with a preamble.
    """
    preamble = ""
    nest_type = None

    def __init__(self, value):
        Token.__init__(self)
        self.value = value
        try:
            self.parsed = self.nest_type(
                self.nest_type.expr().parseString(
                    value.val.decode(),
                    parseAll=True
                )
            )
        except pp.ParseException as v:
            raise exceptions.ParseException(v.msg, v.line, v.col)

    @classmethod
    def expr(cls):
        e = pp.Literal(cls.preamble).suppress()
        e = e + TokValueLiteral.expr()
        return e.setParseAction(lambda x: cls(*x))

    def values(self, settings):
        return [
            self.value.get_generator(settings),
        ]

    def spec(self):
        return "%s%s" % (self.preamble, self.value.spec())

    def freeze(self, settings):
        f = self.parsed.freeze(settings).spec()
        return self.__class__(TokValueLiteral(strutils.bytes_to_escaped_str(f.encode(), escape_single_quotes=True)))