lua: lnum: fix strtoul based number parsing

Lua's LNUM patch currently doesn't parse properly certain numbers as
it's visible from the following simple tests.

On x86_64 host (stock Lua 5.1.5, expected output):

 $ /usr/bin/lua -e 'print(0x80000000); print(0x80000000000); print(0x100000000)'

  2147483648
  8796093022208
  4294967296

On x86_64 host:

 $ staging_dir/hostpkg/bin/lua -e 'print(0x80000000); print(0x80000000000); print(0x100000000)'

  -2147483648
  0
  0

On x86_64 target:

 $ lua -e 'print(0x80000000); print(0x80000000000); print(0x100000000)'

  -2147483648
  0
  0

On ath79 target:

 $ lua -e 'print(0x80000000); print(0x80000000000); print(0x100000000)'

  -2147483648
  8796093022208
  4294967296

It's caused by two issues fixed in this patch, first issue is caused by
unhadled strtoul overflow and second one is caused by the cast of
unsigned to signed Lua integer when parsing from hex literal.

Run tested on:

 * Zidoo Z9S with RTD1296 CPU (aarch64_cortex-a53)
 * qemu/x86_64
 * qemu/armvirt_64
 * ath79

Signed-off-by: Liangbin Lian <jjm2473@gmail.com>
[commit subject/message touches, fixed From to match SOB, fixed another
 unhandled case in luaO_str2i, host Lua, package bump]
Signed-off-by: Petr Štetiar <ynezz@true.cz>

3 files changed, 97 insertions, 1 deletions

diff --git a/package/utils/lua/Makefile b/package/utils/lua/Makefile
index c34e569b67..077a60fbf3 100644
--- a/package/utils/lua/Makefile
+++ b/package/utils/lua/Makefile
@@ -9,7 +9,7 @@ include $(TOPDIR)/rules.mk
 
 PKG_NAME:=lua
 PKG_VERSION:=5.1.5
-PKG_RELEASE:=2
+PKG_RELEASE:=3
 
 PKG_SOURCE:=$(PKG_NAME)-$(PKG_VERSION).tar.gz
 PKG_SOURCE_URL:=http://www.lua.org/ftp/ \
diff --git a/package/utils/lua/patches-host/013-lnum-strtoul-parsing-fixes.patch b/package/utils/lua/patches-host/013-lnum-strtoul-parsing-fixes.patch
new file mode 100644
index 0000000000..7f00c8c3a2
--- /dev/null
+++ b/package/utils/lua/patches-host/013-lnum-strtoul-parsing-fixes.patch
@@ -0,0 +1,48 @@
+diff --git a/src/lnum.c b/src/lnum.c
+index 1456b6a2ed23..b0632b04c2b7 100644
+--- a/src/lnum.c
++++ b/src/lnum.c
+@@ -127,6 +127,8 @@ static int luaO_str2i (const char *s, lua_Integer *res, char **endptr_ref) {
+ #else
+       return 0;  /* Reject the number */
+ #endif
++    } else if (v > LUA_INTEGER_MAX) {
++      return TK_NUMBER;
+     }
+   } else if ((v > LUA_INTEGER_MAX) || (*endptr && (!isspace(*endptr)))) {
+     return TK_NUMBER;	/* not in signed range, or has '.', 'e' etc. trailing */
+@@ -310,3 +312,13 @@ int try_unmint( lua_Integer *r, lua_Integer ib ) {
+   return 0;
+ }
+ 
++#ifdef LONG_OVERFLOW_LUA_INTEGER
++unsigned LUA_INTEGER lua_str2ul( const char *str, char **endptr, int base ) {
++  unsigned long v= strtoul(str, endptr, base);
++  if ( v > LUA_INTEGER_MAX ) {
++    errno= ERANGE;
++    v= ULONG_MAX;
++  }
++  return (unsigned LUA_INTEGER)v;
++}
++#endif
+diff --git a/src/lnum_config.h b/src/lnum_config.h
+index 19d7a4231a49..1092eead6629 100644
+--- a/src/lnum_config.h
++++ b/src/lnum_config.h
+@@ -141,7 +141,12 @@
+ #endif
+ 
+ #ifndef lua_str2ul
+-# define lua_str2ul (unsigned LUA_INTEGER)strtoul
++# if LONG_MAX > LUA_INTEGER_MAX
++#   define LONG_OVERFLOW_LUA_INTEGER
++    unsigned LUA_INTEGER lua_str2ul( const char *str, char **endptr, int base );
++# else
++#  define lua_str2ul (unsigned LUA_INTEGER)strtoul
++# endif
+ #endif
+ #ifndef LUA_INTEGER_MIN
+ # define LUA_INTEGER_MIN (-LUA_INTEGER_MAX -1)  /* -2^16|32 */
+-- 
+1.9.1
+
diff --git a/package/utils/lua/patches/013-lnum-strtoul-parsing-fixes.patch b/package/utils/lua/patches/013-lnum-strtoul-parsing-fixes.patch
new file mode 100644
index 0000000000..7f00c8c3a2
--- /dev/null
+++ b/package/utils/lua/patches/013-lnum-strtoul-parsing-fixes.patch
@@ -0,0 +1,48 @@
+diff --git a/src/lnum.c b/src/lnum.c
+index 1456b6a2ed23..b0632b04c2b7 100644
+--- a/src/lnum.c
++++ b/src/lnum.c
+@@ -127,6 +127,8 @@ static int luaO_str2i (const char *s, lua_Integer *res, char **endptr_ref) {
+ #else
+       return 0;  /* Reject the number */
+ #endif
++    } else if (v > LUA_INTEGER_MAX) {
++      return TK_NUMBER;
+     }
+   } else if ((v > LUA_INTEGER_MAX) || (*endptr && (!isspace(*endptr)))) {
+     return TK_NUMBER;	/* not in signed range, or has '.', 'e' etc. trailing */
+@@ -310,3 +312,13 @@ int try_unmint( lua_Integer *r, lua_Integer ib ) {
+   return 0;
+ }
+ 
++#ifdef LONG_OVERFLOW_LUA_INTEGER
++unsigned LUA_INTEGER lua_str2ul( const char *str, char **endptr, int base ) {
++  unsigned long v= strtoul(str, endptr, base);
++  if ( v > LUA_INTEGER_MAX ) {
++    errno= ERANGE;
++    v= ULONG_MAX;
++  }
++  return (unsigned LUA_INTEGER)v;
++}
++#endif
+diff --git a/src/lnum_config.h b/src/lnum_config.h
+index 19d7a4231a49..1092eead6629 100644
+--- a/src/lnum_config.h
++++ b/src/lnum_config.h
+@@ -141,7 +141,12 @@
+ #endif
+ 
+ #ifndef lua_str2ul
+-# define lua_str2ul (unsigned LUA_INTEGER)strtoul
++# if LONG_MAX > LUA_INTEGER_MAX
++#   define LONG_OVERFLOW_LUA_INTEGER
++    unsigned LUA_INTEGER lua_str2ul( const char *str, char **endptr, int base );
++# else
++#  define lua_str2ul (unsigned LUA_INTEGER)strtoul
++# endif
+ #endif
+ #ifndef LUA_INTEGER_MIN
+ # define LUA_INTEGER_MIN (-LUA_INTEGER_MAX -1)  /* -2^16|32 */
+-- 
+1.9.1
+