kernel: backport improved checksum function for ARM64

Improves network performance in some cases when checksum offload is not
available

Signed-off-by: Felix Fietkau <nbd@nbd.name>

2 files changed, 204 insertions, 0 deletions

diff --git a/target/linux/generic/backport-5.4/041-v5.5-arm64-Implement-optimised-checksum-routine.patch b/target/linux/generic/backport-5.4/041-v5.5-arm64-Implement-optimised-checksum-routine.patch
new file mode 100644
index 0000000000..00ec7d0207
--- /dev/null
+++ b/target/linux/generic/backport-5.4/041-v5.5-arm64-Implement-optimised-checksum-routine.patch
@@ -0,0 +1,176 @@
+From: Robin Murphy <robin.murphy@arm.com>
+Date: Wed, 15 Jan 2020 16:42:39 +0000
+Subject: [PATCH] arm64: Implement optimised checksum routine
+
+Apparently there exist certain workloads which rely heavily on software
+checksumming, for which the generic do_csum() implementation becomes a
+significant bottleneck. Therefore let's give arm64 its own optimised
+version - for ease of maintenance this foregoes assembly or intrisics,
+and is thus not actually arm64-specific, but does rely heavily on C
+idioms that translate well to the A64 ISA and the typical load/store
+capabilities of most ARMv8 CPU cores.
+
+The resulting increase in checksum throughput scales nicely with buffer
+size, tending towards 4x for a small in-order core (Cortex-A53), and up
+to 6x or more for an aggressive big core (Ampere eMAG).
+
+Reported-by: Lingyan Huang <huanglingyan2@huawei.com>
+Tested-by: Lingyan Huang <huanglingyan2@huawei.com>
+Signed-off-by: Robin Murphy <robin.murphy@arm.com>
+Signed-off-by: Will Deacon <will@kernel.org>
+---
+ create mode 100644 arch/arm64/lib/csum.c
+
+--- a/arch/arm64/include/asm/checksum.h
++++ b/arch/arm64/include/asm/checksum.h
+@@ -36,6 +36,9 @@ static inline __sum16 ip_fast_csum(const
+ }
+ #define ip_fast_csum ip_fast_csum
+ 
++extern unsigned int do_csum(const unsigned char *buff, int len);
++#define do_csum do_csum
++
+ #include <asm-generic/checksum.h>
+ 
+ #endif	/* __ASM_CHECKSUM_H */
+--- a/arch/arm64/lib/Makefile
++++ b/arch/arm64/lib/Makefile
+@@ -1,9 +1,9 @@
+ # SPDX-License-Identifier: GPL-2.0
+ lib-y		:= clear_user.o delay.o copy_from_user.o		\
+ 		   copy_to_user.o copy_in_user.o copy_page.o		\
+-		   clear_page.o memchr.o memcpy.o memmove.o memset.o	\
+-		   memcmp.o strcmp.o strncmp.o strlen.o strnlen.o	\
+-		   strchr.o strrchr.o tishift.o
++		   clear_page.o csum.o memchr.o memcpy.o memmove.o	\
++		   memset.o memcmp.o strcmp.o strncmp.o strlen.o	\
++		   strnlen.o strchr.o strrchr.o tishift.o
+ 
+ ifeq ($(CONFIG_KERNEL_MODE_NEON), y)
+ obj-$(CONFIG_XOR_BLOCKS)	+= xor-neon.o
+--- /dev/null
++++ b/arch/arm64/lib/csum.c
+@@ -0,0 +1,123 @@
++// SPDX-License-Identifier: GPL-2.0-only
++// Copyright (C) 2019-2020 Arm Ltd.
++
++#include <linux/compiler.h>
++#include <linux/kasan-checks.h>
++#include <linux/kernel.h>
++
++#include <net/checksum.h>
++
++/* Looks dumb, but generates nice-ish code */
++static u64 accumulate(u64 sum, u64 data)
++{
++	__uint128_t tmp = (__uint128_t)sum + data;
++	return tmp + (tmp >> 64);
++}
++
++unsigned int do_csum(const unsigned char *buff, int len)
++{
++	unsigned int offset, shift, sum;
++	const u64 *ptr;
++	u64 data, sum64 = 0;
++
++	offset = (unsigned long)buff & 7;
++	/*
++	 * This is to all intents and purposes safe, since rounding down cannot
++	 * result in a different page or cache line being accessed, and @buff
++	 * should absolutely not be pointing to anything read-sensitive. We do,
++	 * however, have to be careful not to piss off KASAN, which means using
++	 * unchecked reads to accommodate the head and tail, for which we'll
++	 * compensate with an explicit check up-front.
++	 */
++	kasan_check_read(buff, len);
++	ptr = (u64 *)(buff - offset);
++	len = len + offset - 8;
++
++	/*
++	 * Head: zero out any excess leading bytes. Shifting back by the same
++	 * amount should be at least as fast as any other way of handling the
++	 * odd/even alignment, and means we can ignore it until the very end.
++	 */
++	shift = offset * 8;
++	data = READ_ONCE_NOCHECK(*ptr++);
++#ifdef __LITTLE_ENDIAN
++	data = (data >> shift) << shift;
++#else
++	data = (data << shift) >> shift;
++#endif
++
++	/*
++	 * Body: straightforward aligned loads from here on (the paired loads
++	 * underlying the quadword type still only need dword alignment). The
++	 * main loop strictly excludes the tail, so the second loop will always
++	 * run at least once.
++	 */
++	while (unlikely(len > 64)) {
++		__uint128_t tmp1, tmp2, tmp3, tmp4;
++
++		tmp1 = READ_ONCE_NOCHECK(*(__uint128_t *)ptr);
++		tmp2 = READ_ONCE_NOCHECK(*(__uint128_t *)(ptr + 2));
++		tmp3 = READ_ONCE_NOCHECK(*(__uint128_t *)(ptr + 4));
++		tmp4 = READ_ONCE_NOCHECK(*(__uint128_t *)(ptr + 6));
++
++		len -= 64;
++		ptr += 8;
++
++		/* This is the "don't dump the carry flag into a GPR" idiom */
++		tmp1 += (tmp1 >> 64) | (tmp1 << 64);
++		tmp2 += (tmp2 >> 64) | (tmp2 << 64);
++		tmp3 += (tmp3 >> 64) | (tmp3 << 64);
++		tmp4 += (tmp4 >> 64) | (tmp4 << 64);
++		tmp1 = ((tmp1 >> 64) << 64) | (tmp2 >> 64);
++		tmp1 += (tmp1 >> 64) | (tmp1 << 64);
++		tmp3 = ((tmp3 >> 64) << 64) | (tmp4 >> 64);
++		tmp3 += (tmp3 >> 64) | (tmp3 << 64);
++		tmp1 = ((tmp1 >> 64) << 64) | (tmp3 >> 64);
++		tmp1 += (tmp1 >> 64) | (tmp1 << 64);
++		tmp1 = ((tmp1 >> 64) << 64) | sum64;
++		tmp1 += (tmp1 >> 64) | (tmp1 << 64);
++		sum64 = tmp1 >> 64;
++	}
++	while (len > 8) {
++		__uint128_t tmp;
++
++		sum64 = accumulate(sum64, data);
++		tmp = READ_ONCE_NOCHECK(*(__uint128_t *)ptr);
++
++		len -= 16;
++		ptr += 2;
++
++#ifdef __LITTLE_ENDIAN
++		data = tmp >> 64;
++		sum64 = accumulate(sum64, tmp);
++#else
++		data = tmp;
++		sum64 = accumulate(sum64, tmp >> 64);
++#endif
++	}
++	if (len > 0) {
++		sum64 = accumulate(sum64, data);
++		data = READ_ONCE_NOCHECK(*ptr);
++		len -= 8;
++	}
++	/*
++	 * Tail: zero any over-read bytes similarly to the head, again
++	 * preserving odd/even alignment.
++	 */
++	shift = len * -8;
++#ifdef __LITTLE_ENDIAN
++	data = (data << shift) >> shift;
++#else
++	data = (data >> shift) << shift;
++#endif
++	sum64 = accumulate(sum64, data);
++
++	/* Finally, folding */
++	sum64 += (sum64 >> 32) | (sum64 << 32);
++	sum = sum64 >> 32;
++	sum += (sum >> 16) | (sum << 16);
++	if (offset & 1)
++		return (u16)swab32(sum);
++
++	return sum >> 16;
++}
diff --git a/target/linux/generic/backport-5.4/042-v5.5-arm64-csum-Fix-pathological-zero-length-calls.patch b/target/linux/generic/backport-5.4/042-v5.5-arm64-csum-Fix-pathological-zero-length-calls.patch
new file mode 100644
index 0000000000..50b210e14f
--- /dev/null
+++ b/target/linux/generic/backport-5.4/042-v5.5-arm64-csum-Fix-pathological-zero-length-calls.patch
@@ -0,0 +1,28 @@
+From: Robin Murphy <robin.murphy@arm.com>
+Date: Fri, 17 Jan 2020 15:48:39 +0000
+Subject: [PATCH] arm64: csum: Fix pathological zero-length calls
+
+In validating the checksumming results of the new routine, I sadly
+neglected to test its not-checksumming results. Thus it slipped through
+that the one case where @buff is already dword-aligned and @len = 0
+manages to defeat the tail-masking logic and behave as if @len = 8.
+For a zero length it doesn't make much sense to deference @buff anyway,
+so just add an early return (which has essentially zero impact on
+performance).
+
+Signed-off-by: Robin Murphy <robin.murphy@arm.com>
+Signed-off-by: Will Deacon <will@kernel.org>
+---
+
+--- a/arch/arm64/lib/csum.c
++++ b/arch/arm64/lib/csum.c
+@@ -20,6 +20,9 @@ unsigned int do_csum(const unsigned char
+ 	const u64 *ptr;
+ 	u64 data, sum64 = 0;
+ 
++	if (unlikely(len == 0))
++		return 0;
++
+ 	offset = (unsigned long)buff & 7;
+ 	/*
+ 	 * This is to all intents and purposes safe, since rounding down cannot