diff options
Diffstat (limited to 'target/linux/generic/backport-5.4/080-wireguard-0069-crypto-arm64-chacha-simplify-tail-block-handling.patch')
| -rw-r--r-- | target/linux/generic/backport-5.4/080-wireguard-0069-crypto-arm64-chacha-simplify-tail-block-handling.patch | 324 |
1 files changed, 324 insertions, 0 deletions
diff --git a/target/linux/generic/backport-5.4/080-wireguard-0069-crypto-arm64-chacha-simplify-tail-block-handling.patch b/target/linux/generic/backport-5.4/080-wireguard-0069-crypto-arm64-chacha-simplify-tail-block-handling.patch new file mode 100644 index 0000000000..55e16247d9 --- /dev/null +++ b/target/linux/generic/backport-5.4/080-wireguard-0069-crypto-arm64-chacha-simplify-tail-block-handling.patch @@ -0,0 +1,324 @@ +From af8c75e27b20e01464aa6ad43ca3095534c81a8b Mon Sep 17 00:00:00 2001 +From: Ard Biesheuvel <ardb@kernel.org> +Date: Fri, 6 Nov 2020 17:39:38 +0100 +Subject: [PATCH 069/124] crypto: arm64/chacha - simplify tail block handling + +commit c4fc6328d6c67690a7e6e03f43a5a976a13120ef upstream. + +Based on lessons learnt from optimizing the 32-bit version of this driver, +we can simplify the arm64 version considerably, by reordering the final +two stores when the last block is not a multiple of 64 bytes. This removes +the need to use permutation instructions to calculate the elements that are +clobbered by the final overlapping store, given that the store of the +penultimate block now follows it, and that one carries the correct values +for those elements already. + +While at it, simplify the overlapping loads as well, by calculating the +address of the final overlapping load upfront, and switching to this +address for every load that would otherwise extend past the end of the +source buffer. + +There is no impact on performance, but the resulting code is substantially +smaller and easier to follow. + +Cc: Eric Biggers <ebiggers@google.com> +Cc: "Jason A . Donenfeld" <Jason@zx2c4.com> +Signed-off-by: Ard Biesheuvel <ardb@kernel.org> +Signed-off-by: Herbert Xu <herbert@gondor.apana.org.au> +Signed-off-by: Jason A. Donenfeld <Jason@zx2c4.com> +--- + arch/arm64/crypto/chacha-neon-core.S | 193 ++++++++++----------------- + 1 file changed, 69 insertions(+), 124 deletions(-) + +--- a/arch/arm64/crypto/chacha-neon-core.S ++++ b/arch/arm64/crypto/chacha-neon-core.S +@@ -195,7 +195,6 @@ ENTRY(chacha_4block_xor_neon) + adr_l x10, .Lpermute + and x5, x4, #63 + add x10, x10, x5 +- add x11, x10, #64 + + // + // This function encrypts four consecutive ChaCha blocks by loading +@@ -645,11 +644,11 @@ CPU_BE( rev a15, a15 ) + zip2 v31.4s, v14.4s, v15.4s + eor a15, a15, w9 + +- mov x3, #64 ++ add x3, x2, x4 ++ sub x3, x3, #128 // start of last block ++ + subs x5, x4, #128 +- add x6, x5, x2 +- csel x3, x3, xzr, ge +- csel x2, x2, x6, ge ++ csel x2, x2, x3, ge + + // interleave 64-bit words in state n, n+2 + zip1 v0.2d, v16.2d, v18.2d +@@ -658,13 +657,10 @@ CPU_BE( rev a15, a15 ) + zip1 v8.2d, v17.2d, v19.2d + zip2 v12.2d, v17.2d, v19.2d + stp a2, a3, [x1, #-56] +- ld1 {v16.16b-v19.16b}, [x2], x3 + + subs x6, x4, #192 +- ccmp x3, xzr, #4, lt +- add x7, x6, x2 +- csel x3, x3, xzr, eq +- csel x2, x2, x7, eq ++ ld1 {v16.16b-v19.16b}, [x2], #64 ++ csel x2, x2, x3, ge + + zip1 v1.2d, v20.2d, v22.2d + zip2 v5.2d, v20.2d, v22.2d +@@ -672,13 +668,10 @@ CPU_BE( rev a15, a15 ) + zip1 v9.2d, v21.2d, v23.2d + zip2 v13.2d, v21.2d, v23.2d + stp a6, a7, [x1, #-40] +- ld1 {v20.16b-v23.16b}, [x2], x3 + + subs x7, x4, #256 +- ccmp x3, xzr, #4, lt +- add x8, x7, x2 +- csel x3, x3, xzr, eq +- csel x2, x2, x8, eq ++ ld1 {v20.16b-v23.16b}, [x2], #64 ++ csel x2, x2, x3, ge + + zip1 v2.2d, v24.2d, v26.2d + zip2 v6.2d, v24.2d, v26.2d +@@ -686,12 +679,10 @@ CPU_BE( rev a15, a15 ) + zip1 v10.2d, v25.2d, v27.2d + zip2 v14.2d, v25.2d, v27.2d + stp a10, a11, [x1, #-24] +- ld1 {v24.16b-v27.16b}, [x2], x3 + + subs x8, x4, #320 +- ccmp x3, xzr, #4, lt +- add x9, x8, x2 +- csel x2, x2, x9, eq ++ ld1 {v24.16b-v27.16b}, [x2], #64 ++ csel x2, x2, x3, ge + + zip1 v3.2d, v28.2d, v30.2d + zip2 v7.2d, v28.2d, v30.2d +@@ -699,151 +690,105 @@ CPU_BE( rev a15, a15 ) + zip1 v11.2d, v29.2d, v31.2d + zip2 v15.2d, v29.2d, v31.2d + stp a14, a15, [x1, #-8] ++ ++ tbnz x5, #63, .Lt128 + ld1 {v28.16b-v31.16b}, [x2] + + // xor with corresponding input, write to output +- tbnz x5, #63, 0f + eor v16.16b, v16.16b, v0.16b + eor v17.16b, v17.16b, v1.16b + eor v18.16b, v18.16b, v2.16b + eor v19.16b, v19.16b, v3.16b +- st1 {v16.16b-v19.16b}, [x1], #64 +- cbz x5, .Lout + +- tbnz x6, #63, 1f ++ tbnz x6, #63, .Lt192 ++ + eor v20.16b, v20.16b, v4.16b + eor v21.16b, v21.16b, v5.16b + eor v22.16b, v22.16b, v6.16b + eor v23.16b, v23.16b, v7.16b +- st1 {v20.16b-v23.16b}, [x1], #64 +- cbz x6, .Lout + +- tbnz x7, #63, 2f ++ st1 {v16.16b-v19.16b}, [x1], #64 ++ tbnz x7, #63, .Lt256 ++ + eor v24.16b, v24.16b, v8.16b + eor v25.16b, v25.16b, v9.16b + eor v26.16b, v26.16b, v10.16b + eor v27.16b, v27.16b, v11.16b +- st1 {v24.16b-v27.16b}, [x1], #64 +- cbz x7, .Lout + +- tbnz x8, #63, 3f ++ st1 {v20.16b-v23.16b}, [x1], #64 ++ tbnz x8, #63, .Lt320 ++ + eor v28.16b, v28.16b, v12.16b + eor v29.16b, v29.16b, v13.16b + eor v30.16b, v30.16b, v14.16b + eor v31.16b, v31.16b, v15.16b ++ ++ st1 {v24.16b-v27.16b}, [x1], #64 + st1 {v28.16b-v31.16b}, [x1] + + .Lout: frame_pop + ret + +- // fewer than 128 bytes of in/output +-0: ld1 {v8.16b}, [x10] +- ld1 {v9.16b}, [x11] +- movi v10.16b, #16 +- sub x2, x1, #64 +- add x1, x1, x5 +- ld1 {v16.16b-v19.16b}, [x2] +- tbl v4.16b, {v0.16b-v3.16b}, v8.16b +- tbx v20.16b, {v16.16b-v19.16b}, v9.16b +- add v8.16b, v8.16b, v10.16b +- add v9.16b, v9.16b, v10.16b +- tbl v5.16b, {v0.16b-v3.16b}, v8.16b +- tbx v21.16b, {v16.16b-v19.16b}, v9.16b +- add v8.16b, v8.16b, v10.16b +- add v9.16b, v9.16b, v10.16b +- tbl v6.16b, {v0.16b-v3.16b}, v8.16b +- tbx v22.16b, {v16.16b-v19.16b}, v9.16b +- add v8.16b, v8.16b, v10.16b +- add v9.16b, v9.16b, v10.16b +- tbl v7.16b, {v0.16b-v3.16b}, v8.16b +- tbx v23.16b, {v16.16b-v19.16b}, v9.16b +- +- eor v20.16b, v20.16b, v4.16b +- eor v21.16b, v21.16b, v5.16b +- eor v22.16b, v22.16b, v6.16b +- eor v23.16b, v23.16b, v7.16b +- st1 {v20.16b-v23.16b}, [x1] +- b .Lout +- + // fewer than 192 bytes of in/output +-1: ld1 {v8.16b}, [x10] +- ld1 {v9.16b}, [x11] +- movi v10.16b, #16 +- add x1, x1, x6 +- tbl v0.16b, {v4.16b-v7.16b}, v8.16b +- tbx v20.16b, {v16.16b-v19.16b}, v9.16b +- add v8.16b, v8.16b, v10.16b +- add v9.16b, v9.16b, v10.16b +- tbl v1.16b, {v4.16b-v7.16b}, v8.16b +- tbx v21.16b, {v16.16b-v19.16b}, v9.16b +- add v8.16b, v8.16b, v10.16b +- add v9.16b, v9.16b, v10.16b +- tbl v2.16b, {v4.16b-v7.16b}, v8.16b +- tbx v22.16b, {v16.16b-v19.16b}, v9.16b +- add v8.16b, v8.16b, v10.16b +- add v9.16b, v9.16b, v10.16b +- tbl v3.16b, {v4.16b-v7.16b}, v8.16b +- tbx v23.16b, {v16.16b-v19.16b}, v9.16b +- +- eor v20.16b, v20.16b, v0.16b +- eor v21.16b, v21.16b, v1.16b +- eor v22.16b, v22.16b, v2.16b +- eor v23.16b, v23.16b, v3.16b +- st1 {v20.16b-v23.16b}, [x1] ++.Lt192: cbz x5, 1f // exactly 128 bytes? ++ ld1 {v28.16b-v31.16b}, [x10] ++ add x5, x5, x1 ++ tbl v28.16b, {v4.16b-v7.16b}, v28.16b ++ tbl v29.16b, {v4.16b-v7.16b}, v29.16b ++ tbl v30.16b, {v4.16b-v7.16b}, v30.16b ++ tbl v31.16b, {v4.16b-v7.16b}, v31.16b ++ ++0: eor v20.16b, v20.16b, v28.16b ++ eor v21.16b, v21.16b, v29.16b ++ eor v22.16b, v22.16b, v30.16b ++ eor v23.16b, v23.16b, v31.16b ++ st1 {v20.16b-v23.16b}, [x5] // overlapping stores ++1: st1 {v16.16b-v19.16b}, [x1] + b .Lout + ++ // fewer than 128 bytes of in/output ++.Lt128: ld1 {v28.16b-v31.16b}, [x10] ++ add x5, x5, x1 ++ sub x1, x1, #64 ++ tbl v28.16b, {v0.16b-v3.16b}, v28.16b ++ tbl v29.16b, {v0.16b-v3.16b}, v29.16b ++ tbl v30.16b, {v0.16b-v3.16b}, v30.16b ++ tbl v31.16b, {v0.16b-v3.16b}, v31.16b ++ ld1 {v16.16b-v19.16b}, [x1] // reload first output block ++ b 0b ++ + // fewer than 256 bytes of in/output +-2: ld1 {v4.16b}, [x10] +- ld1 {v5.16b}, [x11] +- movi v6.16b, #16 +- add x1, x1, x7 ++.Lt256: cbz x6, 2f // exactly 192 bytes? ++ ld1 {v4.16b-v7.16b}, [x10] ++ add x6, x6, x1 + tbl v0.16b, {v8.16b-v11.16b}, v4.16b +- tbx v24.16b, {v20.16b-v23.16b}, v5.16b +- add v4.16b, v4.16b, v6.16b +- add v5.16b, v5.16b, v6.16b +- tbl v1.16b, {v8.16b-v11.16b}, v4.16b +- tbx v25.16b, {v20.16b-v23.16b}, v5.16b +- add v4.16b, v4.16b, v6.16b +- add v5.16b, v5.16b, v6.16b +- tbl v2.16b, {v8.16b-v11.16b}, v4.16b +- tbx v26.16b, {v20.16b-v23.16b}, v5.16b +- add v4.16b, v4.16b, v6.16b +- add v5.16b, v5.16b, v6.16b +- tbl v3.16b, {v8.16b-v11.16b}, v4.16b +- tbx v27.16b, {v20.16b-v23.16b}, v5.16b +- +- eor v24.16b, v24.16b, v0.16b +- eor v25.16b, v25.16b, v1.16b +- eor v26.16b, v26.16b, v2.16b +- eor v27.16b, v27.16b, v3.16b +- st1 {v24.16b-v27.16b}, [x1] ++ tbl v1.16b, {v8.16b-v11.16b}, v5.16b ++ tbl v2.16b, {v8.16b-v11.16b}, v6.16b ++ tbl v3.16b, {v8.16b-v11.16b}, v7.16b ++ ++ eor v28.16b, v28.16b, v0.16b ++ eor v29.16b, v29.16b, v1.16b ++ eor v30.16b, v30.16b, v2.16b ++ eor v31.16b, v31.16b, v3.16b ++ st1 {v28.16b-v31.16b}, [x6] // overlapping stores ++2: st1 {v20.16b-v23.16b}, [x1] + b .Lout + + // fewer than 320 bytes of in/output +-3: ld1 {v4.16b}, [x10] +- ld1 {v5.16b}, [x11] +- movi v6.16b, #16 +- add x1, x1, x8 ++.Lt320: cbz x7, 3f // exactly 256 bytes? ++ ld1 {v4.16b-v7.16b}, [x10] ++ add x7, x7, x1 + tbl v0.16b, {v12.16b-v15.16b}, v4.16b +- tbx v28.16b, {v24.16b-v27.16b}, v5.16b +- add v4.16b, v4.16b, v6.16b +- add v5.16b, v5.16b, v6.16b +- tbl v1.16b, {v12.16b-v15.16b}, v4.16b +- tbx v29.16b, {v24.16b-v27.16b}, v5.16b +- add v4.16b, v4.16b, v6.16b +- add v5.16b, v5.16b, v6.16b +- tbl v2.16b, {v12.16b-v15.16b}, v4.16b +- tbx v30.16b, {v24.16b-v27.16b}, v5.16b +- add v4.16b, v4.16b, v6.16b +- add v5.16b, v5.16b, v6.16b +- tbl v3.16b, {v12.16b-v15.16b}, v4.16b +- tbx v31.16b, {v24.16b-v27.16b}, v5.16b ++ tbl v1.16b, {v12.16b-v15.16b}, v5.16b ++ tbl v2.16b, {v12.16b-v15.16b}, v6.16b ++ tbl v3.16b, {v12.16b-v15.16b}, v7.16b + + eor v28.16b, v28.16b, v0.16b + eor v29.16b, v29.16b, v1.16b + eor v30.16b, v30.16b, v2.16b + eor v31.16b, v31.16b, v3.16b +- st1 {v28.16b-v31.16b}, [x1] ++ st1 {v28.16b-v31.16b}, [x7] // overlapping stores ++3: st1 {v24.16b-v27.16b}, [x1] + b .Lout + ENDPROC(chacha_4block_xor_neon) + +@@ -851,7 +796,7 @@ ENDPROC(chacha_4block_xor_neon) + .align L1_CACHE_SHIFT + .Lpermute: + .set .Li, 0 +- .rept 192 ++ .rept 128 + .byte (.Li - 64) + .set .Li, .Li + 1 + .endr |